You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2018/08/02 13:40:13 UTC
[airavata] branch develop updated: Load generating client initial
version
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/develop by this push:
new 6118d04 Load generating client initial version
6118d04 is described below
commit 6118d045b43f6d630770e2e41b096302af53d05e
Author: dimuthu <di...@gmail.com>
AuthorDate: Thu Aug 2 09:40:01 2018 -0400
Load generating client initial version
---
.../apache/airavata/agents/api/AgentAdaptor.java | 2 +
.../airavata/helix/agent/ssh/SshAgentAdaptor.java | 5 +
.../airavata/helix/adaptor/SSHJAgentAdaptor.java | 14 ++
pom.xml | 1 +
tools/gsissh-cli-tools/pom.xml | 2 +-
tools/load-client/pom.xml | 76 +++++++++
.../src/main/assembly/load-client-bin-assembly.xml | 82 +++++++++
.../apache/airavata/tools/load/Configuration.java | 186 +++++++++++++++++++++
.../apache/airavata/tools/load/Configurations.java | 17 ++
.../org/apache/airavata/tools/load/LoadClient.java | 140 ++++++++++++++++
.../airavata/tools/load/SecurityManager.java | 72 ++++++++
.../tools/load/StorageResourceManager.java | 96 +++++++++++
.../org/apache/airavata/tools/load/UnitLoad.java | 150 +++++++++++++++++
.../src/main/resources/bin/load-client.sh | 76 +++++++++
tools/load-client/src/main/resources/bin/setenv.sh | 46 +++++
.../src/main/resources/client_truststore.jks | Bin 0 -> 5312 bytes
.../src/main/resources/conf/load-config.yml | 22 +++
tools/phoebus-integration/pom.xml | 2 +-
tools/pom.xml | 5 +-
19 files changed, 989 insertions(+), 5 deletions(-)
diff --git a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java
index bbeaf92..5355d5c 100644
--- a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java
+++ b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java
@@ -32,6 +32,8 @@ public interface AgentAdaptor {
public void init(String computeResource, String gatewayId, String userId, String token) throws AgentException;
+ public void destroy();
+
public CommandOutput executeCommand(String command, String workingDirectory) throws AgentException;
public void createDirectory(String path) throws AgentException;
diff --git a/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java b/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java
index ffa9109c..e4cffe9 100644
--- a/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java
+++ b/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java
@@ -106,6 +106,11 @@ public class SshAgentAdaptor implements AgentAdaptor {
}
}
+ @Override
+ public void destroy() {
+
+ }
+
public CommandOutput executeCommand(String command, String workingDirectory) throws AgentException {
StandardOutReader commandOutput = new StandardOutReader();
ChannelExec channelExec = null;
diff --git a/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java b/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java
index 4349844..791724a 100644
--- a/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java
+++ b/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java
@@ -128,6 +128,20 @@ public class SSHJAgentAdaptor implements AgentAdaptor {
}
@Override
+ public void destroy() {
+ try {
+ if (sshjClient != null) {
+ sshjClient.disconnect();
+ sshjClient.close();
+ }
+ } catch (IOException e) {
+ logger.warn("Failed to stop sshj client for host " + sshjClient.getHost() + " and user " +
+ sshjClient.getUsername() + " due to : " + e.getMessage());
+ // ignore
+ }
+ }
+
+ @Override
public CommandOutput executeCommand(String command, String workingDirectory) throws AgentException {
try (Session session = sshjClient.startSessionWrapper()) {
diff --git a/pom.xml b/pom.xml
index 29caf3f..d554eae 100644
--- a/pom.xml
+++ b/pom.xml
@@ -770,6 +770,7 @@
<module>modules/compute-account-provisioning</module>
<module>modules/airavata-helix</module>
<module>modules/job-monitor</module>
+ <module>tools</module>
</modules>
</profile>
<profile>
diff --git a/tools/gsissh-cli-tools/pom.xml b/tools/gsissh-cli-tools/pom.xml
index 0c5b020..b99f1ad 100644
--- a/tools/gsissh-cli-tools/pom.xml
+++ b/tools/gsissh-cli-tools/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.airavata</groupId>
<artifactId>airavata</artifactId>
- <version>0.14-SNAPSHOT</version>
+ <version>0.17-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/tools/load-client/pom.xml b/tools/load-client/pom.xml
new file mode 100644
index 0000000..56ac6a4
--- /dev/null
+++ b/tools/load-client/pom.xml
@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-tools-parent</artifactId>
+ <version>0.17-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>load-client</artifactId>
+ <name>Airavata Load Testing Client</name>
+ <description>Puts a load to Airavata through API</description>
+ <url>http://airavata.apache.org/</url>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-api-stubs</artifactId>
+ <version>0.17-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.jboss.resteasy</groupId>
+ <artifactId>resteasy-client</artifactId>
+ <version>3.0.14.Final</version>
+ </dependency>
+ <dependency>
+ <groupId>org.keycloak</groupId>
+ <artifactId>keycloak-admin-client</artifactId>
+ <version>2.5.5.Final</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>sshj-agent</artifactId>
+ <version>0.17-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <version>1.15</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>1.2</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.5.5</version>
+ <executions>
+ <execution>
+ <id>load-client-distribution-package</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <tarLongFileMode>posix</tarLongFileMode>
+ <finalName>load-client-${project.version}</finalName>
+ <descriptors>
+ <descriptor>src/main/assembly/load-client-bin-assembly.xml</descriptor>
+ </descriptors>
+ <attach>false</attach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/tools/load-client/src/main/assembly/load-client-bin-assembly.xml b/tools/load-client/src/main/assembly/load-client-bin-assembly.xml
new file mode 100644
index 0000000..60d7f7f
--- /dev/null
+++ b/tools/load-client/src/main/assembly/load-client-bin-assembly.xml
@@ -0,0 +1,82 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<!DOCTYPE assembly [
+ <!ELEMENT assembly (id|includeBaseDirectory|baseDirectory|formats|fileSets|dependencySets)*>
+ <!ELEMENT id (#PCDATA)>
+ <!ELEMENT includeBaseDirectory (#PCDATA)>
+ <!ELEMENT baseDirectory (#PCDATA)>
+ <!ELEMENT formats (format)*>
+ <!ELEMENT format (#PCDATA)>
+ <!ELEMENT fileSets (fileSet)*>
+ <!ELEMENT fileSet (directory|outputDirectory|fileMode|includes)*>
+ <!ELEMENT directory (#PCDATA)>
+ <!ELEMENT outputDirectory (#PCDATA)>
+ <!ELEMENT includes (include)*>
+ <!ELEMENT include (#PCDATA)>
+ <!ELEMENT dependencySets (dependencySet)*>
+ <!ELEMENT dependencySet (outputDirectory|outputFileNameMapping|includes)*>
+ ]>
+<assembly>
+ <id>bin</id>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <baseDirectory>load-client</baseDirectory>
+ <formats>
+ <format>tar.gz</format>
+ <format>zip</format>
+ </formats>
+
+ <fileSets>
+ <fileSet>
+ <directory>src/main/resources/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <fileMode>777</fileMode>
+ <includes>
+ <include>load-client.sh</include>
+ <include>setenv.sh</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/resources</directory>
+ <outputDirectory>bin</outputDirectory>
+ <fileMode>777</fileMode>
+ <includes>
+ <include>client_truststore.jks</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/resources/conf</directory>
+ <outputDirectory>conf</outputDirectory>
+ <includes>
+ <include>load-config.yml</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>true</useProjectArtifact>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>*:*:jar</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
diff --git a/tools/load-client/src/main/java/org/apache/airavata/tools/load/Configuration.java b/tools/load-client/src/main/java/org/apache/airavata/tools/load/Configuration.java
new file mode 100644
index 0000000..c8484ea
--- /dev/null
+++ b/tools/load-client/src/main/java/org/apache/airavata/tools/load/Configuration.java
@@ -0,0 +1,186 @@
+package org.apache.airavata.tools.load;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class Configuration {
+
+ private String userId;
+
+ private String gatewayId;
+ private String projectId;
+ private String applicationInterfaceId;
+ private String computeResourceId;
+ private String storageResourceId;
+
+ private String experimentBaseName;
+
+ private String queue;
+ private int wallTime;
+ private int cpuCount;
+ private int nodeCount;
+ private int physicalMemory;
+
+ private int concurrentUsers;
+ private int iterationsPerUser;
+ private int randomMSDelayWithinSubmissions;
+
+ private List<Input> inputs = new ArrayList<>();
+
+ public String getUserId() {
+ return userId;
+ }
+
+ public void setUserId(String userId) {
+ this.userId = userId;
+ }
+
+ public String getGatewayId() {
+ return gatewayId;
+ }
+
+ public void setGatewayId(String gatewayId) {
+ this.gatewayId = gatewayId;
+ }
+
+ public String getProjectId() {
+ return projectId;
+ }
+
+ public void setProjectId(String projectId) {
+ this.projectId = projectId;
+ }
+
+ public String getApplicationInterfaceId() {
+ return applicationInterfaceId;
+ }
+
+ public void setApplicationInterfaceId(String applicationInterfaceId) {
+ this.applicationInterfaceId = applicationInterfaceId;
+ }
+
+ public String getComputeResourceId() {
+ return computeResourceId;
+ }
+
+ public void setComputeResourceId(String computeResourceId) {
+ this.computeResourceId = computeResourceId;
+ }
+
+ public String getStorageResourceId() {
+ return storageResourceId;
+ }
+
+ public void setStorageResourceId(String storageResourceId) {
+ this.storageResourceId = storageResourceId;
+ }
+
+ public String getExperimentBaseName() {
+ return experimentBaseName;
+ }
+
+ public void setExperimentBaseName(String experimentBaseName) {
+ this.experimentBaseName = experimentBaseName;
+ }
+
+ public String getQueue() {
+ return queue;
+ }
+
+ public void setQueue(String queue) {
+ this.queue = queue;
+ }
+
+ public int getWallTime() {
+ return wallTime;
+ }
+
+ public void setWallTime(int wallTime) {
+ this.wallTime = wallTime;
+ }
+
+ public int getCpuCount() {
+ return cpuCount;
+ }
+
+ public void setCpuCount(int cpuCount) {
+ this.cpuCount = cpuCount;
+ }
+
+ public int getNodeCount() {
+ return nodeCount;
+ }
+
+ public void setNodeCount(int nodeCount) {
+ this.nodeCount = nodeCount;
+ }
+
+ public int getPhysicalMemory() {
+ return physicalMemory;
+ }
+
+ public void setPhysicalMemory(int physicalMemory) {
+ this.physicalMemory = physicalMemory;
+ }
+
+ public int getConcurrentUsers() {
+ return concurrentUsers;
+ }
+
+ public void setConcurrentUsers(int concurrentUsers) {
+ this.concurrentUsers = concurrentUsers;
+ }
+
+ public int getIterationsPerUser() {
+ return iterationsPerUser;
+ }
+
+ public void setIterationsPerUser(int iterationsPerUser) {
+ this.iterationsPerUser = iterationsPerUser;
+ }
+
+ public int getRandomMSDelayWithinSubmissions() {
+ return randomMSDelayWithinSubmissions;
+ }
+
+ public void setRandomMSDelayWithinSubmissions(int randomMSDelayWithinSubmissions) {
+ this.randomMSDelayWithinSubmissions = randomMSDelayWithinSubmissions;
+ }
+
+ public List<Input> getInputs() {
+ return inputs;
+ }
+
+ public void setInputs(List<Input> inputs) {
+ this.inputs = inputs;
+ }
+
+ public static class Input {
+ private String name;
+ private String value;
+
+ public Input() {
+ }
+
+ public Input(String name, String value) {
+ this.name = name;
+ this.value = value;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public void setValue(String value) {
+ this.value = value;
+ }
+ }
+}
diff --git a/tools/load-client/src/main/java/org/apache/airavata/tools/load/Configurations.java b/tools/load-client/src/main/java/org/apache/airavata/tools/load/Configurations.java
new file mode 100644
index 0000000..b267f1f
--- /dev/null
+++ b/tools/load-client/src/main/java/org/apache/airavata/tools/load/Configurations.java
@@ -0,0 +1,17 @@
+package org.apache.airavata.tools.load;
+
+import org.apache.airavata.tools.load.Configuration;
+
+import java.util.List;
+
+public class Configurations {
+ private List<Configuration> configurations;
+
+ public List<Configuration> getConfigurations() {
+ return configurations;
+ }
+
+ public void setConfigurations(List<Configuration> configurations) {
+ this.configurations = configurations;
+ }
+}
diff --git a/tools/load-client/src/main/java/org/apache/airavata/tools/load/LoadClient.java b/tools/load-client/src/main/java/org/apache/airavata/tools/load/LoadClient.java
new file mode 100644
index 0000000..283499a
--- /dev/null
+++ b/tools/load-client/src/main/java/org/apache/airavata/tools/load/LoadClient.java
@@ -0,0 +1,140 @@
+package org.apache.airavata.tools.load;
+
+import org.apache.airavata.api.Airavata;
+import org.apache.airavata.api.client.AiravataClientFactory;
+import org.apache.airavata.model.appcatalog.gatewayprofile.StoragePreference;
+import org.apache.airavata.model.appcatalog.storageresource.StorageResourceDescription;
+import org.apache.airavata.model.security.AuthzToken;
+import org.apache.commons.cli.*;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CompletionService;
+
+public class LoadClient {
+
+ private String apiHost;
+ private int apiPort = 9930;
+
+ private String privateKeyFile = System.getProperty("user.home") + "/.ssh/id_rsa";
+ private String publicKeyFile = System.getProperty("user.home") + "/.ssh/id_rsa.pub";
+ private String passPhrase = null;
+ private String configFile;
+
+ private SecurityManager securityManager = new SecurityManager();
+ private Map<String, StorageResourceManager> storageResourceManagerStore = new HashMap<>();
+ private Configurations configurations;
+
+ public void init() throws Exception {
+ securityManager.loadCertificate(apiHost, apiPort);
+
+ if (configFile == null) {
+ try (InputStream in = LoadClient.class.getResourceAsStream("/conf/load-config.yml")) {
+ Yaml yaml = new Yaml();
+ configurations = yaml.loadAs(in, Configurations.class);
+ createStorageResourceManagers(configurations);
+ }
+ } else {
+ try (InputStream in = new FileInputStream(configFile)) {
+ Yaml yaml = new Yaml();
+ configurations = yaml.loadAs(in, Configurations.class);
+ createStorageResourceManagers(configurations);
+ }
+ }
+ }
+
+ public void start() throws Exception {
+ for (Configuration configuration : configurations.getConfigurations()) {
+ UnitLoad unitLoad = new UnitLoad(apiHost, apiPort, securityManager.getTrustStorePath(), securityManager.getTrustStorePassword(),
+ storageResourceManagerStore.get(configuration.getStorageResourceId()));
+ CompletionService<Boolean> completion = unitLoad.execute(configuration);
+
+ for (int i = 0; i < configuration.getConcurrentUsers(); i++) {
+ completion.take();
+ }
+ }
+ destroyStorageResourceManagers();
+ System.out.println("Finished load");
+ System.exit(0);
+ }
+
+ private void createStorageResourceManagers(Configurations configurations) throws Exception {
+
+ Airavata.Client airavataClient = AiravataClientFactory.createAiravataSecureClient(apiHost, apiPort,
+ securityManager.getTrustStorePath(), securityManager.getTrustStorePassword(), 100000);
+
+ for (Configuration configuration : configurations.getConfigurations()) {
+ String storageResourceId = configuration.getStorageResourceId();
+
+ if (!storageResourceManagerStore.containsKey(storageResourceId)) {
+ StorageResourceDescription storageResource = airavataClient.getStorageResource(new AuthzToken(""), storageResourceId);
+ StoragePreference gatewayStoragePreference = airavataClient.getGatewayStoragePreference(new AuthzToken(""), configuration.getGatewayId(), storageResourceId);
+ StorageResourceManager storageResourceManager = new StorageResourceManager(gatewayStoragePreference, storageResource, privateKeyFile, publicKeyFile, passPhrase);
+ storageResourceManager.init();
+ storageResourceManagerStore.put(storageResourceId, storageResourceManager);
+ }
+ }
+ }
+
+ private void destroyStorageResourceManagers() {
+ storageResourceManagerStore.values().forEach(StorageResourceManager::destroy);
+ }
+
+ public static void main(String args[]) throws Exception {
+
+ Options options = new Options();
+ options.addOption("config", true, "Load configuration file in yaml format");
+ options.addOption("apiHost", true, "API Server host name");
+ options.addOption("apiPort", true, "API Server port");
+ options.addOption("privateKeyPath", true, "SSH private key path to communicate with storage resources (Defaults to user private key in ~/.ssh/id_rsa)");
+ options.addOption("publicKeyPath", true, "SSH public key path to communicate with storage resources (Defaults to user public key in ~/.ssh/id_rsa.pub)");
+ options.addOption("passPhrase", true, "SSH private key pass phrase (if any)");
+
+ CommandLineParser parser = new GnuParser();
+ CommandLine cmd = parser.parse( options, args);
+
+ LoadClient loadClient = new LoadClient();
+
+ if (cmd.hasOption("config")) {
+ loadClient.configFile = cmd.getOptionValue("config");
+ } else {
+ System.out.println("Error : Load config file should be specified");
+ System.exit(0);
+ }
+
+ if (cmd.hasOption("apiHost")) {
+ loadClient.apiHost = cmd.getOptionValue("apiHost");
+ } else {
+ System.out.println("Error : API host should be specified");
+ System.exit(0);
+ }
+
+ if (cmd.hasOption("apiPort")) {
+ loadClient.apiPort = Integer.parseInt(cmd.getOptionValue("apiPort"));
+ } else {
+ System.out.println("Using default API port " + loadClient.apiPort);
+ }
+
+ if (cmd.hasOption("privateKeyPath")) {
+ loadClient.privateKeyFile = cmd.getOptionValue("privateKeyPath");
+ } else {
+ System.out.println("Using default private key file " + loadClient.privateKeyFile);
+ }
+
+ if (cmd.hasOption("publicKeyPath")) {
+ loadClient.publicKeyFile = cmd.getOptionValue("publicKeyPath");
+ } else {
+ System.out.println("Using default public key file " + loadClient.publicKeyFile);
+ }
+
+ if (cmd.hasOption("passPhrase")) {
+ loadClient.passPhrase = cmd.getOptionValue("passPhrase");
+ }
+
+ loadClient.init();
+ loadClient.start();
+ }
+}
diff --git a/tools/load-client/src/main/java/org/apache/airavata/tools/load/SecurityManager.java b/tools/load-client/src/main/java/org/apache/airavata/tools/load/SecurityManager.java
new file mode 100644
index 0000000..410c2af
--- /dev/null
+++ b/tools/load-client/src/main/java/org/apache/airavata/tools/load/SecurityManager.java
@@ -0,0 +1,72 @@
+package org.apache.airavata.tools.load;
+
+import javax.net.ssl.*;
+import java.io.*;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.security.*;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+
+public class SecurityManager {
+
+ private String trustStoreName = "client_truststore.jks";
+ private String trustStorePassword = "airavata";
+
+ public void loadCertificate(String host, int port) throws CertificateException, NoSuchAlgorithmException, IOException, KeyStoreException, KeyManagementException, URISyntaxException {
+
+ TrustManager[] trustAllCerts = new TrustManager[]{
+ new X509TrustManager() {
+ public java.security.cert.X509Certificate[] getAcceptedIssuers() {
+ return null;
+ }
+ public void checkClientTrusted(
+ java.security.cert.X509Certificate[] certs, String authType) {
+ }
+ public void checkServerTrusted(
+ java.security.cert.X509Certificate[] certs, String authType) {
+ }
+ }
+ };
+
+ SSLContext sslContext = SSLContext.getInstance("SSL");
+ sslContext.init(null, trustAllCerts, new java.security.SecureRandom());
+ SSLSocket socket = (SSLSocket) sslContext.getSocketFactory().createSocket(host, port);
+ socket.startHandshake();
+ SSLSession sslSession = socket.getSession();
+ Certificate[] certificates = sslSession.getPeerCertificates();
+
+ FileInputStream is = new FileInputStream(getTrustStorePath());
+
+ KeyStore keystore = KeyStore.getInstance(KeyStore.getDefaultType());
+ keystore.load(is, trustStorePassword.toCharArray());
+ is.close();
+
+ File keystoreFile = new File(getTrustStorePath());
+
+ String certificateAlias = host;
+ keystore.setCertificateEntry(certificateAlias, certificates[0]);
+
+ FileOutputStream out = new FileOutputStream(keystoreFile);
+ keystore.store(out, trustStorePassword.toCharArray());
+ out.close();
+
+ System.out.println("Certificates successfully loaded for " + host + ":" + port);
+ }
+
+ public String getTrustStorePath() throws URISyntaxException {
+ URL trustStoreUrl = SecurityManager.class.getClassLoader().getResource(trustStoreName);
+
+ String trustStorePath;
+ if (trustStoreUrl.toURI().getPath() != null) {
+ trustStorePath = trustStoreUrl.toURI().getPath();
+ } else {
+ trustStorePath = System.getProperty("airavata.home") + "/bin/" + trustStoreName;
+ }
+ return trustStorePath;
+ }
+
+ public String getTrustStorePassword() {
+ return trustStorePassword;
+ }
+}
diff --git a/tools/load-client/src/main/java/org/apache/airavata/tools/load/StorageResourceManager.java b/tools/load-client/src/main/java/org/apache/airavata/tools/load/StorageResourceManager.java
new file mode 100644
index 0000000..7f7527f
--- /dev/null
+++ b/tools/load-client/src/main/java/org/apache/airavata/tools/load/StorageResourceManager.java
@@ -0,0 +1,96 @@
+package org.apache.airavata.tools.load;
+
+import org.apache.airavata.agents.api.AgentException;
+import org.apache.airavata.api.Airavata;
+import org.apache.airavata.helix.adaptor.SSHJStorageAdaptor;
+import org.apache.airavata.model.appcatalog.gatewayprofile.StoragePreference;
+import org.apache.airavata.model.appcatalog.storageresource.StorageResourceDescription;
+import org.apache.airavata.model.data.replica.*;
+import org.apache.airavata.model.security.AuthzToken;
+import org.apache.thrift.TException;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Collections;
+
+public class StorageResourceManager {
+
+ private StoragePreference gatewayStoragePreference;
+ private StorageResourceDescription storageResource;
+ private String storageResourceId;
+
+ private String privateKeyFile;
+ private String publicKeyFile;
+ private String passPhrase;
+
+ private SSHJStorageAdaptor storageAdaptor = new SSHJStorageAdaptor();
+
+ public StorageResourceManager(StoragePreference gatewayStoragePreference, StorageResourceDescription storageResource,
+ String privateKeyFile, String publicKeyFile, String passPhrase) {
+ this.storageResourceId = storageResource.getStorageResourceId();
+ this.storageResource = storageResource;
+ this.gatewayStoragePreference = gatewayStoragePreference;
+ this.privateKeyFile = privateKeyFile;
+ this.publicKeyFile = publicKeyFile;
+ this.passPhrase = passPhrase;
+ }
+
+ public void init() throws IOException, AgentException {
+ storageAdaptor.init(gatewayStoragePreference.getLoginUserName(),
+ storageResource.getHostName(),
+ readFile(publicKeyFile, Charset.defaultCharset()),
+ readFile(privateKeyFile, Charset.defaultCharset()),
+ passPhrase);
+ }
+
+ public void destroy() {
+ storageAdaptor.destroy();
+ }
+
+ public String uploadInputFile(Airavata.Client airavataClient, String filePath, String user, String project, String experiment, String gatewayId) throws TException, AgentException {
+
+ String experimentDirectory = getExperimentDirectory(user, project, experiment);
+
+ String uploadFilePath = experimentDirectory.concat(File.separator).concat(new File(filePath).getName());
+ storageAdaptor.uploadFile(filePath, uploadFilePath);
+
+ DataProductModel dataProductModel = new DataProductModel();
+ dataProductModel.setGatewayId(gatewayId);
+ dataProductModel.setOwnerName(user);
+ dataProductModel.setDataProductType(DataProductType.FILE);
+
+ DataReplicaLocationModel replicaLocationModel = new DataReplicaLocationModel();
+ replicaLocationModel.setStorageResourceId(storageResourceId);
+ replicaLocationModel.setReplicaName((new File(filePath).getName()) + " gateway data store copy");
+ replicaLocationModel.setReplicaLocationCategory(ReplicaLocationCategory.GATEWAY_DATA_STORE);
+ replicaLocationModel.setReplicaPersistentType(ReplicaPersistentType.TRANSIENT);
+ replicaLocationModel.setFilePath("file://" + storageResource.getHostName() + ":" + uploadFilePath);
+
+ dataProductModel.setReplicaLocations(Collections.singletonList(replicaLocationModel));
+ System.out.println("Registring " + uploadFilePath);
+ return airavataClient.registerDataProduct(new AuthzToken(""), dataProductModel);
+ }
+
+ public void createExperimentDirectory(String user, String project, String experiment) throws AgentException {
+ String experimentDirectory = getExperimentDirectory(user, project, experiment);
+ storageAdaptor.createDirectory(experimentDirectory, true);
+ }
+
+ private String getExperimentDirectory(String user, String project, String experiment) {
+ return gatewayStoragePreference.getFileSystemRootLocation()
+ .concat(File.separator)
+ .concat(user)
+ .concat(File.separator)
+ .concat(project)
+ .concat(File.separator)
+ .concat(experiment);
+ }
+
+ static String readFile(String path, Charset encoding) throws IOException {
+ byte[] encoded = Files.readAllBytes(Paths.get(path));
+ return new String(encoded, encoding);
+ }
+}
diff --git a/tools/load-client/src/main/java/org/apache/airavata/tools/load/UnitLoad.java b/tools/load-client/src/main/java/org/apache/airavata/tools/load/UnitLoad.java
new file mode 100644
index 0000000..97e7496
--- /dev/null
+++ b/tools/load-client/src/main/java/org/apache/airavata/tools/load/UnitLoad.java
@@ -0,0 +1,150 @@
+package org.apache.airavata.tools.load;
+
+import org.apache.airavata.agents.api.AgentException;
+import org.apache.airavata.api.Airavata;
+import org.apache.airavata.api.client.AiravataClientFactory;
+import org.apache.airavata.model.application.io.DataType;
+import org.apache.airavata.model.application.io.InputDataObjectType;
+import org.apache.airavata.model.experiment.ExperimentModel;
+import org.apache.airavata.model.experiment.ExperimentType;
+import org.apache.airavata.model.experiment.UserConfigurationDataModel;
+import org.apache.airavata.model.scheduling.ComputationalResourceSchedulingModel;
+import org.apache.airavata.model.security.AuthzToken;
+import org.apache.airavata.tools.load.Configuration;
+import org.apache.airavata.tools.load.StorageResourceManager;
+import org.apache.thrift.TException;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.*;
+
+public class UnitLoad {
+
+ private String apiHost;
+ private int apiPort;
+ private String trustStorePath;
+ private String trustStorePassword;
+ private StorageResourceManager storageResourceManager;
+
+ public UnitLoad(String apiHost, int apiPort, String trustStorePath, String trustStorePassword, StorageResourceManager storageResourceManager) {
+ this.apiHost = apiHost;
+ this.apiPort = apiPort;
+ this.trustStorePath = trustStorePath;
+ this.trustStorePassword = trustStorePassword;
+ this.storageResourceManager = storageResourceManager;
+ }
+
+ public CompletionService<Boolean> execute(Configuration config) {
+ String randomUUID = UUID.randomUUID().toString();
+ ExecutorService executorService = Executors.newFixedThreadPool(config.getConcurrentUsers());
+ CompletionService<Boolean> completionService = new ExecutorCompletionService<>(executorService);
+
+ for (int i = 0; i < config.getConcurrentUsers(); i++) {
+ completionService.submit(new Worker(config, randomUUID + "-" + i, config.getIterationsPerUser(), config.getRandomMSDelayWithinSubmissions()));
+ }
+ return completionService;
+ }
+
+ public class Worker implements Callable<Boolean> {
+
+ private final String id;
+ private final int iterations;
+ private final int delay;
+ private final Configuration config;
+
+ public Worker(Configuration config, String id, int iterations, int delay) {
+ this.id = id;
+ this.iterations = iterations;
+ this.delay = delay;
+ this.config = config;
+ }
+
+ @Override
+ public Boolean call() {
+ for (int i = 0; i < iterations; i++) {
+ try {
+ submitExperiment(config,id + "-" + i);
+ Thread.sleep(delay);
+ } catch (TException e) {
+ e.printStackTrace();
+ } catch (AgentException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ return true;
+ }
+ }
+
+ private void submitExperiment(Configuration config, String suffix) throws TException, AgentException {
+
+ String experimentName = config.getExperimentBaseName() + suffix;
+
+ ExperimentModel experimentModel = new ExperimentModel();
+ experimentModel.setExperimentName(experimentName);
+ experimentModel.setProjectId(config.getProjectId());
+ experimentModel.setUserName(config.getUserId());
+ experimentModel.setGatewayId(config.getGatewayId());
+ experimentModel.setExecutionId(config.getApplicationInterfaceId());
+
+ ComputationalResourceSchedulingModel computationalResourceSchedulingModel = new ComputationalResourceSchedulingModel();
+ computationalResourceSchedulingModel.setQueueName(config.getQueue());
+ computationalResourceSchedulingModel.setNodeCount(config.getNodeCount());
+ computationalResourceSchedulingModel.setTotalCPUCount(config.getCpuCount());
+ computationalResourceSchedulingModel.setWallTimeLimit(config.getWallTime());
+ computationalResourceSchedulingModel.setTotalPhysicalMemory(config.getPhysicalMemory());
+ computationalResourceSchedulingModel.setResourceHostId(config.getComputeResourceId());
+
+ UserConfigurationDataModel userConfigurationDataModel = new UserConfigurationDataModel();
+ userConfigurationDataModel.setComputationalResourceScheduling(computationalResourceSchedulingModel);
+ userConfigurationDataModel.setAiravataAutoSchedule(false);
+ userConfigurationDataModel.setOverrideManualScheduledParams(false);
+ userConfigurationDataModel.setStorageId(config.getStorageResourceId());
+ userConfigurationDataModel.setExperimentDataDir(config.getUserId()
+ .concat(File.separator)
+ .concat(config.getProjectId())
+ .concat(File.separator)
+ .concat(experimentName));
+
+ experimentModel.setUserConfigurationData(userConfigurationDataModel);
+
+ Airavata.Client airavataClient = AiravataClientFactory.createAiravataSecureClient(apiHost, apiPort, trustStorePath, trustStorePassword, 100000);
+
+ List<InputDataObjectType> applicationInputs = airavataClient.getApplicationInputs(new AuthzToken(""),
+ config.getApplicationInterfaceId());
+ List<InputDataObjectType> experimentInputs = new ArrayList<>();
+
+ storageResourceManager.createExperimentDirectory(config.getUserId(), config.getProjectId(), experimentName);
+
+ for (InputDataObjectType inputDataObjectType: applicationInputs) {
+
+ Optional<Configuration.Input> input = config.getInputs().stream().filter(inp -> inp.getName().equals(inputDataObjectType.getName())).findFirst();
+
+ if (input.isPresent()) {
+ if (inputDataObjectType.getType() == DataType.URI) {
+ String localFilePath = input.get().getValue();
+ String uploadedPath = storageResourceManager.uploadInputFile(airavataClient, localFilePath, config.getUserId(), config.getProjectId(), experimentName, config.getGatewayId());
+ inputDataObjectType.setValue(uploadedPath);
+
+ } else if (inputDataObjectType.getType() == DataType.STRING) {
+ inputDataObjectType.setValue(input.get().getValue());
+ }
+ }
+ experimentInputs.add(inputDataObjectType);
+ }
+
+ experimentModel.setExperimentInputs(experimentInputs);
+ experimentModel.setExperimentOutputs(airavataClient.getApplicationOutputs(new AuthzToken(""), config.getApplicationInterfaceId()));
+ experimentModel.setExperimentType(ExperimentType.SINGLE_APPLICATION);
+
+ String experimentId = airavataClient.createExperiment(new AuthzToken(""), config.getGatewayId(), experimentModel);
+
+ airavataClient.launchExperiment(new AuthzToken(""), experimentId, config.getGatewayId());
+ System.out.println(experimentId);
+
+ }
+}
diff --git a/tools/load-client/src/main/resources/bin/load-client.sh b/tools/load-client/src/main/resources/bin/load-client.sh
new file mode 100644
index 0000000..fe05e8d
--- /dev/null
+++ b/tools/load-client/src/main/resources/bin/load-client.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+. `dirname $0`/setenv.sh
+# Capture user's working dir before changing directory
+CWD="$PWD"
+cd ${AIRAVATA_HOME}/bin
+LOGO_FILE="logo.txt"
+
+JAVA_OPTS="-Dairavata.config.dir=${AIRAVATA_HOME}/conf -Dairavata.home=${AIRAVATA_HOME}"
+AIRAVATA_COMMAND=""
+EXTRA_ARGS=""
+SERVERS=""
+IS_SUBSET=false
+SUBSET=""
+DEFAULT_LOG_FILE="${AIRAVATA_HOME}/logs/airavata.out"
+LOG_FILE=$DEFAULT_LOG_FILE
+
+# parse command arguments
+for var in "$@"
+do
+ case ${var} in
+ -xdebug)
+ AIRAVATA_COMMAND="${AIRAVATA_COMMAND}"
+ JAVA_OPTS="$JAVA_OPTS -Xdebug -Xnoagent -Xrunjdwp:transport=dt_socket,server=y,address=8000"
+ shift
+ ;;
+ -log)
+ shift
+ LOG_FILE="$1"
+ shift
+ # If relative path, expand to absolute path using the user's $CWD
+ if [ -z "`echo "$LOG_FILE" | egrep "^/"`" ]; then
+ LOG_FILE="${CWD}/${LOG_FILE}"
+ fi
+ ;;
+ -h)
+ echo "Usage: load-client.sh"
+
+ echo "command options:"
+ echo " -config Load configuration file in yml format"
+ echo " -apiHost API Server host name"
+ echo " -apiPort API Server port"
+ echo " -privateKeyPath SSH private key path to communicate with storage resources (Defaults to user private key in ~/.ssh/id_rsa)"
+ echo " -publicKeyPath SSH public key path to communicate with storage resources (Defaults to user public key in ~/.ssh/id_rsa.pub)"
+ echo " -passPhrase SSH private key pass phrase (if any)"
+ echo " -h Display this help and exit"
+ shift
+ exit 0
+ ;;
+ *)
+ EXTRA_ARGS="${EXTRA_ARGS} ${var}"
+ shift
+ ;;
+ esac
+done
+
+echo $*
+java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+ org.apache.airavata.tools.load.LoadClient ${AIRAVATA_COMMAND} ${EXTRA_ARGS} $*
diff --git a/tools/load-client/src/main/resources/bin/setenv.sh b/tools/load-client/src/main/resources/bin/setenv.sh
new file mode 100644
index 0000000..9e894e1
--- /dev/null
+++ b/tools/load-client/src/main/resources/bin/setenv.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# resolve links - $0 may be a softlink
+PRG="$0"
+
+while [ -h "$PRG" ]; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '.*/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "$PRG"`/"$link"
+ fi
+done
+
+PRGDIR=`dirname "$PRG"`
+
+# Only set AIRAVATA_HOME if not already set
+[ -z "$AIRAVATA_HOME" ] && AIRAVATA_HOME=`cd "$PRGDIR/.." ; pwd`
+
+AIRAVATA_CLASSPATH=""
+
+for f in "$AIRAVATA_HOME"/lib/*.jar
+do
+ AIRAVATA_CLASSPATH="$AIRAVATA_CLASSPATH":$f
+done
+
+export AIRAVATA_HOME
+export AIRAVATA_CLASSPATH
diff --git a/tools/load-client/src/main/resources/client_truststore.jks b/tools/load-client/src/main/resources/client_truststore.jks
new file mode 100644
index 0000000..4ff588f
Binary files /dev/null and b/tools/load-client/src/main/resources/client_truststore.jks differ
diff --git a/tools/load-client/src/main/resources/conf/load-config.yml b/tools/load-client/src/main/resources/conf/load-config.yml
new file mode 100644
index 0000000..a193c36
--- /dev/null
+++ b/tools/load-client/src/main/resources/conf/load-config.yml
@@ -0,0 +1,22 @@
+configurations:
+ - experimentBaseName: "LoadTest"
+ userId: "dimuthu"
+ gatewayId: "seagrid"
+ projectId: "DefaultProject_7ac38275-0ca1-433a-ab6a-630c8c1df2ef"
+ applicationInterfaceId: "Echo_3f480d1f-ea86-4018-94bb-015423d66a1c"
+ computeResourceId: "js-156-93.jetstream-cloud.org_696c097d-a138-4445-b254-cd7e55c84fad"
+ storageResourceId: "testing.seagrid.org_20dbec4c-223e-4568-a267-63d7efc6267e"
+
+ inputs:
+ - name: "Input-to-Echo"
+ value: "Test"
+
+ queue: "cloud"
+ wallTime: 60
+ cpuCount: 2
+ nodeCount: 1
+ physicalMemory: 512
+
+ concurrentUsers: 10
+ iterationsPerUser: 2
+ randomMSDelayWithinSubmissions: 100
diff --git a/tools/phoebus-integration/pom.xml b/tools/phoebus-integration/pom.xml
index 90c9832..95adc10 100644
--- a/tools/phoebus-integration/pom.xml
+++ b/tools/phoebus-integration/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.airavata</groupId>
<artifactId>airavata-tools-parent</artifactId>
- <version>0.12-SNAPSHOT</version>
+ <version>0.17-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/tools/pom.xml b/tools/pom.xml
index 4a46c29..2223efb 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -42,9 +42,8 @@
<activeByDefault>true</activeByDefault>
</activation>
<modules>
- <module>registry-tool</module>
- <module>gsissh</module>
- </modules>
+ <module>load-client</module>
+ </modules>
</profile>
</profiles>
<!--<properties>-->