You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sc...@apache.org on 2017/06/27 16:18:59 UTC
[1/2] airavata-sandbox git commit: adding airavata layered prototype
Repository: airavata-sandbox
Updated Branches:
refs/heads/master 85fb44b38 -> 43a79e3a5
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/test/java/org/apache/airavata/runners/ssh/SSHRunnerTest.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/test/java/org/apache/airavata/runners/ssh/SSHRunnerTest.java b/airavata-layered-architecture/src/test/java/org/apache/airavata/runners/ssh/SSHRunnerTest.java
new file mode 100644
index 0000000..cd92522
--- /dev/null
+++ b/airavata-layered-architecture/src/test/java/org/apache/airavata/runners/ssh/SSHRunnerTest.java
@@ -0,0 +1,63 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.runners.ssh;
+
+import org.apache.airavata.Constants;
+import org.apache.commons.io.IOUtils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.UUID;
+
+public class SSHRunnerTest {
+ private final static Logger logger = LoggerFactory.getLogger(SSHRunnerTest.class);
+
+ @Test
+ public void test() {
+ try{
+ SSHKeyAuthentication br2SshAuthentication = new SSHKeyAuthentication(
+ Constants.loginUserName,
+ IOUtils.toByteArray(SSHRunnerTest.class.getClassLoader().getResourceAsStream("ssh/id_rsa")),
+ IOUtils.toByteArray(SSHRunnerTest.class.getClassLoader().getResourceAsStream("ssh/id_rsa.pub")),
+ "dummy",
+ SSHRunnerTest.class.getClassLoader().getResource("ssh/known_hosts").getPath(),
+ false
+ );
+ SSHServerInfo br2 = new SSHServerInfo(Constants.loginUserName, "bigred2.uits.iu.edu", br2SshAuthentication,22);
+
+ String routingKey = UUID.randomUUID().toString();
+ SSHRunner sshExecutor = new SSHRunner();
+ sshExecutor.executeCommand(routingKey, "mkdir -p airavata", br2, br2SshAuthentication);
+ sshExecutor.scpTo(routingKey, SSHRunnerTest.class.getClassLoader().getResource("job_tf.pbs").getPath(),
+ "~/airavata/", br2, br2SshAuthentication);
+ sshExecutor.scpTo(routingKey, SSHRunnerTest.class.getClassLoader().getResource("code_tf.py").getPath(),
+ "~/airavata/", br2, br2SshAuthentication);
+ sshExecutor.executeCommand(routingKey, new String[]{"cd ~/airavata", "qsub ~/airavata/job_tf.pbs"}, br2, br2SshAuthentication);
+
+ }catch (Exception ex){
+ ex.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/test/resources/code_tf.py
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/test/resources/code_tf.py b/airavata-layered-architecture/src/test/resources/code_tf.py
new file mode 100644
index 0000000..221e960
--- /dev/null
+++ b/airavata-layered-architecture/src/test/resources/code_tf.py
@@ -0,0 +1,30 @@
+import tensorflow as tf
+import numpy as np
+import numpy.polynomial.polynomial as poly
+
+from random import randint
+
+tf.reset_default_graph()
+num_neurons = 5000
+
+indices = []
+values = []
+for i in range(num_neurons):
+ for j in range(num_neurons):
+ x = 3
+ if i != j:
+ number = randint(0, 99)
+ if number < 5:
+ indices.append([i, j])
+ values.append(1.0/5)
+
+connections = tf.SparseTensor(indices=indices, values=values, dense_shape=[num_neurons, num_neurons])
+
+neuron_values = tf.Variable(np.ones(num_neurons), dtype=tf.float32)
+
+mul_product = tf.sparse_tensor_dense_matmul(connections, tf.reshape(neuron_values, shape=(num_neurons, 1)))
+
+sess = tf.Session()
+sess.run(tf.global_variables_initializer())
+
+output = sess.run(mul_product)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/test/resources/job_tf.pbs
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/test/resources/job_tf.pbs b/airavata-layered-architecture/src/test/resources/job_tf.pbs
new file mode 100644
index 0000000..a4d3485
--- /dev/null
+++ b/airavata-layered-architecture/src/test/resources/job_tf.pbs
@@ -0,0 +1,18 @@
+#!/bin/bash
+# file to submit non interactive jobs to bigred2
+
+#PBS -l nodes=1:ppn=16
+#PBS -l gres=ccm
+#PBS -q debug_gpu
+#PBS -l walltime=00:30:00
+
+#PBS -o STDOUT.txt
+#PBS -e STDERR.txt
+
+#PBS -m bea
+#PBS -M supun.nakandala@gmail.com
+
+module load ccm
+module load anaconda2/4.2.0
+source activate tensorflow_env
+ccmrun python ~/airavata/code_tf.py
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/test/resources/ssh/id_rsa
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/test/resources/ssh/id_rsa b/airavata-layered-architecture/src/test/resources/ssh/id_rsa
new file mode 100644
index 0000000..3f1f2d1
--- /dev/null
+++ b/airavata-layered-architecture/src/test/resources/ssh/id_rsa
@@ -0,0 +1,30 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: AES-128-CBC,3AF46750EC6C202FA7E19DD44994DCBC
+
+T1iV/k7DS8Li4pwYyySEBpPv8OWbd5Z1h5Z1a4FNScLZGsLm8CvduJxvDrQWNOfd
+qC5M8VNXvATN2K3+vj82aqLQ9qpYoQAC1yggHWEfqemlolKk89ICLK4LVjMZX2dw
+l3kHwNgIa0rDZtE8rxV+8M9u+IQ7/j88FcLgqVzPjBHLlnrWveZQ+IhuxFTg8E5I
+1iolp/gaAkE9cfnEPpje3vf+m0Dkb0Td2vO7V2keBvyIG8s4CwvKWQiifRYu4jQJ
+Vit6/UgySbEzd+SxYBPOruOCM8IEeLpUX7xE6Zt2fgA9CwMqTpuIFPKmCfj2ON5h
+bgJkLA3pLgs+imzBOYEZLBB5UO2iNuDbYTfF7nYvSUXaSCn3ekzleGgtJ6L2+03U
+D5bntrVDwIV510dRAN2EiyB744p+tzOm5mbTPFHtrefiZdfumsj2er4rPy/9FA09
+4aBlPPS0yCGTV87Edtuve4UcFxkpc5f3QgdxUOAZQI1KHILluZjQI93yMAt++KeF
+Fvf7dEtT8VVrwzsCGb9Pc1lyboGJxohm63Xe7os2Rn2i4LHXVX6NhoPC4L+U8YJC
+53q68G3ExNjeIu/N7y2QKtrXYLnqzvxZ0Tm1jDVp2UyG2yVzOeiITrXCG9qPBmrs
+59fcIXsWb3xFQMOeKN6WzOaMEGPnCI+SJJwIc2PzR0OKeBB/rBy8zNaIOb8gqHb7
+uSd0UJxASa5801TlJtdhtGyVGZJtG7vyHIn8vF/GT+VQY7ehbRRPGRRij3J+ruvS
+rMZU8oTntvMUdDLkI/JeD6yInzX4n1rWGLoHeck75nA4CHuHBYF+Pgr5q8I/dts+
+gFUgqFq2s9vn3haP/pnJfQikuEWfd62S1IU7x+8Ia96LomitbRBwZM1fG3Tphtv0
+7i+bLIUx+voRn/ayPRnasnQVoQgYe7g1CTm6zE1N8d8OFY9P6AoUhOmSxyrC6W8M
+ZttxFntaHwF1s07bfl+Yr1KJaodMfvP5JBpCjK+dMyk0temC8Y3dzC4Fu0CLEwrD
+p25iD0xvY0QYa4oRgZMD6gGmww6olPoda7/txZp+gu0GOcgYogozT8mcUSq0RZor
+cWL/lrjcjZHbeDbyWQehChdV4i6y7CGjQhry1f3V+TQtSDRxX5344UpQalhKLjsf
+XdMMJYuD2u/1BV4Lg+O6aFwRwGQtNkF5AMTq3FDluptWdAcAU3dsO9sNTCTmgCwx
+lD4Pew4lBYZe0WZ+2o+0X1efyGJJ6/lKjfpVF0eNwQqFQ8SxSFZvUrwJsK+NujjH
+nhjobs64n4MRAPnckVqO+vlY408uUDUoNOzROAC9zDl216ZXT8zTmWeoNkQS6DKb
+JXtbXPDHWdo9VLik5h08FjmwdNoMS18mkSDpkwkP3hGCQnWhKZDjlU6kfksRbeee
+t6WI4pKIbzHQJV+CLm1uzC1o7EfsD4dbSpefpJUD5D+mTH8MBkjb7p5+E2/TER62
+rIdXjpHEA/s8W3bmdTSam1mqA3VUBGVQryIfAv2qOyVjQx2Y28+pDTAtMT6FnYUZ
+sVUucxUp+TE7WqIQX8AzjNAudu25R7YPQvgQXjSrA6G6bxNQNC/TQeT+vgO4g8Ah
+-----END RSA PRIVATE KEY-----
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/test/resources/ssh/id_rsa.pub
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/test/resources/ssh/id_rsa.pub b/airavata-layered-architecture/src/test/resources/ssh/id_rsa.pub
new file mode 100644
index 0000000..e378be0
--- /dev/null
+++ b/airavata-layered-architecture/src/test/resources/ssh/id_rsa.pub
@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCr9VP22p0I+2W5o/klPv/OvfTihvcBQwZXKPrSLFF+OB9nVNtIfDDETIwwex7mknn3Kks1jFvEdKMrvRjOFeFInDv3N40LjohHu4v2tiawAON7MOLpz/iX5dWp0wteixlDKfGe7PAEMAk054kLSDiB3em2zBK4d9ApedA5k2JG1dmAsNK0KkbfgFPd5+iXrzgTg4XiefHQoaCSUyS7w6t8645djbYOP+b+SJtgslaf2RqeoBVvrA6YQJE1pUYjcm9yL4KwyqaPo+N/2XZ6xys5+WN8svtL3uRduENU1MQSTpdFq+GLCY4SgLMFgLJKoxHjcjPRfKyE/eYk1gQA7b/Z snakanda@149-161-141-51.dhcp-bl.indiana.edu
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/test/resources/ssh/known_hosts
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/test/resources/ssh/known_hosts b/airavata-layered-architecture/src/test/resources/ssh/known_hosts
new file mode 100644
index 0000000..e69de29
[2/2] airavata-sandbox git commit: adding airavata layered prototype
Posted by sc...@apache.org.
adding airavata layered prototype
Project: http://git-wip-us.apache.org/repos/asf/airavata-sandbox/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata-sandbox/commit/43a79e3a
Tree: http://git-wip-us.apache.org/repos/asf/airavata-sandbox/tree/43a79e3a
Diff: http://git-wip-us.apache.org/repos/asf/airavata-sandbox/diff/43a79e3a
Branch: refs/heads/master
Commit: 43a79e3a5df6f8ed71ac439c71c322ece54b38a2
Parents: 85fb44b
Author: scnakandala <su...@gmail.com>
Authored: Tue Jun 27 12:18:54 2017 -0400
Committer: scnakandala <su...@gmail.com>
Committed: Tue Jun 27 12:18:54 2017 -0400
----------------------------------------------------------------------
airavata-layered-architecture/pom.xml | 58 +++
.../airavata/resources/Authentication.java | 30 ++
.../apache/airavata/resources/OutputParser.java | 66 ++++
.../apache/airavata/resources/ServerInfo.java | 49 +++
.../resources/batch/BatchJobOutputParser.java | 89 +++++
.../airavata/resources/batch/GroovyMap.java | 93 +++++
.../resources/batch/HPCBatchResource.java | 200 ++++++++++
.../batch/JobManagerConfiguration.java | 51 +++
.../airavata/resources/batch/JobStatus.java | 28 ++
.../resources/batch/JobSubmissionOutput.java | 88 +++++
.../resources/batch/PBSJobConfiguration.java | 120 ++++++
.../resources/batch/RawCommandInfo.java | 41 ++
.../apache/airavata/resources/batch/Script.java | 61 +++
.../apache/airavata/runners/CommandOutput.java | 57 +++
.../airavata/runners/ssh/SSHApiException.java | 35 ++
.../runners/ssh/SSHCommandOutputReader.java | 90 +++++
.../runners/ssh/SSHKeyAuthentication.java | 92 +++++
.../apache/airavata/runners/ssh/SSHRunner.java | 394 +++++++++++++++++++
.../airavata/runners/ssh/SSHServerInfo.java | 52 +++
.../airavata/runners/ssh/SSHUserInfo.java | 66 ++++
.../resources/batch/PBS_Groovy.template | 34 ++
.../java/org/apache/airavata/Constants.java | 30 ++
.../resources/batch/HPCBatchResourceTest.java | 86 ++++
.../airavata/runners/ssh/SSHRunnerTest.java | 63 +++
.../src/test/resources/code_tf.py | 30 ++
.../src/test/resources/job_tf.pbs | 18 +
.../src/test/resources/ssh/id_rsa | 30 ++
.../src/test/resources/ssh/id_rsa.pub | 1 +
.../src/test/resources/ssh/known_hosts | 0
29 files changed, 2052 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/pom.xml
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/pom.xml b/airavata-layered-architecture/pom.xml
new file mode 100644
index 0000000..ff289c6
--- /dev/null
+++ b/airavata-layered-architecture/pom.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata</artifactId>
+ <version>1.0-SNAPSHOT</version>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ <version>0.1.53</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>19.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-templates</artifactId>
+ <version>2.4.7</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.10</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.5.1</version>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/Authentication.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/Authentication.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/Authentication.java
new file mode 100644
index 0000000..426f55e
--- /dev/null
+++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/Authentication.java
@@ -0,0 +1,30 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.resources;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Authentication {
+ private final static Logger logger = LoggerFactory.getLogger(Authentication.class);
+
+ protected String userName;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/OutputParser.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/OutputParser.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/OutputParser.java
new file mode 100644
index 0000000..de523e4
--- /dev/null
+++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/OutputParser.java
@@ -0,0 +1,66 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.resources;
+
+import org.apache.airavata.resources.batch.JobStatus;
+
+import java.util.Map;
+
+public interface OutputParser {
+ /**
+ * This can be used to parseSingleJob the result of a job submission to get the JobID
+ * @param rawOutput
+ * @return the job id as a String, or null if no job id found
+ */
+ public String parseJobSubmission(String rawOutput)throws Exception;
+
+
+ /**
+ * Parse output return by job submission task and identify jobSubmission failures.
+ * @param rawOutput
+ * @return true if job submission has been failed, false otherwise.
+ */
+ public boolean isJobSubmissionFailed(String rawOutput);
+
+
+ /**
+ * This can be used to get the job status from the output
+ * @param jobID
+ * @param rawOutput
+ */
+ public JobStatus parseJobStatus(String jobID, String rawOutput)throws Exception;
+
+ /**
+ * This can be used to parseSingleJob a big output and get multipleJob statuses
+ * @param statusMap list of status map will return and key will be the job ID
+ * @param rawOutput
+ */
+ public void parseJobStatuses(String userName, Map<String, JobStatus> statusMap, String rawOutput)throws Exception;
+
+ /**
+ * filter the jobId value of given JobName from rawOutput
+ * @param jobName
+ * @param rawOutput
+ * @return
+ * @throws Exception
+ */
+ public String parseJobId(String jobName, String rawOutput) throws Exception;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/ServerInfo.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/ServerInfo.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/ServerInfo.java
new file mode 100644
index 0000000..0bb46f5
--- /dev/null
+++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/ServerInfo.java
@@ -0,0 +1,49 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.resources;
+
+public class ServerInfo {
+
+ public static enum ComProtocol {SSH, LOCAL}
+
+ protected String host;
+ protected String userName;
+ protected int port;
+ protected ComProtocol comProtocol;
+
+ public ServerInfo(){}
+
+ public ServerInfo(String userName, String host, ComProtocol comProtocol, int port) {
+ this.userName = userName;
+ this.host = host;
+ this.comProtocol = comProtocol;
+ this.port = port;
+ }
+
+ public String getHost() {
+ return host;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/BatchJobOutputParser.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/BatchJobOutputParser.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/BatchJobOutputParser.java
new file mode 100644
index 0000000..2278663
--- /dev/null
+++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/BatchJobOutputParser.java
@@ -0,0 +1,89 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.resources.batch;
+
+import org.apache.airavata.resources.OutputParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+public class BatchJobOutputParser implements OutputParser {
+ private final static Logger logger = LoggerFactory.getLogger(BatchJobOutputParser.class);
+
+ /**
+ * This can be used to parseSingleJob the result of a job submission to get the JobID
+ *
+ * @param rawOutput
+ * @return the job id as a String, or null if no job id found
+ */
+ @Override
+ public String parseJobSubmission(String rawOutput) throws Exception {
+ return null;
+ }
+
+ /**
+ * Parse output return by job submission task and identify jobSubmission failures.
+ *
+ * @param rawOutput
+ * @return true if job submission has been failed, false otherwise.
+ */
+ @Override
+ public boolean isJobSubmissionFailed(String rawOutput) {
+ return false;
+ }
+
+ /**
+ * This can be used to get the job status from the output
+ *
+ * @param jobID
+ * @param rawOutput
+ */
+ @Override
+ public JobStatus parseJobStatus(String jobID, String rawOutput) throws Exception {
+ return null;
+ }
+
+ /**
+ * This can be used to parseSingleJob a big output and get multipleJob statuses
+ *
+ * @param userName
+ * @param statusMap list of status map will return and key will be the job ID
+ * @param rawOutput
+ */
+ @Override
+ public void parseJobStatuses(String userName, Map<String, JobStatus> statusMap, String rawOutput) throws Exception {
+
+ }
+
+ /**
+ * filter the jobId value of given JobName from rawOutput
+ *
+ * @param jobName
+ * @param rawOutput
+ * @return
+ * @throws Exception
+ */
+ @Override
+ public String parseJobId(String jobName, String rawOutput) throws Exception {
+ return null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/GroovyMap.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/GroovyMap.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/GroovyMap.java
new file mode 100644
index 0000000..8709e8f
--- /dev/null
+++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/GroovyMap.java
@@ -0,0 +1,93 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.resources.batch;
+
+import java.util.HashMap;
+import java.util.Optional;
+
+public class GroovyMap extends HashMap<String, Object> {
+
+
+ public GroovyMap() {
+ super();
+ // to mitigate groovy exception groovy.lang.MissingPropertyException: No such property: <name> for class: groovy.lang.Binding
+ addDefaultValues();
+ }
+
+ public GroovyMap add(Script name, Object value){
+ put(name.name, value);
+ return this;
+ }
+
+ @Override
+ public Object get(Object key) {
+ return super.getOrDefault(key, null);
+ }
+
+ public Object get(Script script) {
+ return get(script.name);
+ }
+
+ public Optional<String> getStringValue(Script script) {
+ Object obj = get(script);
+ if (obj instanceof String) {
+ return Optional.of((String) obj);
+ } else if (obj == null) {
+ return Optional.empty();
+ } else {
+ throw new IllegalArgumentException("Value is not String type");
+ }
+ }
+
+ private void addDefaultValues() {
+ this.add(Script.SHELL_NAME, null)
+ .add(Script.QUEUE_NAME, null)
+ .add(Script.NODES, null)
+ .add(Script.CPU_COUNT, null)
+ .add(Script.MAIL_ADDRESS, null)
+ .add(Script.ACCOUNT_STRING, null)
+ .add(Script.MAX_WALL_TIME, null)
+ .add(Script.JOB_NAME, null)
+ .add(Script.STANDARD_OUT_FILE, null)
+ .add(Script.STANDARD_ERROR_FILE, null)
+ .add(Script.QUALITY_OF_SERVICE, null)
+ .add(Script.RESERVATION, null)
+ .add(Script.EXPORTS, null)
+ .add(Script.MODULE_COMMANDS, null)
+ .add(Script.SCRATCH_LOCATION, null)
+ .add(Script.WORKING_DIR, null)
+ .add(Script.PRE_JOB_COMMANDS, null)
+ .add(Script.JOB_SUBMITTER_COMMAND, null)
+ .add(Script.EXECUTABLE_PATH, null)
+ .add(Script.INPUTS, null)
+ .add(Script.POST_JOB_COMMANDS, null)
+ .add(Script.USED_MEM, null)
+ .add(Script.PROCESS_PER_NODE, null)
+ .add(Script.CHASSIS_NAME, null)
+ .add(Script.INPUT_DIR, null)
+ .add(Script.OUTPUT_DIR, null)
+ .add(Script.USER_NAME, null)
+ .add(Script.GATEWAY_ID, null)
+ .add(Script.GATEWAY_USER_NAME, null)
+ .add(Script.APPLICATION_NAME, null);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/HPCBatchResource.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/HPCBatchResource.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/HPCBatchResource.java
new file mode 100644
index 0000000..46f2a48
--- /dev/null
+++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/HPCBatchResource.java
@@ -0,0 +1,200 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.resources.batch;
+
+import com.jcraft.jsch.JSchException;
+import groovy.lang.Writable;
+import groovy.text.GStringTemplateEngine;
+import groovy.text.TemplateEngine;
+import org.apache.airavata.resources.Authentication;
+import org.apache.airavata.resources.OutputParser;
+import org.apache.airavata.resources.ServerInfo;
+import org.apache.airavata.runners.ssh.SSHCommandOutputReader;
+import org.apache.airavata.runners.ssh.SSHKeyAuthentication;
+import org.apache.airavata.runners.ssh.SSHRunner;
+import org.apache.airavata.runners.ssh.SSHServerInfo;
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+
+public class HPCBatchResource {
+ private final static Logger logger = LoggerFactory.getLogger(HPCBatchResource.class);
+
+ private static final int MAX_RETRY_COUNT = 3;
+
+ private ServerInfo serverInfo;
+ private Authentication authentication;
+ private JobManagerConfiguration jobManagerConfiguration;
+ private OutputParser outputParser;
+
+ public HPCBatchResource(ServerInfo serverInfo, JobManagerConfiguration jobManagerConfiguration, Authentication
+ authentication) throws Exception {
+ if(!(serverInfo instanceof SSHServerInfo) || !(authentication instanceof SSHKeyAuthentication)){
+ throw new Exception("Currently only SSH based communication is enabled for HPCRemote clusters");
+ }
+ if(!(jobManagerConfiguration instanceof PBSJobConfiguration)){
+ throw new Exception("Currently only PBS job configuration is enabled");
+ }
+
+ this.serverInfo = (SSHServerInfo) serverInfo;
+ this.authentication = (SSHKeyAuthentication) authentication;
+ this.jobManagerConfiguration = jobManagerConfiguration;
+ this.outputParser = jobManagerConfiguration.getParser();
+ }
+
+ public JobSubmissionOutput submitBatchJob(String routingKey, GroovyMap groovyMap, String workingDirectory) throws Exception {
+ File tempJobFile = File.createTempFile("temp_job", jobManagerConfiguration.getScriptExtension());
+ FileUtils.writeStringToFile(tempJobFile, generateScript(groovyMap));
+
+ String jobScriptFilePath = tempJobFile.getPath();
+ JobSubmissionOutput jsoutput = new JobSubmissionOutput();
+ copyTo(routingKey, jobScriptFilePath, workingDirectory+"/demo_job"
+ + jobManagerConfiguration.getScriptExtension()); // scp script file to working directory
+ RawCommandInfo submitCommand = jobManagerConfiguration.getSubmitCommand(workingDirectory, workingDirectory+"/demo_job"
+ + jobManagerConfiguration.getScriptExtension());
+ submitCommand.setRawCommand("cd " + workingDirectory + " && " + submitCommand.getRawCommand());
+ SSHCommandOutputReader reader = executeCommand(routingKey, submitCommand);
+
+ jsoutput.setJobId(outputParser.parseJobSubmission(reader.getStdOutputString()));
+ if (jsoutput.getJobId() == null) {
+ if (outputParser.isJobSubmissionFailed(reader.getStdOutputString())) {
+ jsoutput.setJobSubmissionFailed(true);
+ jsoutput.setFailureReason("stdout : " + reader.getStdOutputString() +
+ "\n stderr : " + reader.getStdErrorString());
+ }
+ }
+ jsoutput.setExitCode(reader.getExitCode());
+ if (jsoutput.getExitCode() != 0) {
+ jsoutput.setJobSubmissionFailed(true);
+ jsoutput.setFailureReason("stdout : " + reader.getStdOutputString() +
+ "\n stderr : " + reader.getStdErrorString());
+ }
+ jsoutput.setStdOut(reader.getStdOutputString());
+ jsoutput.setStdErr(reader.getStdErrorString());
+ return jsoutput;
+ }
+
+ public void copyTo(String routingKey, String localFile, String remoteFile) throws Exception {
+ int retry = 3;
+ while (retry > 0) {
+ try {
+ SSHRunner sshRunner = new SSHRunner();
+ logger.info("Transferring localhost:" + localFile + " to " + serverInfo.getHost() + ":" + remoteFile);
+ sshRunner.scpTo("", localFile, remoteFile, (SSHServerInfo) serverInfo, (SSHKeyAuthentication) authentication);
+ retry = 0;
+ } catch (Exception e) {
+ retry--;
+ if (retry == 0) {
+ throw new Exception("Failed to scp localhost:" + localFile + " to " + serverInfo.getHost() +
+ ":" + remoteFile, e);
+ } else {
+ logger.info("Retry transfer localhost:" + localFile + " to " + serverInfo.getHost() + ":" +
+ remoteFile);
+ }
+ }
+ }
+ }
+
+ public void copyFrom(String routingKey, String remoteFile, String localFile) throws Exception {
+ int retry = 0;
+ while(retry < MAX_RETRY_COUNT) {
+ try {
+ logger.info("Transferring " + serverInfo.getHost() + ":" + remoteFile + " To localhost:" + localFile);
+ SSHRunner sshRunner = new SSHRunner();
+ sshRunner.scpFrom(routingKey, remoteFile, localFile, (SSHServerInfo) serverInfo, (SSHKeyAuthentication)authentication);
+ retry=0;
+ } catch (Exception e) {
+ retry++;
+ if (retry == 0) {
+ throw new Exception("Failed to scp " + serverInfo.getHost() + ":" + remoteFile + " to " +
+ "localhost:" + localFile, e);
+ } else {
+ logger.info("Retry transfer " + serverInfo.getHost() + ":" + remoteFile + " to localhost:" + localFile);
+ }
+ }
+ }
+ }
+
+ public void makeDirectory(String routingKey, String directoryPath) throws Exception {
+ int retryCount = 0;
+ try {
+ while (retryCount < MAX_RETRY_COUNT) {
+ retryCount++;
+ logger.info("Creating directory: " + serverInfo.getHost() + ":" + directoryPath);
+ try {
+ SSHRunner sshRunner = new SSHRunner();
+ sshRunner.makeDirectory(routingKey, directoryPath, (SSHServerInfo) serverInfo, (SSHKeyAuthentication)authentication);
+ break; // Exit while loop
+ } catch (JSchException e) {
+ if (retryCount == MAX_RETRY_COUNT) {
+ logger.error("Retry count " + MAX_RETRY_COUNT + " exceeded for creating directory: "
+ + serverInfo.getHost() + ":" + directoryPath, e);
+
+ throw e;
+ }
+ logger.error("Issue with jsch, Retry creating directory: " + serverInfo.getHost() + ":" + directoryPath);
+ }
+ }
+ } catch (JSchException | IOException e) {
+ throw new Exception("Failed to create directory " + serverInfo.getHost() + ":" + directoryPath, e);
+ }
+ }
+
+ private SSHCommandOutputReader executeCommand(String routingKey, RawCommandInfo commandInfo) throws Exception {
+ String command = commandInfo.getCommand();
+ int retryCount = 0;
+ try {
+ while (retryCount < MAX_RETRY_COUNT) {
+ retryCount++;
+
+ SSHRunner sshRunner = new SSHRunner();
+ SSHCommandOutputReader commandOutput = sshRunner.executeCommand(routingKey, command, (SSHServerInfo)serverInfo, (SSHKeyAuthentication)authentication);
+ logger.info("Executing command {}", commandInfo.getCommand());
+ return commandOutput;
+ }
+ throw new Exception("Unable to execute command after "+retryCount+" retry attempts - " + command);
+ } catch (JSchException e) {
+ throw new Exception("Unable to execute command - " + command, e);
+ }
+ }
+
+
+ private String generateScript(GroovyMap groovyMap) throws Exception {
+ URL templateUrl = new URL("file://"+jobManagerConfiguration.getJobDescriptionTemplateName());
+ if (templateUrl == null) {
+ String error = "Template file '" + jobManagerConfiguration.getJobDescriptionTemplateName() + "' not found";
+ throw new Exception(error);
+ }
+ File template = new File(templateUrl.getPath());
+ TemplateEngine engine = new GStringTemplateEngine();
+ Writable make;
+ try {
+ make = engine.createTemplate(template).make(groovyMap);
+ } catch (Exception e) {
+ throw new Exception("Error while generating script using groovy map");
+ }
+ return make.toString();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobManagerConfiguration.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobManagerConfiguration.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobManagerConfiguration.java
new file mode 100644
index 0000000..bc0bfcd
--- /dev/null
+++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobManagerConfiguration.java
@@ -0,0 +1,51 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.resources.batch;
+
+import org.apache.airavata.resources.OutputParser;
+
+public interface JobManagerConfiguration {
+ public static enum JobManagerCommand{JOB_MONITORING, SUBMISSION, DELETION};
+
+ public RawCommandInfo getCancelCommand(String jobID);
+
+ public String getJobDescriptionTemplateName();
+
+ public RawCommandInfo getMonitorCommand(String jobID);
+
+ public RawCommandInfo getUserBasedMonitorCommand(String userName);
+
+ public RawCommandInfo getJobIdMonitorCommand(String jobName , String userName);
+
+ public String getScriptExtension();
+
+ public RawCommandInfo getSubmitCommand(String workingDirectory, String pbsFilePath);
+
+ public OutputParser getParser();
+
+ public String getInstalledPath();
+
+ public String getBaseCancelCommand();
+
+ public String getBaseMonitorCommand();
+
+ public String getBaseSubmitCommand();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobStatus.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobStatus.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobStatus.java
new file mode 100644
index 0000000..faa17e2
--- /dev/null
+++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobStatus.java
@@ -0,0 +1,28 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.resources.batch;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class JobStatus {
+ private final static Logger logger = LoggerFactory.getLogger(JobStatus.class);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobSubmissionOutput.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobSubmissionOutput.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobSubmissionOutput.java
new file mode 100644
index 0000000..44bec6b
--- /dev/null
+++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/JobSubmissionOutput.java
@@ -0,0 +1,88 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.resources.batch;
+
+public class JobSubmissionOutput {
+
+ private int exitCode = Integer.MIN_VALUE;
+ private String stdOut;
+ private String stdErr;
+ private String command;
+ private String jobId;
+ private boolean isJobSubmissionFailed;
+ private String failureReason;
+
+ public int getExitCode() {
+ return exitCode;
+ }
+
+ public void setExitCode(int exitCode) {
+ this.exitCode = exitCode;
+ }
+
+ public String getStdOut() {
+ return stdOut;
+ }
+
+ public void setStdOut(String stdOut) {
+ this.stdOut = stdOut;
+ }
+
+ public String getStdErr() {
+ return stdErr;
+ }
+
+ public void setStdErr(String stdErr) {
+ this.stdErr = stdErr;
+ }
+
+ public String getCommand() {
+ return command;
+ }
+
+ public void setCommand(String command) {
+ this.command = command;
+ }
+
+ public String getJobId() {
+ return jobId;
+ }
+
+ public void setJobId(String jobId) {
+ this.jobId = jobId;
+ }
+
+ public boolean isJobSubmissionFailed() {
+ return isJobSubmissionFailed;
+ }
+
+ public void setJobSubmissionFailed(boolean jobSubmissionFailed) {
+ isJobSubmissionFailed = jobSubmissionFailed;
+ }
+
+ public String getFailureReason() {
+ return failureReason;
+ }
+
+ public void setFailureReason(String failureReason) {
+ this.failureReason = failureReason;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/PBSJobConfiguration.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/PBSJobConfiguration.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/PBSJobConfiguration.java
new file mode 100644
index 0000000..07bccc0
--- /dev/null
+++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/PBSJobConfiguration.java
@@ -0,0 +1,120 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.resources.batch;
+
+import org.apache.airavata.resources.OutputParser;
+import org.apache.commons.io.FilenameUtils;
+
+import java.io.File;
+import java.util.Map;
+
+public class PBSJobConfiguration implements JobManagerConfiguration {
+
+ private final Map<JobManagerCommand, String> jobManagerCommands;
+ private String jobDescriptionTemplateName;
+ private String scriptExtension;
+ private String installedPath;
+ private OutputParser parser;
+
+ public PBSJobConfiguration(String jobDescriptionTemplateName, String scriptExtension, String installedPath,
+ Map<JobManagerCommand, String> jobManagerCommands, OutputParser parser) {
+ this.jobDescriptionTemplateName = jobDescriptionTemplateName;
+ this.scriptExtension = scriptExtension;
+ this.parser = parser;
+ installedPath = installedPath.trim();
+ if (installedPath.endsWith("/")) {
+ this.installedPath = installedPath;
+ } else {
+ this.installedPath = installedPath + "/";
+ }
+ this.jobManagerCommands = jobManagerCommands;
+ }
+
+ public RawCommandInfo getCancelCommand(String jobID) {
+ return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.DELETION).trim() + " " +
+ jobID);
+ }
+
+ public String getJobDescriptionTemplateName() {
+ return jobDescriptionTemplateName;
+ }
+
+ public void setJobDescriptionTemplateName(String jobDescriptionTemplateName) {
+ this.jobDescriptionTemplateName = jobDescriptionTemplateName;
+ }
+
+ public RawCommandInfo getMonitorCommand(String jobID) {
+ return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.JOB_MONITORING).trim()
+ + " -f " + jobID);
+ }
+
+ public String getScriptExtension() {
+ return scriptExtension;
+ }
+
+ public RawCommandInfo getSubmitCommand(String workingDirectory, String pbsFilePath) {
+ return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.SUBMISSION).trim() + " " +
+ workingDirectory + File.separator + FilenameUtils.getName(pbsFilePath));
+ }
+
+ public String getInstalledPath() {
+ return installedPath;
+ }
+
+ public void setInstalledPath(String installedPath) {
+ this.installedPath = installedPath;
+ }
+
+ public OutputParser getParser() {
+ return parser;
+ }
+
+ public void setParser(OutputParser parser) {
+ this.parser = parser;
+ }
+
+ public RawCommandInfo getUserBasedMonitorCommand(String userName) {
+ return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.JOB_MONITORING).trim()
+ + " -u " + userName);
+ }
+
+ @Override
+ public RawCommandInfo getJobIdMonitorCommand(String jobName, String userName) {
+ // For PBS there is no option to get jobDetails by JobName, so we search with userName
+ return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.JOB_MONITORING).trim()
+ + " -u " + userName + " -f | grep \"Job_Name = " + jobName + "\" -B1");
+ }
+
+ @Override
+ public String getBaseCancelCommand() {
+ return jobManagerCommands.get(JobManagerCommand.DELETION).trim();
+ }
+
+ @Override
+ public String getBaseMonitorCommand() {
+ return jobManagerCommands.get(JobManagerCommand.JOB_MONITORING).trim();
+ }
+
+ @Override
+ public String getBaseSubmitCommand() {
+ return jobManagerCommands.get(JobManagerCommand.SUBMISSION).trim();
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/RawCommandInfo.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/RawCommandInfo.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/RawCommandInfo.java
new file mode 100644
index 0000000..3193505
--- /dev/null
+++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/RawCommandInfo.java
@@ -0,0 +1,41 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.resources.batch;
+
+public class RawCommandInfo {
+ private String rawCommand;
+
+ public RawCommandInfo(String cmd) {
+ this.rawCommand = cmd;
+ }
+
+ public String getCommand() {
+ return this.rawCommand;
+ }
+
+ public String getRawCommand() {
+ return rawCommand;
+ }
+
+ public void setRawCommand(String rawCommand) {
+ this.rawCommand = rawCommand;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/Script.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/Script.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/Script.java
new file mode 100644
index 0000000..7f49753
--- /dev/null
+++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/resources/batch/Script.java
@@ -0,0 +1,61 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.resources.batch;
+
+public enum Script {
+
+ SHELL_NAME("shellName"),
+ QUEUE_NAME("queueName"),
+ NODES("nodes"),
+ CPU_COUNT("cpuCount"),
+ MAIL_ADDRESS("mailAddress"),
+ ACCOUNT_STRING("accountString"),
+ MAX_WALL_TIME("maxWallTime"),
+ JOB_NAME("jobName"),
+ STANDARD_OUT_FILE("standardOutFile"),
+ STANDARD_ERROR_FILE("standardErrorFile"),
+ QUALITY_OF_SERVICE("qualityOfService"),
+ RESERVATION("reservation"),
+ EXPORTS("exports"),
+ MODULE_COMMANDS("moduleCommands"),
+ SCRATCH_LOCATION("scratchLocation"),
+ WORKING_DIR("workingDirectory"),
+ PRE_JOB_COMMANDS("preJobCommands"),
+ JOB_SUBMITTER_COMMAND("jobSubmitterCommand"),
+ EXECUTABLE_PATH("executablePath"),
+ INPUTS("inputs"),
+ POST_JOB_COMMANDS("postJobCommands"),
+ USED_MEM("usedMem"),
+ PROCESS_PER_NODE("processPerNode"),
+ CHASSIS_NAME("chassisName"),
+ INPUT_DIR("inputDir"),
+ OUTPUT_DIR("outputDir"),
+ USER_NAME("userName"),
+ GATEWAY_ID("gatewayId"),
+ GATEWAY_USER_NAME("gatewayUserName"),
+ APPLICATION_NAME("applicationName"),
+ ;
+
+ String name;
+ Script(String name) {
+ this.name = name;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/CommandOutput.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/CommandOutput.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/CommandOutput.java
new file mode 100644
index 0000000..c3f8877
--- /dev/null
+++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/CommandOutput.java
@@ -0,0 +1,57 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.runners;
+
+
+import com.jcraft.jsch.Channel;
+
+import java.io.OutputStream;
+
+/**
+ * Output of a certain command.
+ */
+public interface CommandOutput {
+
+ /**
+ * Gets the output of the command as a stream.
+ * @param channel Command output as a stream.
+ */
+ void onOutput(Channel channel);
+
+ /**
+ * Gets standard error as a output stream.
+ * @return Command error as a stream.
+ */
+ OutputStream getStandardError();
+
+ /**
+ * The command exit code.
+ * @param code The program exit code
+ */
+ void exitCode(int code);
+
+ /**
+ * Return the exit code of the command execution.
+ * @return exit code
+ */
+ int getExitCode();
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHApiException.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHApiException.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHApiException.java
new file mode 100644
index 0000000..e117351
--- /dev/null
+++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHApiException.java
@@ -0,0 +1,35 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.runners.ssh;
+
+/**
+ * An exception class to wrap SSH command execution related errors.
+ */
+public class SSHApiException extends Exception {
+
+ public SSHApiException(String message) {
+ super(message);
+ }
+
+ public SSHApiException(String message, Exception e) {
+ super(message, e);
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHCommandOutputReader.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHCommandOutputReader.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHCommandOutputReader.java
new file mode 100644
index 0000000..cdc45a9
--- /dev/null
+++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHCommandOutputReader.java
@@ -0,0 +1,90 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.runners.ssh;
+
+import com.jcraft.jsch.Channel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+public class SSHCommandOutputReader{
+
+ private static final Logger logger = LoggerFactory.getLogger(SSHCommandOutputReader.class);
+ String stdOutputString = null;
+ String stdErrorString = null;
+ ByteArrayOutputStream errorStream = new ByteArrayOutputStream();
+ private int exitCode;
+
+ public void onOutput(Channel channel) {
+ try {
+ this.setStdOutputString(getOutputStream(channel, channel.getInputStream()));
+ this.setStdErrorString(new String(errorStream.toByteArray(), "UTF-8"));
+ this.exitCode = channel.getExitStatus();
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ }
+
+ }
+
+ private String getOutputStream(Channel channel, InputStream inputStream) throws IOException {
+ StringBuilder output = new StringBuilder("");
+ byte[] tmp = new byte[1024];
+ do {
+ while (inputStream.available() > 0) {
+ int i = inputStream.read(tmp, 0, 1024);
+ if (i < 0) break;
+ output.append(new String(tmp, 0, i));
+ }
+ } while (!channel.isClosed()) ;
+ return output.toString();
+ }
+
+ public void setExitCode(int exitCode) {
+ this.exitCode = exitCode;
+ }
+
+ public int getExitCode() {
+ return exitCode;
+ }
+
+ public String getStdOutputString() {
+ return stdOutputString;
+ }
+
+ public void setStdOutputString(String stdOutputString) {
+ this.stdOutputString = stdOutputString;
+ }
+
+ public String getStdErrorString() {
+ return stdErrorString;
+ }
+
+ public void setStdErrorString(String stdErrorString) {
+ this.stdErrorString = stdErrorString;
+ }
+
+ public ByteArrayOutputStream getErrorStream() {
+ return errorStream;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHKeyAuthentication.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHKeyAuthentication.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHKeyAuthentication.java
new file mode 100644
index 0000000..0acb357
--- /dev/null
+++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHKeyAuthentication.java
@@ -0,0 +1,92 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.runners.ssh;
+
+import org.apache.airavata.resources.Authentication;
+
+public class SSHKeyAuthentication extends Authentication{
+ private byte[] privateKey;
+ private byte[] publicKey;
+ private String passphrase;
+ private String knownHostsFilePath;
+ private String strictHostKeyChecking; // yes or no
+
+ public SSHKeyAuthentication(String userName, byte[] privateKey, byte[] publicKey, String passphrase, String knownHostsFilePath, boolean strictHostKeyChecking) {
+ this.userName = userName;
+ this.privateKey = privateKey;
+ this.publicKey = publicKey;
+ this.passphrase = passphrase;
+ this.knownHostsFilePath = knownHostsFilePath;
+ if(strictHostKeyChecking){
+ this.strictHostKeyChecking = "yes";
+ }else{
+ this.strictHostKeyChecking = "no";
+ }
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public byte[] getPrivateKey() {
+ return privateKey;
+ }
+
+ public void setPrivateKey(byte[] privateKey) {
+ this.privateKey = privateKey;
+ }
+
+ public byte[] getPublicKey() {
+ return publicKey;
+ }
+
+ public void setPublicKey(byte[] publicKey) {
+ this.publicKey = publicKey;
+ }
+
+ public String getPassphrase() {
+ return passphrase;
+ }
+
+ public void setPassphrase(String passphrase) {
+ this.passphrase = passphrase;
+ }
+
+ public String getKnownHostsFilePath() {
+ return knownHostsFilePath;
+ }
+
+ public void setKnownHostsFilePath(String knownHostsFilePath) {
+ this.knownHostsFilePath = knownHostsFilePath;
+ }
+
+ public String getStrictHostKeyChecking() {
+ return strictHostKeyChecking;
+ }
+
+ public void setStrictHostKeyChecking(String strictHostKeyChecking) {
+ this.strictHostKeyChecking = strictHostKeyChecking;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHRunner.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHRunner.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHRunner.java
new file mode 100644
index 0000000..6bb29a9
--- /dev/null
+++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHRunner.java
@@ -0,0 +1,394 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.runners.ssh;
+
+import com.jcraft.jsch.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.util.*;
+
+public class SSHRunner {
+ private final static Logger log = LoggerFactory.getLogger(SSHRunner.class);
+
+ public Session createSSHSession(SSHServerInfo serverInfo, SSHKeyAuthentication authentication) throws JSchException {
+ JSch jSch = new JSch();
+ jSch.addIdentity(UUID.randomUUID().toString(), authentication.getPrivateKey(), authentication.getPublicKey(),
+ authentication.getPassphrase().getBytes());
+ Session session = jSch.getSession(serverInfo.getUserName(), serverInfo.getHost(),
+ serverInfo.getSshPort());
+ session.setUserInfo(new SSHUserInfo(serverInfo.getUserName(), null, authentication.getPassphrase()));
+ if (authentication.getStrictHostKeyChecking().equals("yes")) {
+ jSch.setKnownHosts(authentication.getKnownHostsFilePath());
+ } else {
+ session.setConfig("StrictHostKeyChecking", "no");
+ }
+ session.connect();
+
+ return session;
+ }
+
+ public String scpTo(String routingKey, String localFile, String remoteFile, SSHServerInfo serverInfo,
+ SSHKeyAuthentication authentication) throws IOException, JSchException, SSHApiException {
+
+ Session session = createSSHSession(serverInfo, authentication);
+
+ FileInputStream fis = null;
+ String prefix = null;
+ if (new File(localFile).isDirectory()) {
+ prefix = localFile + File.separator;
+ }
+ boolean ptimestamp = true;
+
+ // exec 'scp -t rfile' remotely
+ String command = "scp " + (ptimestamp ? "-p" : "") + " -t " + remoteFile;
+ Channel channel = session.openChannel("exec");
+
+ SSHCommandOutputReader stdOutReader = new SSHCommandOutputReader();
+ ((ChannelExec) channel).setCommand(command);
+ ((ChannelExec) channel).setErrStream(stdOutReader.getErrorStream());
+
+ // get I/O streams for remote scp
+ OutputStream out = channel.getOutputStream();
+ InputStream in = channel.getInputStream();
+
+ channel.connect();
+
+ if (checkAck(in) != 0) {
+ String error = "Error Reading input Stream";
+ log.error(error);
+ throw new SSHApiException(error);
+ }
+
+ File _lfile = new File(localFile);
+
+ if (ptimestamp) {
+ command = "T" + (_lfile.lastModified() / 1000) + " 0";
+ // The access time should be sent here,
+ // but it is not accessible with JavaAPI ;-<
+ command += (" " + (_lfile.lastModified() / 1000) + " 0\n");
+ out.write(command.getBytes());
+ out.flush();
+ if (checkAck(in) != 0) {
+ String error = "Error Reading input Stream";
+ log.error(error);
+ throw new SSHApiException(error);
+ }
+ }
+
+ // send "C0644 filesize filename", where filename should not include '/'
+ long filesize = _lfile.length();
+ command = "C0644 " + filesize + " ";
+ if (localFile.lastIndexOf('/') > 0) {
+ command += localFile.substring(localFile.lastIndexOf('/') + 1);
+ } else {
+ command += localFile;
+ }
+ command += "\n";
+ out.write(command.getBytes());
+ out.flush();
+ if (checkAck(in) != 0) {
+ String error = "Error Reading input Stream";
+ log.error(error);
+ throw new SSHApiException(error);
+ }
+
+ // send a content of localFile
+ fis = new FileInputStream(localFile);
+ byte[] buf = new byte[1024];
+ while (true) {
+ int len = fis.read(buf, 0, buf.length);
+ if (len <= 0) break;
+ out.write(buf, 0, len); //out.flush();
+ }
+ fis.close();
+ fis = null;
+ // send '\0'
+ buf[0] = 0;
+ out.write(buf, 0, 1);
+ out.flush();
+ if (checkAck(in) != 0) {
+ String error = "Error Reading input Stream";
+ log.error(error);
+ throw new SSHApiException(error);
+ }
+ out.close();
+ stdOutReader.onOutput(channel);
+
+
+ channel.disconnect();
+ if (stdOutReader.getStdErrorString().contains("scp:")) {
+ throw new SSHApiException(stdOutReader.getStdErrorString());
+ }
+
+ session.disconnect();
+
+ //since remote file is always a file we just return the file
+ return remoteFile;
+ }
+
+ /**
+ * This method will copy a remote file to a local directory
+ *
+ * @param remoteFile remote file path, this has to be a full qualified path
+ * @param localFile This is the local file to copy, this can be a directory too
+ * @return returns the final local file path of the new file came from the remote resource
+ */
+ public void scpFrom(String routingKey, String remoteFile, String localFile, SSHServerInfo serverInfo,
+ SSHKeyAuthentication authentication) throws IOException,
+ JSchException, SSHApiException {
+ Session session = createSSHSession(serverInfo, authentication);
+ FileOutputStream fos = null;
+ try {
+ String prefix = null;
+ if (new File(localFile).isDirectory()) {
+ prefix = localFile + File.separator;
+ }
+
+ SSHCommandOutputReader stdOutReader = new SSHCommandOutputReader();
+
+ // exec 'scp -f remotefile' remotely
+ String command = "scp -f " + remoteFile;
+ Channel channel = session.openChannel("exec");
+ ((ChannelExec) channel).setCommand(command);
+ ((ChannelExec) channel).setErrStream(stdOutReader.getErrorStream());
+
+ // get I/O streams for remote scp
+ OutputStream out = channel.getOutputStream();
+ InputStream in = channel.getInputStream();
+
+ if (!channel.isClosed()){
+ channel.connect();
+ }
+
+ byte[] buf = new byte[1024];
+
+ // send '\0'
+ buf[0] = 0;
+ out.write(buf, 0, 1);
+ out.flush();
+
+ while (true) {
+ int c = checkAck(in);
+ if (c != 'C') {
+ break;
+ }
+
+ // read '0644 '
+ in.read(buf, 0, 5);
+
+ long filesize = 0L;
+ while (true) {
+ if (in.read(buf, 0, 1) < 0) {
+ // error
+ break;
+ }
+ if (buf[0] == ' ') break;
+ filesize = filesize * 10L + (long) (buf[0] - '0');
+ }
+
+ String file = null;
+ for (int i = 0; ; i++) {
+ in.read(buf, i, 1);
+ if (buf[i] == (byte) 0x0a) {
+ file = new String(buf, 0, i);
+ break;
+ }
+ }
+
+ //System.out.println("filesize="+filesize+", file="+file);
+
+ // send '\0'
+ buf[0] = 0;
+ out.write(buf, 0, 1);
+ out.flush();
+
+ // read a content of lfile
+ fos = new FileOutputStream(prefix == null ? localFile : prefix + file);
+ int foo;
+ while (true) {
+ if (buf.length < filesize) foo = buf.length;
+ else foo = (int) filesize;
+ foo = in.read(buf, 0, foo);
+ if (foo < 0) {
+ // error
+ break;
+ }
+ fos.write(buf, 0, foo);
+ filesize -= foo;
+ if (filesize == 0L) break;
+ }
+ fos.close();
+ fos = null;
+
+ if (checkAck(in) != 0) {
+ String error = "Error transfering the file content";
+ log.error(error);
+ throw new SSHApiException(error);
+ }
+
+ // send '\0'
+ buf[0] = 0;
+ out.write(buf, 0, 1);
+ out.flush();
+ }
+ stdOutReader.onOutput(channel);
+ if (stdOutReader.getStdErrorString().contains("scp:")) {
+ throw new SSHApiException(stdOutReader.getStdErrorString());
+ }
+
+ } catch (Exception e) {
+ log.error(e.getMessage(), e);
+ } finally {
+ try {
+ if (fos != null) fos.close();
+ session.disconnect();
+ } catch (Exception ee) {
+ }
+ }
+ }
+
+ public void makeDirectory(String routingKey, String path, SSHServerInfo serverInfo, SSHKeyAuthentication authentication)
+ throws IOException, JSchException, Exception {
+
+ Session session = createSSHSession(serverInfo, authentication);
+
+ // exec 'scp -t rfile' remotely
+ String command = "mkdir -p " + path;
+ Channel channel = session.openChannel("exec");
+ SSHCommandOutputReader stdOutReader = new SSHCommandOutputReader();
+
+ ((ChannelExec) channel).setCommand(command);
+ ((ChannelExec) channel).setErrStream(stdOutReader.getErrorStream());
+
+ try {
+ channel.connect();
+ } catch (JSchException e) {
+
+ channel.disconnect();
+ log.error("Unable to retrieve command output. Command - " + command +
+ " on server - " + session.getHost() + ":" + session.getPort() +
+ " connecting user name - "
+ + session.getUserName());
+ throw e;
+ }
+ stdOutReader.onOutput(channel);
+ if (stdOutReader.getStdErrorString().contains("mkdir:")) {
+ throw new Exception(stdOutReader.getStdErrorString());
+ }
+
+ channel.disconnect();
+ session.disconnect();
+ }
+
+ public List<String> listDirectory(String routingKey, String path, SSHServerInfo serverInfo, SSHKeyAuthentication authentication)
+ throws IOException, JSchException, Exception {
+
+ Session session = createSSHSession(serverInfo, authentication);
+
+ // exec 'scp -t rfile' remotely
+ String command = "ls " + path;
+ Channel channel = session.openChannel("exec");
+ SSHCommandOutputReader stdOutReader = new SSHCommandOutputReader();
+
+ ((ChannelExec) channel).setCommand(command);
+ ((ChannelExec) channel).setErrStream(stdOutReader.getErrorStream());
+
+ try {
+ channel.connect();
+ } catch (JSchException e) {
+
+ channel.disconnect();
+
+ throw new Exception("Unable to retrieve command output. Command - " + command +
+ " on server - " + session.getHost() + ":" + session.getPort() +
+ " connecting user name - "
+ + session.getUserName(), e);
+ }
+ if (stdOutReader.getStdErrorString().contains("ls:")) {
+ throw new Exception(stdOutReader.getStdErrorString());
+ }
+ channel.disconnect();
+ session.disconnect();
+ return Arrays.asList(stdOutReader.getStdOutputString().split("\n"));
+ }
+
+ public SSHCommandOutputReader executeCommand(String routingKey, String command, SSHServerInfo serverInfo,
+ SSHKeyAuthentication authentication) throws Exception {
+
+ Session session = createSSHSession(serverInfo, authentication);
+
+ Map<String, String> results = new HashMap<>();
+
+ Channel channel = session.openChannel("exec");
+ SSHCommandOutputReader stdOutReader = new SSHCommandOutputReader();
+
+ ((ChannelExec) channel).setCommand(command);
+ ((ChannelExec) channel).setErrStream(stdOutReader.getErrorStream());
+
+ try {
+ channel.connect();
+ } catch (JSchException e) {
+
+ channel.disconnect();
+
+ throw new Exception("Unable to retrieve command output. Command - " + command +
+ " on server - " + session.getHost() + ":" + session.getPort() +
+ " connecting user name - "
+ + session.getUserName(), e);
+ }
+ stdOutReader.onOutput(channel);
+ session.disconnect();
+
+ return stdOutReader;
+ }
+
+ public SSHCommandOutputReader executeCommand(String routingKey, String[] commands, SSHServerInfo serverInfo,
+ SSHKeyAuthentication authentication) throws Exception {
+ return executeCommand(routingKey, String.join(" && ", commands), serverInfo, authentication);
+ }
+
+ private int checkAck(InputStream in) throws IOException {
+ int b = in.read();
+ if (b == 0) return b;
+ if (b == -1) return b;
+
+ if (b == 1 || b == 2) {
+ StringBuffer sb = new StringBuffer();
+ int c;
+ do {
+ c = in.read();
+ sb.append((char) c);
+ }
+ while (c != '\n');
+ if (b == 1) { // error
+ System.out.print(sb.toString());
+ }
+ if (b == 2) { // fatal error
+ System.out.print(sb.toString());
+ }
+ log.warn(sb.toString());
+ }
+ return b;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHServerInfo.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHServerInfo.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHServerInfo.java
new file mode 100644
index 0000000..6a99bc1
--- /dev/null
+++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHServerInfo.java
@@ -0,0 +1,52 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.runners.ssh;
+
+import org.apache.airavata.resources.ServerInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SSHServerInfo extends ServerInfo {
+ private final static Logger logger = LoggerFactory.getLogger(SSHServerInfo.class);
+
+ SSHKeyAuthentication authentication;
+ int sshPort;
+
+ public SSHServerInfo(String userName, String host, SSHKeyAuthentication authentication, int port){
+ super(userName, host, ComProtocol.SSH, port);
+ this.authentication = authentication;
+ this.sshPort = port;
+ }
+
+ public SSHServerInfo(String userName, String host, SSHKeyAuthentication authentication){
+ super(userName, host, ComProtocol.SSH, 22);
+ this.authentication = authentication;
+ this.sshPort = 22;
+ }
+
+ public SSHKeyAuthentication getAuthentication() {
+ return authentication;
+ }
+
+ public int getSshPort() {
+ return sshPort;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHUserInfo.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHUserInfo.java b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHUserInfo.java
new file mode 100644
index 0000000..9f2792b
--- /dev/null
+++ b/airavata-layered-architecture/src/main/java/org/apache/airavata/runners/ssh/SSHUserInfo.java
@@ -0,0 +1,66 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.runners.ssh;
+
+import com.jcraft.jsch.UserInfo;
+
+public class SSHUserInfo implements UserInfo {
+
+ private String userName;
+ private String password;
+ private String passphrase;
+
+ public SSHUserInfo(String userName, String password, String passphrase) {
+ this.userName = userName;
+ this.password = password;
+ this.passphrase = passphrase;
+ }
+
+ @Override
+ public String getPassphrase() {
+ return null;
+ }
+
+ @Override
+ public String getPassword() {
+ return null;
+ }
+
+ @Override
+ public boolean promptPassword(String s) {
+ return false;
+ }
+
+ @Override
+ public boolean promptPassphrase(String s) {
+ return false;
+ }
+
+ @Override
+ public boolean promptYesNo(String s) {
+ return false;
+ }
+
+ @Override
+ public void showMessage(String s) {
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/main/resources/resources/batch/PBS_Groovy.template
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/main/resources/resources/batch/PBS_Groovy.template b/airavata-layered-architecture/src/main/resources/resources/batch/PBS_Groovy.template
new file mode 100644
index 0000000..7e30b7b
--- /dev/null
+++ b/airavata-layered-architecture/src/main/resources/resources/batch/PBS_Groovy.template
@@ -0,0 +1,34 @@
+#!${shellName}
+
+<%
+ if(shellName != null && shellName != "") out.print '#PBS -S ' + shellName + '\n'
+ if (queueName != null && queueName != "") out.print '#PBS -q ' + queueName + '\n'
+ if (jobName != null && jobName != "") out.print '#PBS -N ' + jobName + '\n'
+ if (mailAddress != null && mailAddress != "") out.print '#PBS -M ' + mailAddress + '\n'
+ if (accountString != null && accountString != "") out.print '#PBS -A ' + accountString + '\n'
+ if (maxWallTime != null && maxWallTime != "") out.print '#PBS -l walltime=' + maxWallTime + '\n'
+ if (jobSubmitterCommand != null && jobSubmitterCommand != "" && jobSubmitterCommand == "ccmrun")
+ out.print '#PBS -l gres=ccm \n'
+ if (standardOutFile != null && standardOutFile != "") out.print '#PBS -o ' + standardOutFile + '\n'
+ if (standardErrorFile != null && standardErrorFile != "") out.print '#PBS -e ' + standardErrorFile + '\n'
+ if (usedMem != null && usedMem != "") out.print '#PBS -l vmem=' + usedMem + 'M\n'
+ if (nodes != null && nodes != "" && processPerNode != null && processPerNode != "")
+ out.print '#PBS -l nodes=' + nodes + ':ppn=' + processPerNode + '\n'
+%>
+#PBS -m bea
+
+<% if (exports != null) for(com in exports) out.print 'export ' + com +'\n'
+ if (moduleCommands != null) for(mc in moduleCommands) out.print mc +'\n'
+ if (workingDirectory != null && workingDirectory != "") out.print 'cd ' + workingDirectory +'\n'
+ if (preJobCommands != null) for(pjc in preJobCommands) out.print pjc +'\n'
+ if (jobSubmitterCommand != null && jobSubmitterCommand != ""){
+ out.print jobSubmitterCommand + ' '
+ if(jobSubmitterCommand != "ccmrun"){
+ out.print cpuCount + ' '
+ }
+ }
+ if (executablePath != null && executablePath != "") out.print executablePath + ' '
+ if (inputs != null) for(input in inputs) out.print input + ' '
+ out.print '\n'
+ if (postJobCommands != null) for(pjc in postJobCommands) out.print pjc +'\n'
+%>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/test/java/org/apache/airavata/Constants.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/test/java/org/apache/airavata/Constants.java b/airavata-layered-architecture/src/test/java/org/apache/airavata/Constants.java
new file mode 100644
index 0000000..539cfb5
--- /dev/null
+++ b/airavata-layered-architecture/src/test/java/org/apache/airavata/Constants.java
@@ -0,0 +1,30 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Constants {
+ private final static Logger logger = LoggerFactory.getLogger(Constants.class);
+
+ public static final String loginUserName = "snakanda";
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata-sandbox/blob/43a79e3a/airavata-layered-architecture/src/test/java/org/apache/airavata/resources/batch/HPCBatchResourceTest.java
----------------------------------------------------------------------
diff --git a/airavata-layered-architecture/src/test/java/org/apache/airavata/resources/batch/HPCBatchResourceTest.java b/airavata-layered-architecture/src/test/java/org/apache/airavata/resources/batch/HPCBatchResourceTest.java
new file mode 100644
index 0000000..525e28f
--- /dev/null
+++ b/airavata-layered-architecture/src/test/java/org/apache/airavata/resources/batch/HPCBatchResourceTest.java
@@ -0,0 +1,86 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.resources.batch;
+
+import org.apache.airavata.Constants;
+import org.apache.airavata.runners.ssh.SSHKeyAuthentication;
+import org.apache.airavata.runners.ssh.SSHRunnerTest;
+import org.apache.airavata.runners.ssh.SSHServerInfo;
+import org.apache.commons.io.IOUtils;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+public class HPCBatchResourceTest {
+ private final static Logger logger = LoggerFactory.getLogger(HPCBatchResourceTest.class);
+
+ @Test
+ public void test() throws Exception {
+ SSHKeyAuthentication br2SshAuthentication = new SSHKeyAuthentication(
+ Constants.loginUserName,
+ IOUtils.toByteArray(SSHRunnerTest.class.getClassLoader().getResourceAsStream("ssh/id_rsa")),
+ IOUtils.toByteArray(SSHRunnerTest.class.getClassLoader().getResourceAsStream("ssh/id_rsa.pub")),
+ "dummy",
+ SSHRunnerTest.class.getClassLoader().getResource("ssh/known_hosts").getPath(),
+ false
+ );
+ SSHServerInfo br2 = new SSHServerInfo(Constants.loginUserName, "bigred2.uits.iu.edu", br2SshAuthentication,22);
+ Map<JobManagerConfiguration.JobManagerCommand, String> jobManagerCommands = new HashMap<>();
+ jobManagerCommands.put(JobManagerConfiguration.JobManagerCommand.SUBMISSION, "qsub");
+ jobManagerCommands.put(JobManagerConfiguration.JobManagerCommand.JOB_MONITORING, "qstat");
+ jobManagerCommands.put(JobManagerConfiguration.JobManagerCommand.DELETION, "qdel");
+
+ JobManagerConfiguration pbsJobConfiguration = new PBSJobConfiguration(PBSJobConfiguration.class.getClassLoader().
+ getResource("resources/batch/PBS_Groovy.template").getPath(), ".pbs",
+ "/opt/torque/torque-5.0.1/bin", jobManagerCommands, new BatchJobOutputParser());
+ HPCBatchResource hpcBatchResource = new HPCBatchResource(br2, pbsJobConfiguration, br2SshAuthentication);
+
+ GroovyMap jobSpecification = new GroovyMap();
+
+ jobSpecification.add(Script.NODES, 1);
+ jobSpecification.add(Script.PROCESS_PER_NODE, 16);
+ jobSpecification.add(Script.MAX_WALL_TIME, "00:30:00");
+
+ jobSpecification.add(Script.QUEUE_NAME, "debug_gpu");
+
+ jobSpecification.add(Script.MAIL_ADDRESS, "supun.nakandala@gmail.com");
+
+ List<String> moduleLoads = new ArrayList<>();
+ moduleLoads.add("module load ccm");
+ moduleLoads.add("module load singularity");
+ jobSpecification.add(Script.MODULE_COMMANDS, moduleLoads);
+
+ jobSpecification.add(Script.WORKING_DIR, "/N/dc2/scratch/snakanda/work-dirs");
+ List<String> inputs = new ArrayList<>();
+ inputs.add("~/airavata/code_tf.py");
+ jobSpecification.add(Script.INPUTS, inputs);
+ jobSpecification.add(Script.EXECUTABLE_PATH, "singularity exec /N/soft/cle5/singularity/images/tensorflow1.1-ubuntu-py2.7.11-test.img python");
+ jobSpecification.add(Script.JOB_SUBMITTER_COMMAND,"ccmrun");
+
+ jobSpecification.add(Script.STANDARD_OUT_FILE, "STDOUT.txt");
+ jobSpecification.add(Script.STANDARD_ERROR_FILE, "STDERR.txt");
+
+ String routingKey = UUID.randomUUID().toString();
+ hpcBatchResource.submitBatchJob(routingKey, jobSpecification, "~/airavata");
+ }
+}
\ No newline at end of file