You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2014/03/07 00:14:41 UTC
[1/3] Implementing slurm support - AIRAVATA-1057
Repository: airavata
Updated Branches:
refs/heads/master f986e09c1 -> 532f2691e
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
new file mode 100644
index 0000000..ca9094e
--- /dev/null
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
@@ -0,0 +1,417 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gsi.ssh.impl;
+
+import com.jcraft.jsch.*;
+import org.apache.airavata.gsi.ssh.api.*;
+import org.apache.airavata.gsi.ssh.api.authentication.*;
+import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
+import org.apache.airavata.gsi.ssh.api.job.JobManagerConfiguration;
+import org.apache.airavata.gsi.ssh.api.job.OutputParser;
+import org.apache.airavata.gsi.ssh.config.ConfigReader;
+import org.apache.airavata.gsi.ssh.jsch.ExtendedJSch;
+import org.apache.airavata.gsi.ssh.util.CommonUtils;
+import org.apache.airavata.gsi.ssh.util.SSHAPIUIKeyboardInteractive;
+import org.apache.airavata.gsi.ssh.util.SSHKeyPasswordHandler;
+import org.apache.airavata.gsi.ssh.util.SSHUtils;
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.transform.*;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.net.URL;
+import java.security.SecureRandom;
+
+public class GSISSHAbstractCluster implements Cluster {
+ static {
+ JSch.setConfig("gssapi-with-mic.x509", "org.apache.airavata.gsi.ssh.GSSContextX509");
+ JSch.setConfig("userauth.gssapi-with-mic", "com.jcraft.jsch.UserAuthGSSAPIWithMICGSSCredentials");
+
+ }
+
+ private static final Logger log = LoggerFactory.getLogger(GSISSHAbstractCluster.class);
+ public static final String X509_CERT_DIR = "X509_CERT_DIR";
+ public static final String SSH_SESSION_TIMEOUT = "ssh.session.timeout";
+
+ public JobManagerConfiguration jobManagerConfiguration;
+
+ private ServerInfo serverInfo;
+
+ private AuthenticationInfo authenticationInfo;
+
+ private Session session;
+
+ private ConfigReader configReader;
+
+
+ public GSISSHAbstractCluster(ServerInfo serverInfo, AuthenticationInfo authenticationInfo, JobManagerConfiguration config) throws SSHApiException {
+ this(serverInfo, authenticationInfo);
+ this.jobManagerConfiguration = config;
+ }
+
+ public GSISSHAbstractCluster(ServerInfo serverInfo, AuthenticationInfo authenticationInfo) throws SSHApiException {
+
+ this.serverInfo = serverInfo;
+
+ this.authenticationInfo = authenticationInfo;
+
+ if (authenticationInfo instanceof GSIAuthenticationInfo) {
+ System.setProperty(X509_CERT_DIR, (String) ((GSIAuthenticationInfo) authenticationInfo).getProperties().
+ get("X509_CERT_DIR"));
+ }
+
+
+ try {
+ this.configReader = new ConfigReader();
+ } catch (IOException e) {
+ throw new SSHApiException("Unable to load system configurations.", e);
+ }
+ JSch jSch = new ExtendedJSch();
+
+ log.debug("Connecting to server - " + serverInfo.getHost() + ":" + serverInfo.getPort() + " with user name - "
+ + serverInfo.getUserName());
+
+ try {
+ session = jSch.getSession(serverInfo.getUserName(), serverInfo.getHost(), serverInfo.getPort());
+ session.setTimeout(Integer.parseInt(configReader.getConfiguration(SSH_SESSION_TIMEOUT)));
+ } catch (Exception e) {
+ throw new SSHApiException("An exception occurred while creating SSH session." +
+ "Connecting server - " + serverInfo.getHost() + ":" + serverInfo.getPort() +
+ " connecting user name - "
+ + serverInfo.getUserName(), e);
+ }
+
+ java.util.Properties config = this.configReader.getProperties();
+ session.setConfig(config);
+
+
+ //=============================================================
+ // Handling vanilla SSH pieces
+ //=============================================================
+ if (authenticationInfo instanceof SSHPasswordAuthentication) {
+ String password = ((SSHPasswordAuthentication) authenticationInfo).
+ getPassword(serverInfo.getUserName(), serverInfo.getHost());
+
+ session.setUserInfo(new SSHAPIUIKeyboardInteractive(password));
+
+ // TODO figure out why we need to set password to session
+ session.setPassword(password);
+
+ } else if (authenticationInfo instanceof SSHPublicKeyFileAuthentication) {
+ SSHPublicKeyFileAuthentication sshPublicKeyFileAuthentication
+ = (SSHPublicKeyFileAuthentication) authenticationInfo;
+
+ String privateKeyFile = sshPublicKeyFileAuthentication.
+ getPrivateKeyFile(serverInfo.getUserName(), serverInfo.getHost());
+
+ logDebug("The private key file for vanilla SSH " + privateKeyFile);
+
+ String publicKeyFile = sshPublicKeyFileAuthentication.
+ getPrivateKeyFile(serverInfo.getUserName(), serverInfo.getHost());
+
+ logDebug("The public key file for vanilla SSH " + publicKeyFile);
+
+ Identity identityFile;
+
+ try {
+ identityFile = GSISSHIdentityFile.newInstance(privateKeyFile, null, jSch);
+ } catch (JSchException e) {
+ throw new SSHApiException("An exception occurred while initializing keys using files. " +
+ "(private key and public key)." +
+ "Connecting server - " + serverInfo.getHost() + ":" + serverInfo.getPort() +
+ " connecting user name - "
+ + serverInfo.getUserName() + " private key file - " + privateKeyFile + ", public key file - " +
+ publicKeyFile, e);
+ }
+
+ // Add identity to identity repository
+ GSISSHIdentityRepository identityRepository = new GSISSHIdentityRepository(jSch);
+ identityRepository.add(identityFile);
+
+ // Set repository to session
+ session.setIdentityRepository(identityRepository);
+
+ // Set the user info
+ SSHKeyPasswordHandler sshKeyPasswordHandler
+ = new SSHKeyPasswordHandler((SSHKeyAuthentication) authenticationInfo);
+
+ session.setUserInfo(sshKeyPasswordHandler);
+
+ } else if (authenticationInfo instanceof SSHPublicKeyAuthentication) {
+
+ SSHPublicKeyAuthentication sshPublicKeyAuthentication
+ = (SSHPublicKeyAuthentication) authenticationInfo;
+
+ Identity identityFile;
+
+ try {
+ String name = serverInfo.getUserName() + "_" + serverInfo.getHost();
+ identityFile = GSISSHIdentityFile.newInstance(name,
+ sshPublicKeyAuthentication.getPrivateKey(serverInfo.getUserName(), serverInfo.getHost()),
+ sshPublicKeyAuthentication.getPublicKey(serverInfo.getUserName(), serverInfo.getHost()), jSch);
+ } catch (JSchException e) {
+ throw new SSHApiException("An exception occurred while initializing keys using byte arrays. " +
+ "(private key and public key)." +
+ "Connecting server - " + serverInfo.getHost() + ":" + serverInfo.getPort() +
+ " connecting user name - "
+ + serverInfo.getUserName(), e);
+ }
+
+ // Add identity to identity repository
+ GSISSHIdentityRepository identityRepository = new GSISSHIdentityRepository(jSch);
+ identityRepository.add(identityFile);
+
+ // Set repository to session
+ session.setIdentityRepository(identityRepository);
+
+ // Set the user info
+ SSHKeyPasswordHandler sshKeyPasswordHandler
+ = new SSHKeyPasswordHandler((SSHKeyAuthentication) authenticationInfo);
+
+ session.setUserInfo(sshKeyPasswordHandler);
+
+ }
+
+ // Not a good way, but we dont have any choice
+ if (session instanceof ExtendedSession) {
+ if (authenticationInfo instanceof GSIAuthenticationInfo) {
+ ((ExtendedSession) session).setAuthenticationInfo((GSIAuthenticationInfo) authenticationInfo);
+ }
+ }
+
+ try {
+ session.connect();
+ } catch (JSchException e) {
+ throw new SSHApiException("An exception occurred while connecting to server." +
+ "Connecting server - " + serverInfo.getHost() + ":" + serverInfo.getPort() +
+ " connecting user name - "
+ + serverInfo.getUserName(), e);
+ }
+ System.out.println(session.isConnected());
+ }
+
+ public JobDescriptor cancelJob(String jobID) throws SSHApiException {
+ RawCommandInfo rawCommandInfo = jobManagerConfiguration.getCancelCommand(jobID);
+
+ StandardOutReader stdOutReader = new StandardOutReader();
+ CommandExecutor.executeCommand(rawCommandInfo, this.getSession(), stdOutReader);
+ String outputifAvailable = getOutputifAvailable(stdOutReader, "Error reading output of job submission");
+ // this might not be the case for all teh resources, if so Cluster implementation can override this method
+ // because here after cancelling we try to get the job description and return it back
+ JobDescriptor jobById = this.getJobDescriptorById(jobID);
+ if (CommonUtils.isJobFinished(jobById)) {
+ log.debug("Job Cancel operation was successful !");
+ return jobById;
+ } else {
+ log.debug("Job Cancel operation was not successful !");
+ return null;
+ }
+ }
+
+ public String submitBatchJobWithScript(String scriptPath, String workingDirectory) throws SSHApiException {
+ this.scpTo(workingDirectory, scriptPath);
+
+ // since this is a constant we do not ask users to fill this
+
+// RawCommandInfo rawCommandInfo = new RawCommandInfo(this.installedPath + this.jobManagerConfiguration.getSubmitCommand() + " " +
+// workingDirectory + File.separator + FilenameUtils.getName(scriptPath));
+
+ RawCommandInfo rawCommandInfo = jobManagerConfiguration.getSubmitCommand(workingDirectory,scriptPath);
+ StandardOutReader standardOutReader = new StandardOutReader();
+ CommandExecutor.executeCommand(rawCommandInfo, this.session, standardOutReader);
+
+ //Check whether pbs submission is successful or not, if it failed throw and exception in submitJob method
+ // with the error thrown in qsub command
+ //
+ String outputifAvailable = getOutputifAvailable(standardOutReader,"Error reading output of job submission");
+ OutputParser outputParser = jobManagerConfiguration.getParser();
+ return outputParser.parse(outputifAvailable);
+ }
+
+ public String submitBatchJob(JobDescriptor jobDescriptor) throws SSHApiException {
+ TransformerFactory factory = TransformerFactory.newInstance();
+ URL resource = this.getClass().getClassLoader().getResource(jobManagerConfiguration.getJobDescriptionTemplateName());
+
+ if (resource == null) {
+ String error = "System configuration file '" + jobManagerConfiguration.getJobDescriptionTemplateName()
+ + "' not found in the classpath";
+ throw new SSHApiException(error);
+ }
+
+ Source xslt = new StreamSource(new File(resource.getPath()));
+ Transformer transformer;
+ StringWriter results = new StringWriter();
+ File tempPBSFile = null;
+ try {
+ // generate the pbs script using xslt
+ transformer = factory.newTransformer(xslt);
+ Source text = new StreamSource(new ByteArrayInputStream(jobDescriptor.toXML().getBytes()));
+ transformer.transform(text, new StreamResult(results));
+ String scriptContent = results.toString().replaceAll("^[ |\t]*\n$", "");
+ if (scriptContent.startsWith("\n")) {
+ scriptContent = scriptContent.substring(1);
+ }
+// log.debug("generated PBS:" + results.toString());
+
+ // creating a temporary file using pbs script generated above
+ int number = new SecureRandom().nextInt();
+ number = (number < 0 ? -number : number);
+
+ tempPBSFile = new File(Integer.toString(number) + jobManagerConfiguration.getScriptExtension());
+ FileUtils.writeStringToFile(tempPBSFile, scriptContent);
+
+ //reusing submitBatchJobWithScript method to submit a job
+
+ String jobID = this.submitBatchJobWithScript(tempPBSFile.getAbsolutePath(),
+ jobDescriptor.getWorkingDirectory());
+ log.debug("Job has successfully submitted, JobID : " + jobID);
+ return jobID.replace("\n", "");
+ } catch (TransformerConfigurationException e) {
+ throw new SSHApiException("Error parsing PBS transformation", e);
+ } catch (TransformerException e) {
+ throw new SSHApiException("Error generating PBS script", e);
+ } catch (IOException e) {
+ throw new SSHApiException("An exception occurred while connecting to server." +
+ "Connecting server - " + serverInfo.getHost() + ":" + serverInfo.getPort() +
+ " connecting user name - "
+ + serverInfo.getUserName(), e);
+ } finally {
+ if (tempPBSFile != null) {
+ tempPBSFile.delete();
+ }
+ }
+ }
+
+
+
+ public JobDescriptor getJobDescriptorById(String jobID) throws SSHApiException {
+ RawCommandInfo rawCommandInfo = jobManagerConfiguration.getMonitorCommand(jobID);
+ StandardOutReader stdOutReader = new StandardOutReader();
+ CommandExecutor.executeCommand(rawCommandInfo, this.getSession(), stdOutReader);
+ String result = getOutputifAvailable(stdOutReader, "Error getting job information from the resource !");
+ JobDescriptor jobDescriptor = new JobDescriptor();
+ jobManagerConfiguration.getParser().parse(jobDescriptor,result);
+ return jobDescriptor;
+ }
+
+ public JobStatus getJobStatus(String jobID) throws SSHApiException {
+ RawCommandInfo rawCommandInfo = jobManagerConfiguration.getMonitorCommand(jobID);
+ StandardOutReader stdOutReader = new StandardOutReader();
+ CommandExecutor.executeCommand(rawCommandInfo, this.getSession(), stdOutReader);
+ String result = getOutputifAvailable(stdOutReader, "Error getting job information from the resource !");
+ return jobManagerConfiguration.getParser().parse(jobID, result);
+ }
+
+ private static void logDebug(String message) {
+ if (log.isDebugEnabled()) {
+ log.debug(message);
+ }
+ }
+
+ public JobManagerConfiguration getJobManagerConfiguration() {
+ return jobManagerConfiguration;
+ }
+
+ public void setJobManagerConfiguration(JobManagerConfiguration jobManagerConfiguration) {
+ this.jobManagerConfiguration = jobManagerConfiguration;
+ }
+
+ public void scpTo(String remoteFile, String localFile) throws SSHApiException {
+ try {
+ SSHUtils.scpTo(remoteFile, localFile, session);
+ } catch (IOException e) {
+ throw new SSHApiException("Failed during scping local file:" + localFile + " to remote file "
+ + serverInfo.getHost() + ":rFile", e);
+ } catch (JSchException e) {
+ throw new SSHApiException("Failed during scping local file:" + localFile + " to remote file "
+ + serverInfo.getHost() + ":rFile", e);
+ }
+ }
+
+ public void scpFrom(String remoteFile, String localFile) throws SSHApiException {
+ try {
+ SSHUtils.scpFrom(remoteFile, localFile, session);
+ } catch (IOException e) {
+ throw new SSHApiException("Failed during scping local file:" + localFile + " to remote file "
+ + serverInfo.getHost() + ":rFile", e);
+ } catch (JSchException e) {
+ throw new SSHApiException("Failed during scping local file:" + localFile + " to remote file "
+ + serverInfo.getHost() + ":rFile", e);
+ }
+ }
+
+ public void makeDirectory(String directoryPath) throws SSHApiException {
+ try {
+ SSHUtils.makeDirectory(directoryPath, session);
+ } catch (IOException e) {
+ throw new SSHApiException("Failed during creating directory:" + directoryPath + " to remote file "
+ + serverInfo.getHost() + ":rFile", e);
+ } catch (JSchException e) {
+ throw new SSHApiException("Failed during creating directory :" + directoryPath + " to remote file "
+ + serverInfo.getHost() + ":rFile", e);
+ }
+ }
+
+
+ public ServerInfo getServerInfo() {
+ return serverInfo;
+ }
+
+ public AuthenticationInfo getAuthenticationInfo() {
+ return authenticationInfo;
+ }
+
+ /**
+ * This gaurantee to return a valid session
+ *
+ * @return
+ */
+ public Session getSession() {
+ return this.session;
+ }
+
+ /**
+ * This method will read standard output and if there's any it will be parsed
+ *
+ * @param jobIDReaderCommandOutput
+ * @param errorMsg
+ * @return
+ * @throws SSHApiException
+ */
+ private String getOutputifAvailable(StandardOutReader jobIDReaderCommandOutput, String errorMsg) throws SSHApiException {
+ String stdOutputString = jobIDReaderCommandOutput.getStdOutputString();
+ String stdErrorString = jobIDReaderCommandOutput.getStdErrorString();
+
+ if ((stdOutputString == null && "".equals(stdOutputString)) &&
+ ((stdErrorString != null) && !("".equals(stdErrorString)))) {
+ log.error("Standard Error output : " + stdErrorString);
+ throw new SSHApiException(errorMsg + stdErrorString);
+ }
+ return stdOutputString;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/PBSCluster.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/PBSCluster.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/PBSCluster.java
index 7a78b89..4efeef1 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/PBSCluster.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/PBSCluster.java
@@ -23,629 +23,20 @@ package org.apache.airavata.gsi.ssh.impl;
import com.jcraft.jsch.*;
import org.apache.airavata.gsi.ssh.api.*;
import org.apache.airavata.gsi.ssh.api.authentication.*;
-import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
-import org.apache.airavata.gsi.ssh.config.ConfigReader;
-import org.apache.airavata.gsi.ssh.jsch.ExtendedJSch;
-import org.apache.airavata.gsi.ssh.util.CommonUtils;
-import org.apache.airavata.gsi.ssh.util.SSHAPIUIKeyboardInteractive;
-import org.apache.airavata.gsi.ssh.util.SSHKeyPasswordHandler;
-import org.apache.airavata.gsi.ssh.util.SSHUtils;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.FilenameUtils;
+import org.apache.airavata.gsi.ssh.api.job.JobManagerConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.xml.transform.*;
-import javax.xml.transform.stream.StreamResult;
-import javax.xml.transform.stream.StreamSource;
-import java.io.*;
-import java.net.URL;
-import java.security.SecureRandom;
-import java.util.ArrayList;
/**
* This is the default implementation of a cluster.
* this has most of the methods to be used by the end user of the
* library.
*/
-public class PBSCluster implements Cluster {
- static {
- JSch.setConfig("gssapi-with-mic.x509", "org.apache.airavata.gsi.ssh.GSSContextX509");
- JSch.setConfig("userauth.gssapi-with-mic", "com.jcraft.jsch.UserAuthGSSAPIWithMICGSSCredentials");
-
- }
-
+public class PBSCluster extends GSISSHAbstractCluster {
private static final Logger log = LoggerFactory.getLogger(PBSCluster.class);
- public static final String X509_CERT_DIR = "X509_CERT_DIR";
- public static final String SSH_SESSION_TIMEOUT = "ssh.session.timeout";
- public static final String PBSTEMPLATE_XSLT = "PBSTemplate.xslt";
-
- private Node[] Nodes;
-
- private ServerInfo serverInfo;
-
- private AuthenticationInfo authenticationInfo;
-
- private Session session;
-
- private ConfigReader configReader;
-
- private String installedPath;
-
- public PBSCluster(ServerInfo serverInfo, AuthenticationInfo authenticationInfo, String installedPath) throws SSHApiException {
-
- this.serverInfo = serverInfo;
-
- this.authenticationInfo = authenticationInfo;
-
- if (authenticationInfo instanceof GSIAuthenticationInfo) {
- System.setProperty(X509_CERT_DIR, (String) ((GSIAuthenticationInfo) authenticationInfo).getProperties().
- get("X509_CERT_DIR"));
- }
- if(installedPath == null) {
- throw new SSHApiException("Installed path cannot be null !!");
- }
- if (installedPath.endsWith("/")) {
- this.installedPath = installedPath;
- } else {
- this.installedPath = installedPath + "/";
- }
-
-
- try {
- this.configReader = new ConfigReader();
- } catch (IOException e) {
- throw new SSHApiException("Unable to load system configurations.", e);
- }
- JSch jSch = new ExtendedJSch();
-
- log.debug("Connecting to server - " + serverInfo.getHost() + ":" + serverInfo.getPort() + " with user name - "
- + serverInfo.getUserName());
-
- try {
- session = jSch.getSession(serverInfo.getUserName(), serverInfo.getHost(), serverInfo.getPort());
- session.setTimeout(Integer.parseInt(configReader.getConfiguration(SSH_SESSION_TIMEOUT)));
- } catch (Exception e) {
- throw new SSHApiException("An exception occurred while creating SSH session." +
- "Connecting server - " + serverInfo.getHost() + ":" + serverInfo.getPort() +
- " connecting user name - "
- + serverInfo.getUserName(), e);
- }
-
- java.util.Properties config = this.configReader.getProperties();
- session.setConfig(config);
-
-
- //=============================================================
- // Handling vanilla SSH pieces
- //=============================================================
- if (authenticationInfo instanceof SSHPasswordAuthentication) {
- String password = ((SSHPasswordAuthentication) authenticationInfo).
- getPassword(serverInfo.getUserName(), serverInfo.getHost());
-
- session.setUserInfo(new SSHAPIUIKeyboardInteractive(password));
-
- // TODO figure out why we need to set password to session
- session.setPassword(password);
-
- } else if (authenticationInfo instanceof SSHPublicKeyFileAuthentication) {
- SSHPublicKeyFileAuthentication sshPublicKeyFileAuthentication
- = (SSHPublicKeyFileAuthentication) authenticationInfo;
-
- String privateKeyFile = sshPublicKeyFileAuthentication.
- getPrivateKeyFile(serverInfo.getUserName(), serverInfo.getHost());
-
- logDebug("The private key file for vanilla SSH " + privateKeyFile);
-
- String publicKeyFile = sshPublicKeyFileAuthentication.
- getPrivateKeyFile(serverInfo.getUserName(), serverInfo.getHost());
-
- logDebug("The public key file for vanilla SSH " + publicKeyFile);
-
- Identity identityFile;
-
- try {
- identityFile = GSISSHIdentityFile.newInstance(privateKeyFile, null, jSch);
- } catch (JSchException e) {
- throw new SSHApiException("An exception occurred while initializing keys using files. " +
- "(private key and public key)." +
- "Connecting server - " + serverInfo.getHost() + ":" + serverInfo.getPort() +
- " connecting user name - "
- + serverInfo.getUserName() + " private key file - " + privateKeyFile + ", public key file - " +
- publicKeyFile, e);
- }
-
- // Add identity to identity repository
- GSISSHIdentityRepository identityRepository = new GSISSHIdentityRepository(jSch);
- identityRepository.add(identityFile);
-
- // Set repository to session
- session.setIdentityRepository(identityRepository);
-
- // Set the user info
- SSHKeyPasswordHandler sshKeyPasswordHandler
- = new SSHKeyPasswordHandler((SSHKeyAuthentication) authenticationInfo);
-
- session.setUserInfo(sshKeyPasswordHandler);
-
- } else if (authenticationInfo instanceof SSHPublicKeyAuthentication) {
-
- SSHPublicKeyAuthentication sshPublicKeyAuthentication
- = (SSHPublicKeyAuthentication) authenticationInfo;
-
- Identity identityFile;
-
- try {
- String name = serverInfo.getUserName() + "_" + serverInfo.getHost();
- identityFile = GSISSHIdentityFile.newInstance(name,
- sshPublicKeyAuthentication.getPrivateKey(serverInfo.getUserName(), serverInfo.getHost()),
- sshPublicKeyAuthentication.getPublicKey(serverInfo.getUserName(), serverInfo.getHost()), jSch);
- } catch (JSchException e) {
- throw new SSHApiException("An exception occurred while initializing keys using byte arrays. " +
- "(private key and public key)." +
- "Connecting server - " + serverInfo.getHost() + ":" + serverInfo.getPort() +
- " connecting user name - "
- + serverInfo.getUserName(), e);
- }
-
- // Add identity to identity repository
- GSISSHIdentityRepository identityRepository = new GSISSHIdentityRepository(jSch);
- identityRepository.add(identityFile);
-
- // Set repository to session j
- session.setIdentityRepository(identityRepository);
-
- // Set the user info
- SSHKeyPasswordHandler sshKeyPasswordHandler
- = new SSHKeyPasswordHandler((SSHKeyAuthentication) authenticationInfo);
-
- session.setUserInfo(sshKeyPasswordHandler);
-
- }
-
- // Not a good way, but we dont have any choice
- if (session instanceof ExtendedSession) {
- if (authenticationInfo instanceof GSIAuthenticationInfo) {
- ((ExtendedSession) session).setAuthenticationInfo((GSIAuthenticationInfo) authenticationInfo);
- }
- }
-
- try {
- session.connect();
- } catch (JSchException e) {
- throw new SSHApiException("An exception occurred while connecting to server." +
- "Connecting server - " + serverInfo.getHost() + ":" + serverInfo.getPort() +
- " connecting user name - "
- + serverInfo.getUserName(), e);
- }
- System.out.println(session.isConnected());
- }
-
-
- public String submitBatchJobWithPBS(String pbsFilePath, String workingDirectory) throws SSHApiException {
-
- this.scpTo(workingDirectory, pbsFilePath);
-
- // since this is a constant we do not ask users to fill this
- RawCommandInfo rawCommandInfo = new RawCommandInfo(this.installedPath + "qsub " +
- workingDirectory + File.separator + FilenameUtils.getName(pbsFilePath));
-
- StandardOutReader jobIDReaderCommandOutput = new StandardOutReader();
- CommandExecutor.executeCommand(rawCommandInfo, this.session, jobIDReaderCommandOutput);
-
- //Check whether pbs submission is successful or not, if it failed throw and exception in submitJob method
- // with the error thrown in qsub command
- //
- String stdOutputString = getOutputifAvailable(jobIDReaderCommandOutput, "Error submitting job to resource");
- return stdOutputString;
-
- }
-
- /**
- * This method will read standard output and if there's any it will be parsed
- * @param jobIDReaderCommandOutput
- * @param errorMsg
- * @return
- * @throws SSHApiException
- */
- private String getOutputifAvailable(StandardOutReader jobIDReaderCommandOutput, String errorMsg) throws SSHApiException {
- String stdOutputString = jobIDReaderCommandOutput.getStdOutputString();
- String stdErrorString = jobIDReaderCommandOutput.getStdErrorString();
-
- if (stdOutputString == null && "".equals(stdOutputString) ||
- ((stdErrorString != null) && !("".equals(stdErrorString)))) {
- log.error("Standard Error output : " + stdErrorString);
- throw new SSHApiException(errorMsg + stdErrorString);
- }
- return stdOutputString;
- }
-
- public String submitBatchJob(JobDescriptor jobDescriptor) throws SSHApiException {
- TransformerFactory factory = TransformerFactory.newInstance();
- URL resource = this.getClass().getClassLoader().getResource(PBSTEMPLATE_XSLT);
-
- if (resource == null) {
- String error = "System configuration file '" + PBSCluster.PBSTEMPLATE_XSLT
- + "' not found in the classpath";
- throw new SSHApiException(error);
- }
-
- Source xslt = new StreamSource(new File(resource.getPath()));
- Transformer transformer;
- StringWriter results = new StringWriter();
- File tempPBSFile = null;
- try {
- // generate the pbs script using xslt
- transformer = factory.newTransformer(xslt);
- Source text = new StreamSource(new ByteArrayInputStream(jobDescriptor.toXML().getBytes()));
- transformer.transform(text, new StreamResult(results));
-
- System.out.println(results.toString());
-// log.debug("generated PBS:" + results.toString());
-
- // creating a temporary file using pbs script generated above
- int number = new SecureRandom().nextInt();
- number = (number < 0 ? -number : number);
-
- tempPBSFile = new File(Integer.toString(number) + ".pbs");
- FileUtils.writeStringToFile(tempPBSFile, results.toString());
-
- //reusing submitBatchJobWithPBS method to submit a job
-
- String jobID = this.submitBatchJobWithPBS(tempPBSFile.getAbsolutePath(),
- jobDescriptor.getWorkingDirectory());
- log.debug("Job has successfully submitted, JobID : " + jobID);
- return jobID.replace("\n", "");
- } catch (TransformerConfigurationException e) {
- throw new SSHApiException("Error parsing PBS transformation", e);
- } catch (TransformerException e) {
- throw new SSHApiException("Error generating PBS script", e);
- } catch (IOException e) {
- throw new SSHApiException("An exception occurred while connecting to server." +
- "Connecting server - " + serverInfo.getHost() + ":" + serverInfo.getPort() +
- " connecting user name - "
- + serverInfo.getUserName(), e);
- } finally {
- if (tempPBSFile != null) {
- tempPBSFile.delete();
- }
- }
- }
-
-
- public Cluster loadCluster() throws SSHApiException {
- RawCommandInfo rawCommandInfo = new RawCommandInfo(this.installedPath + "qnodes");
-
- StandardOutReader stdOutReader = new StandardOutReader();
- CommandExecutor.executeCommand(rawCommandInfo, this.getSession(), stdOutReader);
-
- String result = getOutputifAvailable(stdOutReader, "Error getting cluster information !");
- String[] Nodes = result.split("\n");
- String[] line;
- String header, value;
- Node Node;
- Core[] Cores = null;
- ArrayList<Node> Machines = new ArrayList<Node>();
- int i = 0;
- while (i < Nodes.length) {
- Node = new Node();
- Node.setName(Nodes[i]);
- i++;
-
- while (i < Nodes.length) {
- if (!Nodes[i].startsWith(" ")) {
- i++;
- break;
- }
-
- line = Nodes[i].split("=");
- header = line[0].trim();
- value = line[1].trim();
-
- if ("state".equals(header))
- Node.setState(value);
- else if ("np".equals(header)) {
- Node.setNp(value);
- int np = Integer.parseInt(Node.getNp());
- Cores = new Core[np];
- for (int n = 0; n < np; n++) {
- Cores[n] = new Core("" + n);
- }
- } else if ("ntype".equals(header))
- Node.setNtype(value);
- else if ("jobs".equals(header)) {
- String[] jobs = value.split(", ");
- JobDescriptor jo;
- //Job[] Jobs = new Job[jobs.length];
- for (String job : jobs) {
- String[] c = job.split("/");
- String Jid = c[1];
- jo = this.getJobDescriptorById(Jid);
- int core = Integer.parseInt(c[0]);
- assert Cores != null;
- Cores[core].setJob(jo);
-
- }
-
-
- }
- i++;
- }
- Node.setCores(Cores);
- Machines.add(Node);
- }
- this.setNodes(Machines.toArray(new Node[Machines.size()]));
- return this;
- }
-
-
- public JobDescriptor getJobDescriptorById(String jobID) throws SSHApiException {
- RawCommandInfo rawCommandInfo = new RawCommandInfo(this.installedPath + "qstat -f " + jobID);
-
- StandardOutReader stdOutReader = new StandardOutReader();
- CommandExecutor.executeCommand(rawCommandInfo, this.getSession(), stdOutReader);
-
- String result = getOutputifAvailable(stdOutReader, "Error getting job information from the resource !");;
- String[] info = result.split("\n");
- JobDescriptor jobDescriptor = new JobDescriptor();
- String[] line;
- for (int i = 0; i < info.length; i++) {
- if (info[i].contains("=")) {
- line = info[i].split("=", 2);
- } else {
- line = info[i].split(":", 2);
- }
- if (line.length >= 2) {
- String header = line[0].trim();
- log.debug("Header = " + header);
- String value = line[1].trim();
- log.debug("value = " + value);
-
- if (header.equals("Variable_List")) {
- while (info[i + 1].startsWith("\t")) {
- value += info[i + 1];
- i++;
- }
- value = value.replaceAll("\t", "");
- jobDescriptor.setVariableList(value);
- } else if ("Job Id".equals(header)) {
- jobDescriptor.setJobID(value);
- } else if ("Job_Name".equals(header)) {
- jobDescriptor.setJobName(value);
- } else if ("Account_Name".equals(header)) {
- jobDescriptor.setAcountString(value);
- } else if ("job_state".equals(header)) {
- jobDescriptor.setStatus(value);
- } else if ("Job_Owner".equals(header)) {
- jobDescriptor.setOwner(value);
- } else if ("resources_used.cput".equals(header)) {
- jobDescriptor.setUsedCPUTime(value);
- } else if ("resources_used.mem".equals(header)) {
- jobDescriptor.setUsedMemory(value);
- } else if ("resources_used.walltime".equals(header)) {
- jobDescriptor.setEllapsedTime(value);
- } else if ("job_state".equals(header)) {
- jobDescriptor.setStatus(value);
- } else if ("queue".equals(header))
- jobDescriptor.setQueueName(value);
- else if ("ctime".equals(header)) {
- jobDescriptor.setCTime(value);
- } else if ("qtime".equals(header)) {
- jobDescriptor.setQTime(value);
- } else if ("mtime".equals(header)) {
- jobDescriptor.setMTime(value);
- } else if ("start_time".equals(header)) {
- jobDescriptor.setSTime(value);
- } else if ("comp_time".equals(header)) {
- jobDescriptor.setCompTime(value);
- } else if ("exec_host".equals(header)) {
- jobDescriptor.setExecuteNode(value);
- } else if ("Output_Path".equals(header)) {
- if (info[i + 1].contains("=") || info[i + 1].contains(":"))
- jobDescriptor.setStandardOutFile(value);
- else {
- jobDescriptor.setStandardOutFile(value + info[i + 1].trim());
- i++;
- }
- } else if ("Error_Path".equals(header)) {
- if (info[i + 1].contains("=") || info[i + 1].contains(":"))
- jobDescriptor.setStandardErrorFile(value);
- else {
- String st = info[i + 1].trim();
- jobDescriptor.setStandardErrorFile(value + st);
- i++;
- }
-
- } else if ("submit_args".equals(header)) {
- while (i + 1 < info.length) {
- if (info[i + 1].startsWith("\t")) {
- value += info[i + 1];
- i++;
- } else
- break;
- }
- value = value.replaceAll("\t", "");
- jobDescriptor.setSubmitArgs(value);
- }
- }
- }
- return jobDescriptor;
- }
-
- public void scpTo(String remoteFile, String localFile) throws SSHApiException {
- try {
- SSHUtils.scpTo(remoteFile, localFile, session);
- } catch (IOException e) {
- throw new SSHApiException("Failed during scping local file:" + localFile + " to remote file "
- + serverInfo.getHost() + ":rFile", e);
- } catch (JSchException e) {
- throw new SSHApiException("Failed during scping local file:" + localFile + " to remote file "
- + serverInfo.getHost() + ":rFile", e);
- }
- }
-
- public void scpFrom(String remoteFile, String localFile) throws SSHApiException {
- try {
- SSHUtils.scpFrom(remoteFile, localFile, session);
- } catch (IOException e) {
- throw new SSHApiException("Failed during scping local file:" + localFile + " to remote file "
- + serverInfo.getHost() + ":rFile", e);
- } catch (JSchException e) {
- throw new SSHApiException("Failed during scping local file:" + localFile + " to remote file "
- + serverInfo.getHost() + ":rFile", e);
- }
- }
-
- public void makeDirectory(String directoryPath) throws SSHApiException {
- try {
- SSHUtils.makeDirectory(directoryPath, session);
- } catch (IOException e) {
- throw new SSHApiException("Failed during creating directory:" + directoryPath + " to remote file "
- + serverInfo.getHost() + ":rFile", e);
- } catch (JSchException e) {
- throw new SSHApiException("Failed during creating directory :" + directoryPath + " to remote file "
- + serverInfo.getHost() + ":rFile", e);
- }
- }
-
-//
-// public String submitAsyncJob(Job jobDescriptor, JobSubmissionListener listener) throws SSHApiException {
-//// final Cluster cluster = this;
-// String jobID = this.submitBatchJob(jobDescriptor);
-//// final JobSubmissionListener jobSubmissionListener = listener;
-// try {
-// // Wait 5 seconds to start the first poll, this is hard coded, user doesn't have
-// // to configure this.
-// Thread.sleep(5000);
-// } catch (InterruptedException e) {
-// log.error("Error during job status monitoring");
-// throw new SSHApiException("Error during job status monitoring", e);
-// }
-// // Get the job status first
-// try {
-////
-//// Thread t = new Thread() {
-//// @Override
-//// public void run() {
-//// try {
-// Job jobById = this.getJobDescriptorById(jobID);
-// while (true) {
-// while (!jobById.getStatus().equals(JobStatus.C.toString())) {
-// if (!jobById.getStatus().equals(listener.getJobStatus().toString())) {
-// listener.setJobStatus(JobStatus.fromString(jobById.getStatus()));
-// listener.statusChanged(jobById);
-// }
-// Thread.sleep(Long.parseLong(configReader.getConfiguration(POLLING_FREQUENCEY)));
-//
-// jobById = this.getJobDescriptorById(jobID);
-// }
-// //Set the job status to Complete
-// listener.setJobStatus(JobStatus.C);
-// listener.statusChanged(jobById);
-// break;
-// }
-//// } catch (InterruptedException e) {
-//// log.error("Error listening to the submitted job", e);
-//// } catch (SSHApiException e) {
-//// log.error("Error listening to the submitted job", e);
-//// }
-//// }
-//// };
-// // This thread runs until the program termination, so that use can provide
-// // any action in onChange method of the listener, without worrying for waiting in the caller thread.
-// //t.setDaemon(true);
-//// t.start();
-// } catch (Exception e) {
-// log.error("Error during job status monitoring");
-// throw new SSHApiException("Error during job status monitoring", e);
-// }
-// return jobID; //To change body of implemented methods use File | Settings | File Templates.
-// }
-
- public JobStatus getJobStatus(String jobID) throws SSHApiException {
- RawCommandInfo rawCommandInfo = new RawCommandInfo(this.installedPath + "qstat -f " + jobID);
-
- StandardOutReader stdOutReader = new StandardOutReader();
- CommandExecutor.executeCommand(rawCommandInfo, this.getSession(), stdOutReader);
-
-
- String result = getOutputifAvailable(stdOutReader,"Error getting job status with job ID: " + jobID);
- String[] info = result.split("\n");
- String[] line = null;
- for (String anInfo : info) {
- if (anInfo.contains("=")) {
- line = anInfo.split("=", 2);
- if (line.length != 0) {
- if (line[0].contains("job_state")) {
- return JobStatus.valueOf(line[1].replaceAll(" ", ""));
- }
- }
- }
- }
- return null;
- }
-
- public JobDescriptor cancelJob(String jobID) throws SSHApiException {
- RawCommandInfo rawCommandInfo = new RawCommandInfo(this.installedPath + "qdel " + jobID);
-
- StandardOutReader stdOutReader = new StandardOutReader();
- CommandExecutor.executeCommand(rawCommandInfo, this.getSession(), stdOutReader);
-
- getOutputifAvailable(stdOutReader, "Error cancelling job with Job ID : " + jobID);
-
- JobDescriptor jobById = this.getJobDescriptorById(jobID);
- if (CommonUtils.isJobFinished(jobById)) {
- log.debug("Job Cancel operation was successful !");
- return jobById;
- } else {
- log.debug("Job Cancel operation was not successful !");
- return null;
- }
- }
-
- public void setServerInfo(ServerInfo serverInfo) {
- this.serverInfo = serverInfo;
- }
-
- public void setAuthenticationInfo(AuthenticationInfo authenticationInfo) {
- this.authenticationInfo = authenticationInfo;
- }
-
- public void setSession(Session session) {
- this.session = session;
- }
-
- /**
- * @return cluster Nodes as array of machines
- */
- public Node[] getNodes() {
- return Nodes;
- }
-
- public void setNodes(Node[] Nodes) {
- this.Nodes = Nodes;
- }
-
- public ServerInfo getServerInfo() {
- return serverInfo;
- }
-
- public AuthenticationInfo getAuthenticationInfo() {
- return authenticationInfo;
- }
-
- /**
- * This gaurantee to return a valid session
- *
- * @return
- */
- public Session getSession() {
- return this.session;
- }
- private static void logDebug(String message) {
- if (log.isDebugEnabled()) {
- log.debug(message);
- }
+ public PBSCluster(ServerInfo serverInfo, AuthenticationInfo authenticationInfo, JobManagerConfiguration config) throws SSHApiException {
+ super(serverInfo, authenticationInfo,config);
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/CommonUtils.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/CommonUtils.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/CommonUtils.java
index c6a17bc..351c460 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/CommonUtils.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/CommonUtils.java
@@ -20,13 +20,15 @@
*/
package org.apache.airavata.gsi.ssh.util;
-import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
+import org.apache.airavata.gsi.ssh.api.job.*;
import org.apache.airavata.gsi.ssh.impl.JobStatus;
+import org.apache.airavata.gsi.ssh.impl.RawCommandInfo;
public class CommonUtils {
/**
* This returns true if the give job is finished
* otherwise false
+ *
* @param job
* @return
*/
@@ -40,6 +42,7 @@ public class CommonUtils {
/**
* This will read
+ *
* @param maxWalltime
* @return
*/
@@ -52,4 +55,11 @@ public class CommonUtils {
return hours + ":" + minutes + ":00";
}
}
+ public static JobManagerConfiguration getPBSJobManager(String installedPath) {
+ return new PBSJobConfiguration("PBSTemplate.xslt",".pbs", installedPath, new PBSOutputParser());
+ }
+
+ public static JobManagerConfiguration getSLURMJobManager(String installedPath) {
+ return new SlurmJobConfiguration("SLURMTemplate.xslt", ".slurm", installedPath, new SlurmOutputParser());
+ }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/tools/gsissh/src/main/resources/SLURMTemplate.xslt
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/resources/SLURMTemplate.xslt b/tools/gsissh/src/main/resources/SLURMTemplate.xslt
new file mode 100644
index 0000000..169dbbf
--- /dev/null
+++ b/tools/gsissh/src/main/resources/SLURMTemplate.xslt
@@ -0,0 +1,69 @@
+<!--Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under
+ the Apache License, Version 2.0 (theĆ "License"); you may not use this file except in compliance with the License. You may
+ obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to
+ in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+ ANY ~ KIND, either express or implied. See the License for the specific language governing permissions and limitations under
+ the License. -->
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:ns="http://airavata.apache.org/gsi/ssh/2012/12">
+<xsl:output method="text" />
+<xsl:template match="/ns:JobDescriptor">
+<xsl:choose>
+<xsl:when test="ns:shellName">
+#!<xsl:value-of select="ns:shellName"/>
+ </xsl:when>
+ </xsl:choose>
+<xsl:choose>
+ <xsl:when test="ns:queueName">
+#SBATCH -p <xsl:value-of select="ns:queueName"/>
+ </xsl:when>
+ </xsl:choose>
+<xsl:choose>
+ <xsl:when test="ns:nodes">
+#SBATCH -n <xsl:value-of select="ns:nodes"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="ns:mailAddress">
+#SBATCH -mail-user=<xsl:value-of select="ns:mailAddress"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="ns:mailType">
+#SBATCH -mail-type=<xsl:value-of select="ns:mailType"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+<xsl:when test="ns:acountString">
+#SBATCH -A <xsl:value-of select="ns:acountString"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="ns:maxWallTime">
+#SBATCH -t <xsl:value-of select="ns:maxWallTime"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="ns:jobName">
+#SBATCH -J <xsl:value-of select="ns:jobName"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="ns:standardOutFile">
+#SBATCH -o <xsl:value-of select="ns:standardOutFile"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="ns:standardOutFile">
+#SBATCH -e <xsl:value-of select="ns:standardErrorFile"/>
+ </xsl:when>
+ </xsl:choose>
+cd <xsl:text> </xsl:text><xsl:value-of select="ns:workingDirectory"/><xsl:text>
</xsl:text>
+ <xsl:choose><xsl:when test="ns:jobSubmitterCommand">
+<xsl:value-of select="ns:jobSubmitterCommand"/><xsl:text> </xsl:text></xsl:when></xsl:choose><xsl:value-of select="ns:executablePath"/><xsl:text> </xsl:text>
+<xsl:for-each select="ns:inputs/ns:input">
+ <xsl:value-of select="."/><xsl:text> </xsl:text>
+ </xsl:for-each>
+</xsl:template>
+
+</xsl:stylesheet>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/tools/gsissh/src/main/resources/schemas/PBSJobDescriptor.xsd
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/resources/schemas/PBSJobDescriptor.xsd b/tools/gsissh/src/main/resources/schemas/PBSJobDescriptor.xsd
index db65e4f..0e84612 100644
--- a/tools/gsissh/src/main/resources/schemas/PBSJobDescriptor.xsd
+++ b/tools/gsissh/src/main/resources/schemas/PBSJobDescriptor.xsd
@@ -15,11 +15,15 @@
<complexType name="pbsParams">
<sequence>
<element name="jobID" type="xsd:string" minOccurs="0" maxOccurs="1"/>
+ <element name="userName" type="xsd:string" minOccurs="0" maxOccurs="1"/>
<element name="shellName" type="xsd:string" minOccurs="0" maxOccurs="1" default="/bin/bash"/>
<element name="queueName" type="xsd:string" minOccurs="0" maxOccurs="1" default="normal"/>
<element name="jobName" type="xsd:string" minOccurs="0" maxOccurs="1"/>
<element name="allEnvExport" type="xsd:boolean" minOccurs="0 " maxOccurs="1" default="true"/>
<element name="mailOptions" type="xsd:string" minOccurs="0" maxOccurs="1" />
+ <element name="mailAddress" type="xsd:string" minOccurs="0" maxOccurs="1" />
+ <element name="partition" type="xsd:string" minOccurs="0" maxOccurs="1" />
+ <element name="mailType" type="xsd:string" minOccurs="0" maxOccurs="1" />
<element name="acountString" type="xsd:string" minOccurs="0" maxOccurs="1" />
<element name="maxWallTime" type="xsd:string" minOccurs="0" maxOccurs="1" default="1:00:00"/>
<element name="standardOutFile" type="xsd:string" minOccurs="0" maxOccurs="1" />
@@ -28,6 +32,7 @@
<element name="inputDirectory" type="xsd:string" minOccurs="0" maxOccurs="1" />
<element name="nodes" type="xsd:int" minOccurs="0" maxOccurs="1" default="1"/>
<element name="processesPerNode" type="xsd:int" minOccurs="0" maxOccurs="1" default="1" />
+ <element name="nodeList" type="xsd:string" minOccurs="0" maxOccurs="1" default="1" />
<element name="workingDirectory" type="xsd:string" minOccurs="0" maxOccurs="1" />
<element name="executablePath" type="xsd:string" minOccurs="0" maxOccurs="1" />
<element name="inputs" type="gsissh:inputList" minOccurs="1" maxOccurs="1"/>
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java b/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java
index c2bc055..c339d9d 100644
--- a/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java
+++ b/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java
@@ -45,17 +45,19 @@ public class DefaultSSHApiTest {
private String certificateLocation;
private String pbsFilePath;
private String workingDirectory;
+ private String slurmWorkingDirectory;
private String jobID;
@BeforeTest
public void setUp() throws Exception {
-// System.setProperty("myproxy.user", "ogce");
-// System.setProperty("myproxy.password", "");
-// System.setProperty("basedir", "/Users/lahirugunathilake/work/airavata/sandbox/gsissh");
-// System.setProperty("gsi.working.directory","/home/ogce");
+ System.setProperty("myproxy.user", "ogce");
+ System.setProperty("myproxy.password", "0Gce3098");
+ System.setProperty("basedir", "/Users/lahirugunathilake/Downloads");
+ System.setProperty("gsi.working.directory", "/home/ogce");
myProxyUserName = System.getProperty("myproxy.user");
myProxyPassword = System.getProperty("myproxy.password");
workingDirectory = System.getProperty("gsi.working.directory");
+ slurmWorkingDirectory = "/home1/01437/ogce";
String pomDirectory = System.getProperty("basedir");
File pomFileDirectory = new File(pomDirectory);
@@ -96,10 +98,8 @@ public class DefaultSSHApiTest {
}
-
-
@Test
- public void testSubmitAsyncJob() throws Exception {
+ public void testPBSAsync() throws Exception {
// Create authentication
GSIAuthenticationInfo authenticationInfo
= new MyProxyAuthenticationInfo(myProxyUserName, myProxyPassword, "myproxy.teragrid.org",
@@ -109,7 +109,7 @@ public class DefaultSSHApiTest {
ServerInfo serverInfo = new ServerInfo("ogce", "trestles.sdsc.edu");
- Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, "/opt/torque/bin/");
+ Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, CommonUtils.getPBSJobManager("/opt/torque/bin/"));
// Execute command
@@ -162,6 +162,72 @@ public class DefaultSSHApiTest {
System.out.println(jobById.getVariableList());
}
+ @Test
+ public void testSLURMAsync() throws Exception {
+ // Create authentication
+ GSIAuthenticationInfo authenticationInfo
+ = new MyProxyAuthenticationInfo(myProxyUserName, myProxyPassword, "myproxy.teragrid.org",
+ 7512, 17280000, certificateLocation);
+
+ // Server info
+ ServerInfo serverInfo = new ServerInfo("ogce", "stampede.tacc.xsede.org");
+ serverInfo.setPort(2222);
+
+ Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, CommonUtils.getSLURMJobManager("/usr/bin/"));
+
+
+ // Execute command
+ System.out.println("Target SLURM file path: " + slurmWorkingDirectory);
+ // constructing the job object
+ JobDescriptor jobDescriptor = new JobDescriptor();
+ jobDescriptor.setWorkingDirectory(slurmWorkingDirectory);
+ jobDescriptor.setShellName("/bin/sh");
+ jobDescriptor.setJobName("GSI_SSH_SLEEP_JOB");
+ jobDescriptor.setExecutablePath("/bin/echo");
+ jobDescriptor.setAllEnvExport(true);
+ jobDescriptor.setMailOptions("n");
+ jobDescriptor.setStandardOutFile("" + File.separator + "application.out");
+ jobDescriptor.setStandardErrorFile("/home1/01437/ogce" + File.separator + "application.err");
+ jobDescriptor.setNodes(1);
+ jobDescriptor.setProcessesPerNode(1);
+ jobDescriptor.setQueueName("normal");
+ jobDescriptor.setMaxWallTime("60");
+ jobDescriptor.setAcountString("TG-STA110014S");
+ jobDescriptor.setJobSubmitter("sbatch");
+ List<String> inputs = new ArrayList<String>();
+ inputs.add("Hello World");
+ jobDescriptor.setInputValues(inputs);
+ //finished construction of job object
+ System.out.println(jobDescriptor.toXML());
+ jobID = pbsCluster.submitBatchJob(jobDescriptor);
+ System.out.println("JobID returned : " + jobID);
+
+// Cluster cluster = sshApi.getCluster(serverInfo, authenticationInfo);
+// Thread.sleep(1000);
+
+ JobDescriptor jobById = pbsCluster.getJobDescriptorById(jobID);
+
+ //printing job data got from previous call
+ AssertJUnit.assertEquals(jobById.getJobId(), jobID);
+ System.out.println(jobById.getAcountString());
+ System.out.println(jobById.getAllEnvExport());
+ System.out.println(jobById.getCompTime());
+ System.out.println(jobById.getExecutablePath());
+ System.out.println(jobById.getEllapsedTime());
+ System.out.println(jobById.getQueueName());
+ System.out.println(jobById.getExecuteNode());
+ System.out.println(jobById.getJobName());
+ System.out.println(jobById.getCTime());
+ System.out.println(jobById.getSTime());
+ System.out.println(jobById.getMTime());
+ System.out.println(jobById.getCompTime());
+ System.out.println(jobById.getOwner());
+ System.out.println(jobById.getQTime());
+ System.out.println(jobById.getUsedCPUTime());
+ System.out.println(jobById.getUsedMemory());
+ System.out.println(jobById.getVariableList());
+ }
+
@Test
public void testGetCluster() throws Exception {
@@ -192,7 +258,7 @@ public class DefaultSSHApiTest {
// Server info
ServerInfo serverInfo = new ServerInfo("ogce", "trestles.sdsc.edu");
- Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, "/opt/torque/bin/");
+ Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, CommonUtils.getPBSJobManager("/opt/torque/bin/"));
// Execute command
System.out.println("Target PBS file path: " + workingDirectory);
@@ -235,7 +301,7 @@ public class DefaultSSHApiTest {
ServerInfo serverInfo = new ServerInfo("ogce", "trestles.sdsc.edu");
- Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, "/opt/torque/bin/");
+ Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, CommonUtils.getPBSJobManager("/opt/torque/bin/"));
// Execute command
@@ -325,7 +391,7 @@ public class DefaultSSHApiTest {
ServerInfo serverInfo = new ServerInfo("ogce", "trestles.sdsc.edu");
- Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, "/opt/torque/bin/");
+ Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, CommonUtils.getPBSJobManager("/opt/torque/bin/"));
// Execute command
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/VanilaSSHTest.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/VanilaSSHTest.java b/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/VanilaSSHTest.java
index 3e46aa4..0f4a205 100644
--- a/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/VanilaSSHTest.java
+++ b/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/VanilaSSHTest.java
@@ -27,6 +27,7 @@ import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
import org.apache.airavata.gsi.ssh.config.ConfigReader;
import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPasswordAuthenticationInfo;
import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPublicKeyFileAuthentication;
+import org.apache.airavata.gsi.ssh.util.CommonUtils;
import org.testng.AssertJUnit;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
@@ -114,7 +115,7 @@ public class VanilaSSHTest {
}
// Server info
ServerInfo serverInfo = new ServerInfo(this.userName, this.hostName);
- Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, "/opt/torque/torque-4.2.3.1/bin/");
+ Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, CommonUtils.getPBSJobManager("/opt/torque/torque-4.2.3.1/bin/"));
String date = new Date().toString();
date = date.replaceAll(" ", "_");
@@ -197,7 +198,7 @@ public class VanilaSSHTest {
}
// Server info
ServerInfo serverInfo = new ServerInfo(this.userName, this.hostName);
- Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, "/opt/torque/torque-4.2.3.1/bin/");
+ Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, CommonUtils.getPBSJobManager("/opt/torque/torque-4.2.3.1/bin/"));
String date = new Date().toString();
date = date.replaceAll(" ", "_");
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/tools/gsissh/src/test/resources/gsissh.properties
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/test/resources/gsissh.properties b/tools/gsissh/src/test/resources/gsissh.properties
new file mode 100644
index 0000000..3fdf76d
--- /dev/null
+++ b/tools/gsissh/src/test/resources/gsissh.properties
@@ -0,0 +1,26 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+###########################################################################
+# Specifies system level configurations as a key/value pairs.
+###########################################################################
+
+StrictHostKeyChecking=no
+ssh.session.timeout=360000
[3/3] git commit: merging changes
Posted by la...@apache.org.
merging changes
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/532f2691
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/532f2691
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/532f2691
Branch: refs/heads/master
Commit: 532f2691e55c800947ef75ef9e30285c51b919f1
Parents: b358106 f986e09
Author: lahiru <la...@apache.org>
Authored: Thu Mar 6 18:14:25 2014 -0500
Committer: lahiru <la...@apache.org>
Committed: Thu Mar 6 18:14:25 2014 -0500
----------------------------------------------------------------------
.../airavata/api/server/AiravataAPIServer.java | 18 +-
.../server/handler/AiravataServerHandler.java | 2 +
.../api/server/util/RegistryInitUtil.java | 2 +-
.../main/resources/airavata-server.properties | 2 +-
.../src/main/resources/registry-mysql.sql | 2 +-
.../client/samples/CreateLaunchExperiment.java | 215 +++++++++++
modules/commons/pom.xml | 2 +-
.../common/utils/ApplicationSettings.java | 4 +
modules/distribution/airavata-client/pom.xml | 4 +-
.../src/main/assembly/bin-assembly.xml | 5 +-
modules/distribution/airavata-server/pom.xml | 21 +-
.../src/main/assembly/bin-assembly.xml | 11 +-
.../resources/conf/airavata-server.properties | 2 +-
modules/distribution/pom.xml | 4 +-
modules/distribution/release/pom.xml | 18 +-
modules/distribution/xbaya-gui/pom.xml | 5 -
.../src/main/assembly/bin-assembly.xml | 2 -
.../gfac/provider/impl/LocalProvider.java | 9 +-
modules/integration-tests/pom.xml | 64 +---
.../registry/jpa/impl/ExperimentRegistry.java | 99 ++++-
.../jpa/resources/TaskDetailResource.java | 2 +-
.../registry/jpa/resources/Utils.java | 4 +-
.../main/resources/airavata-registry-derby.sql | 325 ----------------
.../main/resources/airavata-registry-mysql.sql | 327 ----------------
.../src/main/resources/provenance.sql | 73 ----
.../src/main/resources/registry-mysql.sql | 2 +-
.../test/resources/airavata-registry-derby.sql | 376 -------------------
modules/workflow-model/pom.xml | 4 +-
modules/xbaya-gui/pom.xml | 13 +-
pom.xml | 4 +-
.../airavata-client/create-application/pom.xml | 23 --
samples/airavata-client/workflow-run/pom.xml | 23 --
samples/gateway-developer-guide/pom.xml | 11 -
test.git | 1 -
34 files changed, 374 insertions(+), 1305 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/532f2691/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
----------------------------------------------------------------------
diff --cc airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
index 0000000,9a61243..63184d3
mode 000000,100644..100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
@@@ -1,0 -1,257 +1,215 @@@
+ /*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+ package org.apache.airavata.client.samples;
+
+ import org.apache.airavata.api.error.ExperimentNotFoundException;
+ import org.apache.airavata.common.exception.ApplicationSettingsException;
+ import org.apache.airavata.common.utils.ClientSettings;
+ import org.apache.airavata.model.workspace.experiment.*;
+ import org.apache.airavata.schemas.gfac.DataType;
+ import org.apache.airavata.api.Airavata;
+ import org.apache.airavata.api.client.AiravataClientFactory;
+ import org.apache.airavata.api.error.AiravataClientException;
+ import org.apache.airavata.api.error.AiravataSystemException;
+ import org.apache.airavata.api.error.InvalidRequestException;
+ import org.apache.airavata.client.AiravataAPIFactory;
+ import org.apache.airavata.client.api.AiravataAPI;
+ import org.apache.airavata.client.api.exception.AiravataAPIInvocationException;
+ import org.apache.airavata.client.tools.DocumentCreator;
+ import org.apache.airavata.common.utils.AiravataUtils;
+ import org.apache.airavata.model.util.ExperimentModelUtil;
+ import org.apache.thrift.TException;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ import java.util.ArrayList;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Set;
+
+ public class CreateLaunchExperiment {
+
+ //FIXME: Read from a config file
+ public static final String THRIFT_SERVER_HOST = "localhost";
+ public static final int THRIFT_SERVER_PORT = 8930;
+ private final static Logger logger = LoggerFactory.getLogger(CreateLaunchExperiment.class);
+ private static final String DEFAULT_USER = "defauly.registry.user";
+ private static final String DEFAULT_GATEWAY = "default.registry.gateway";
+
+ public static void main(String[] args) {
+ try {
+ AiravataUtils.setExecutionAsClient();
+ final Airavata.Client airavata = AiravataClientFactory.createAiravataClient(THRIFT_SERVER_HOST, THRIFT_SERVER_PORT);
+ System.out.println("API version is " + airavata.GetAPIVersion());
+ addDescriptors();
+ final String expId = createExperiment(airavata);
+ System.out.println("Experiment ID : " + expId);
+ launchExperiment(airavata, expId);
+ System.out.println("Launched successfully");
+ // try {
+ // Thread.sleep(20000);
+ // } catch (InterruptedException e) {
+ // e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ // }
+ Thread monitor = (new Thread(){
+ public void run() {
+ Map<String, JobStatus> jobStatuses = null;
+ while (true) {
+ try {
+ jobStatuses = airavata.getJobStatuses(expId);
+ Set<String> strings = jobStatuses.keySet();
+ for (String key : strings) {
+ JobStatus jobStatus = jobStatuses.get(key);
+ if(jobStatus == null){
+ return;
+ }else {
+ if (JobState.COMPLETE.equals(jobStatus.getJobState())) {
+ System.out.println("Job completed Job ID: " + key);
+ return;
+ }else{
+ System.out.println("Job ID:" + key + jobStatuses.get(key).getJobState().toString());
+ }
+ }
+ }
+ Thread.sleep(5000);
+ } catch (Exception e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+ }
+ }
+ });
+ // monitor.start();
+ try {
+ monitor.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+
+ // Experiment experiment = airavata.getExperiment(expId);
+ // System.out.println("retrieved exp id : " + experiment.getExperimentID());
+ } catch (TException e) {
+ logger.error("Error while connecting with server", e.getMessage());
+ e.printStackTrace();
+ } catch (ApplicationSettingsException e) {
+ logger.error("Error while creating airavata API object", e.getMessage());
+ e.printStackTrace();
+ } catch (AiravataAPIInvocationException e) {
+ logger.error("Error while creating airavata API object", e.getMessage());
+ e.printStackTrace();
+ }
+ }
+
+ public static void addDescriptors() throws AiravataAPIInvocationException,ApplicationSettingsException {
+ try {
+ DocumentCreator documentCreator = new DocumentCreator(getAiravataAPI());
- documentCreator.createEchoAppLocalHostDocs();
- documentCreator.createEchoAppGramDocs();
- documentCreator.createEchoAppGSISSHDocs();
++ documentCreator.createLocalHostDocs();
++ documentCreator.createGramDocs();
++ documentCreator.createGSISSHDocs();
+ } catch (AiravataAPIInvocationException e) {
+ logger.error("Unable to create airavata API", e.getMessage());
+ throw new AiravataAPIInvocationException(e);
+ } catch (ApplicationSettingsException e) {
+ logger.error("Unable to create airavata API", e.getMessage());
+ throw new ApplicationSettingsException(e.getMessage());
+ }
+ }
+
+ private static AiravataAPI getAiravataAPI() throws AiravataAPIInvocationException, ApplicationSettingsException {
+ AiravataAPI airavataAPI;
+ try {
+ String sysUser = ClientSettings.getSetting(DEFAULT_USER);
+ String gateway = ClientSettings.getSetting(DEFAULT_GATEWAY);
+ airavataAPI = AiravataAPIFactory.getAPI(gateway, sysUser);
+ } catch (AiravataAPIInvocationException e) {
+ logger.error("Unable to create airavata API", e.getMessage());
+ throw new AiravataAPIInvocationException(e);
+ } catch (ApplicationSettingsException e) {
+ logger.error("Unable to create airavata API", e.getMessage());
+ throw new ApplicationSettingsException(e.getMessage());
+ }
+ return airavataAPI;
+ }
+
+ public static String createExperiment (Airavata.Client client) throws AiravataSystemException, InvalidRequestException, AiravataClientException, TException {
+ try{
+ List<DataObjectType> exInputs = new ArrayList<DataObjectType>();
+ DataObjectType input = new DataObjectType();
+ input.setKey("echo_input");
+ input.setType(DataType.STRING.toString());
+ input.setValue("echo_output=Hello World");
+ exInputs.add(input);
+
+ List<DataObjectType> exOut = new ArrayList<DataObjectType>();
+ DataObjectType output = new DataObjectType();
+ output.setKey("echo_output");
+ output.setType(DataType.STRING.toString());
+ output.setValue("");
+ exOut.add(output);
+
+ Experiment simpleExperiment = ExperimentModelUtil.createSimpleExperiment("project1", "admin", "echoExperiment", "SimpleEcho2", "SimpleEcho2", exInputs);
+ simpleExperiment.setExperimentOutputs(exOut);
+
+ ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling("trestles.sdsc.edu", 1, 1, 1, "normal", 0, 0, 1, "sds128");
+ scheduling.setResourceHostId("gsissh-trestles");
+ UserConfigurationData userConfigurationData = new UserConfigurationData();
+ userConfigurationData.setAiravataAutoSchedule(false);
+ userConfigurationData.setOverrideManualScheduledParams(false);
+ userConfigurationData.setComputationalResourceScheduling(scheduling);
+ simpleExperiment.setUserConfigurationData(userConfigurationData);
- return client.createExperiment(simpleExperiment);
- } catch (AiravataSystemException e) {
- logger.error("Error occured while creating the experiment...", e.getMessage());
- throw new AiravataSystemException(e);
- } catch (InvalidRequestException e) {
- logger.error("Error occured while creating the experiment...", e.getMessage());
- throw new InvalidRequestException(e);
- } catch (AiravataClientException e) {
- logger.error("Error occured while creating the experiment...", e.getMessage());
- throw new AiravataClientException(e);
- }catch (TException e) {
- logger.error("Error occured while creating the experiment...", e.getMessage());
- throw new TException(e);
- }
- }
-
- public static String createLocalExperiment (Airavata.Client client) throws AiravataSystemException, InvalidRequestException, AiravataClientException, TException {
- try{
- List<DataObjectType> exInputs = new ArrayList<DataObjectType>();
- DataObjectType input = new DataObjectType();
- input.setKey("echo_input");
- input.setType(DataType.STRING.toString());
- input.setValue("echo_output=Hello World");
- exInputs.add(input);
-
- List<DataObjectType> exOut = new ArrayList<DataObjectType>();
- DataObjectType output = new DataObjectType();
- output.setKey("echo_output");
- output.setType(DataType.STRING.toString());
- output.setValue("");
- exOut.add(output);
-
- Experiment simpleExperiment = ExperimentModelUtil.createSimpleExperiment("project1", "admin", "echoExperiment", "Echo", "Echo", exInputs);
- simpleExperiment.setExperimentOutputs(exOut);
-
- ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling("localhost", 1, 1, 1, "normal", 0, 0, 1, "sds128");
- scheduling.setResourceHostId("localhost");
- UserConfigurationData userConfigurationData = new UserConfigurationData();
- userConfigurationData.setAiravataAutoSchedule(false);
- userConfigurationData.setOverrideManualScheduledParams(false);
- userConfigurationData.setComputationalResourceScheduling(scheduling);
- simpleExperiment.setUserConfigurationData(userConfigurationData);
+ return client.createExperiment(simpleExperiment);
+ } catch (AiravataSystemException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new AiravataSystemException(e);
+ } catch (InvalidRequestException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new InvalidRequestException(e);
+ } catch (AiravataClientException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new AiravataClientException(e);
+ }catch (TException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new TException(e);
+ }
+ }
+
+ public static void launchExperiment (Airavata.Client client, String expId)
+ throws ExperimentNotFoundException, AiravataSystemException, InvalidRequestException,AiravataClientException, TException{
+ try {
+ client.launchExperiment(expId, "testToken");
+ } catch (ExperimentNotFoundException e) {
+ logger.error("Error occured while launching the experiment...", e.getMessage());
+ throw new ExperimentNotFoundException(e);
+ } catch (AiravataSystemException e) {
+ logger.error("Error occured while launching the experiment...", e.getMessage());
+ throw new AiravataSystemException(e);
+ } catch (InvalidRequestException e) {
+ logger.error("Error occured while launching the experiment...", e.getMessage());
+ throw new InvalidRequestException(e);
+ } catch (AiravataClientException e) {
+ logger.error("Error occured while launching the experiment...", e.getMessage());
+ throw new AiravataClientException(e);
+ }catch (TException e) {
+ logger.error("Error occured while launching the experiment...", e.getMessage());
+ throw new TException(e);
+ }
+ }
+ }
[2/3] git commit: Implementing slurm support - AIRAVATA-1057
Posted by la...@apache.org.
Implementing slurm support - AIRAVATA-1057
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/b3581061
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/b3581061
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/b3581061
Branch: refs/heads/master
Commit: b35810615eebd9f529dbcf79ca0d6fae909a7657
Parents: 79cb0ef
Author: lahiru <la...@apache.org>
Authored: Thu Mar 6 17:47:43 2014 -0500
Committer: lahiru <la...@apache.org>
Committed: Thu Mar 6 17:47:43 2014 -0500
----------------------------------------------------------------------
.../client/samples/CreateLaunchExperiment.java | 215 -------
.../airavata/client/tools/DocumentCreator.java | 102 ++-
modules/airavata-job-monitor/pom.xml | 5 +
.../job/monitor/AiravataJobStatusUpdator.java | 5 +-
.../monitor/impl/pull/qstat/QstatMonitor.java | 69 ++-
.../impl/pull/qstat/ResourceConnection.java | 238 +------
.../airavata/job/monitor/util/CommonUtils.java | 20 +
.../main/resources/schemas/HostDescription.xsd | 2 +
.../org/apache/airavata/gfac/cpi/GFacImpl.java | 24 +-
.../gfac/provider/impl/GSISSHProvider.java | 3 +-
.../client/OrchestratorClientFactoryTest.java | 7 +-
.../client/sample/OrchestratorClientSample.java | 2 +-
.../orchestrator/core/BaseOrchestratorTest.java | 3 +-
.../orchestrator/core/NewOrchestratorTest.java | 18 +-
.../xbaya/invoker/EmbeddedGFacInvoker.java | 9 +-
.../apache/airavata/gsi/ssh/api/Cluster.java | 15 +-
.../airavata/gsi/ssh/api/job/JobDescriptor.java | 25 +-
.../ssh/api/job/JobManagerConfiguration.java | 38 ++
.../airavata/gsi/ssh/api/job/OutputParser.java | 58 ++
.../gsi/ssh/api/job/PBSJobConfiguration.java | 93 +++
.../gsi/ssh/api/job/PBSOutputParser.java | 141 +++++
.../gsi/ssh/api/job/SlurmJobConfiguration.java | 93 +++
.../gsi/ssh/api/job/SlurmOutputParser.java | 106 ++++
.../gsi/ssh/impl/GSISSHAbstractCluster.java | 417 +++++++++++++
.../airavata/gsi/ssh/impl/PBSCluster.java | 617 +------------------
.../airavata/gsi/ssh/util/CommonUtils.java | 12 +-
.../src/main/resources/SLURMTemplate.xslt | 69 +++
.../main/resources/schemas/PBSJobDescriptor.xsd | 5 +
.../gsi/ssh/impl/DefaultSSHApiTest.java | 88 ++-
.../airavata/gsi/ssh/impl/VanilaSSHTest.java | 5 +-
.../gsissh/src/test/resources/gsissh.properties | 26 +
31 files changed, 1408 insertions(+), 1122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
deleted file mode 100644
index 63184d3..0000000
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.client.samples;
-
-import org.apache.airavata.api.error.ExperimentNotFoundException;
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.ClientSettings;
-import org.apache.airavata.model.workspace.experiment.*;
-import org.apache.airavata.schemas.gfac.DataType;
-import org.apache.airavata.api.Airavata;
-import org.apache.airavata.api.client.AiravataClientFactory;
-import org.apache.airavata.api.error.AiravataClientException;
-import org.apache.airavata.api.error.AiravataSystemException;
-import org.apache.airavata.api.error.InvalidRequestException;
-import org.apache.airavata.client.AiravataAPIFactory;
-import org.apache.airavata.client.api.AiravataAPI;
-import org.apache.airavata.client.api.exception.AiravataAPIInvocationException;
-import org.apache.airavata.client.tools.DocumentCreator;
-import org.apache.airavata.common.utils.AiravataUtils;
-import org.apache.airavata.model.util.ExperimentModelUtil;
-import org.apache.thrift.TException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-public class CreateLaunchExperiment {
-
- //FIXME: Read from a config file
- public static final String THRIFT_SERVER_HOST = "localhost";
- public static final int THRIFT_SERVER_PORT = 8930;
- private final static Logger logger = LoggerFactory.getLogger(CreateLaunchExperiment.class);
- private static final String DEFAULT_USER = "defauly.registry.user";
- private static final String DEFAULT_GATEWAY = "default.registry.gateway";
-
- public static void main(String[] args) {
- try {
- AiravataUtils.setExecutionAsClient();
- final Airavata.Client airavata = AiravataClientFactory.createAiravataClient(THRIFT_SERVER_HOST, THRIFT_SERVER_PORT);
- System.out.println("API version is " + airavata.GetAPIVersion());
- addDescriptors();
- final String expId = createExperiment(airavata);
- System.out.println("Experiment ID : " + expId);
- launchExperiment(airavata, expId);
- System.out.println("Launched successfully");
-// try {
-// Thread.sleep(20000);
-// } catch (InterruptedException e) {
-// e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
-// }
- Thread monitor = (new Thread(){
- public void run() {
- Map<String, JobStatus> jobStatuses = null;
- while (true) {
- try {
- jobStatuses = airavata.getJobStatuses(expId);
- Set<String> strings = jobStatuses.keySet();
- for (String key : strings) {
- JobStatus jobStatus = jobStatuses.get(key);
- if(jobStatus == null){
- return;
- }else {
- if (JobState.COMPLETE.equals(jobStatus.getJobState())) {
- System.out.println("Job completed Job ID: " + key);
- return;
- }else{
- System.out.println("Job ID:" + key + jobStatuses.get(key).getJobState().toString());
- }
- }
- }
- Thread.sleep(5000);
- } catch (Exception e) {
- e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
- }
- }
- }
- });
-// monitor.start();
- try {
- monitor.join();
- } catch (InterruptedException e) {
- e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
- }
-
-// Experiment experiment = airavata.getExperiment(expId);
-// System.out.println("retrieved exp id : " + experiment.getExperimentID());
- } catch (TException e) {
- logger.error("Error while connecting with server", e.getMessage());
- e.printStackTrace();
- } catch (ApplicationSettingsException e) {
- logger.error("Error while creating airavata API object", e.getMessage());
- e.printStackTrace();
- } catch (AiravataAPIInvocationException e) {
- logger.error("Error while creating airavata API object", e.getMessage());
- e.printStackTrace();
- }
- }
-
- public static void addDescriptors() throws AiravataAPIInvocationException,ApplicationSettingsException {
- try {
- DocumentCreator documentCreator = new DocumentCreator(getAiravataAPI());
- documentCreator.createLocalHostDocs();
- documentCreator.createGramDocs();
- documentCreator.createGSISSHDocs();
- } catch (AiravataAPIInvocationException e) {
- logger.error("Unable to create airavata API", e.getMessage());
- throw new AiravataAPIInvocationException(e);
- } catch (ApplicationSettingsException e) {
- logger.error("Unable to create airavata API", e.getMessage());
- throw new ApplicationSettingsException(e.getMessage());
- }
- }
-
- private static AiravataAPI getAiravataAPI() throws AiravataAPIInvocationException, ApplicationSettingsException {
- AiravataAPI airavataAPI;
- try {
- String sysUser = ClientSettings.getSetting(DEFAULT_USER);
- String gateway = ClientSettings.getSetting(DEFAULT_GATEWAY);
- airavataAPI = AiravataAPIFactory.getAPI(gateway, sysUser);
- } catch (AiravataAPIInvocationException e) {
- logger.error("Unable to create airavata API", e.getMessage());
- throw new AiravataAPIInvocationException(e);
- } catch (ApplicationSettingsException e) {
- logger.error("Unable to create airavata API", e.getMessage());
- throw new ApplicationSettingsException(e.getMessage());
- }
- return airavataAPI;
- }
-
- public static String createExperiment (Airavata.Client client) throws AiravataSystemException, InvalidRequestException, AiravataClientException, TException {
- try{
- List<DataObjectType> exInputs = new ArrayList<DataObjectType>();
- DataObjectType input = new DataObjectType();
- input.setKey("echo_input");
- input.setType(DataType.STRING.toString());
- input.setValue("echo_output=Hello World");
- exInputs.add(input);
-
- List<DataObjectType> exOut = new ArrayList<DataObjectType>();
- DataObjectType output = new DataObjectType();
- output.setKey("echo_output");
- output.setType(DataType.STRING.toString());
- output.setValue("");
- exOut.add(output);
-
- Experiment simpleExperiment = ExperimentModelUtil.createSimpleExperiment("project1", "admin", "echoExperiment", "SimpleEcho2", "SimpleEcho2", exInputs);
- simpleExperiment.setExperimentOutputs(exOut);
-
- ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling("trestles.sdsc.edu", 1, 1, 1, "normal", 0, 0, 1, "sds128");
- scheduling.setResourceHostId("gsissh-trestles");
- UserConfigurationData userConfigurationData = new UserConfigurationData();
- userConfigurationData.setAiravataAutoSchedule(false);
- userConfigurationData.setOverrideManualScheduledParams(false);
- userConfigurationData.setComputationalResourceScheduling(scheduling);
- simpleExperiment.setUserConfigurationData(userConfigurationData);
- return client.createExperiment(simpleExperiment);
- } catch (AiravataSystemException e) {
- logger.error("Error occured while creating the experiment...", e.getMessage());
- throw new AiravataSystemException(e);
- } catch (InvalidRequestException e) {
- logger.error("Error occured while creating the experiment...", e.getMessage());
- throw new InvalidRequestException(e);
- } catch (AiravataClientException e) {
- logger.error("Error occured while creating the experiment...", e.getMessage());
- throw new AiravataClientException(e);
- }catch (TException e) {
- logger.error("Error occured while creating the experiment...", e.getMessage());
- throw new TException(e);
- }
- }
-
- public static void launchExperiment (Airavata.Client client, String expId)
- throws ExperimentNotFoundException, AiravataSystemException, InvalidRequestException,AiravataClientException, TException{
- try {
- client.launchExperiment(expId, "testToken");
- } catch (ExperimentNotFoundException e) {
- logger.error("Error occured while launching the experiment...", e.getMessage());
- throw new ExperimentNotFoundException(e);
- } catch (AiravataSystemException e) {
- logger.error("Error occured while launching the experiment...", e.getMessage());
- throw new AiravataSystemException(e);
- } catch (InvalidRequestException e) {
- logger.error("Error occured while launching the experiment...", e.getMessage());
- throw new InvalidRequestException(e);
- } catch (AiravataClientException e) {
- logger.error("Error occured while launching the experiment...", e.getMessage());
- throw new AiravataClientException(e);
- }catch (TException e) {
- logger.error("Error occured while launching the experiment...", e.getMessage());
- throw new TException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/modules/airavata-client/src/main/java/org/apache/airavata/client/tools/DocumentCreator.java
----------------------------------------------------------------------
diff --git a/modules/airavata-client/src/main/java/org/apache/airavata/client/tools/DocumentCreator.java b/modules/airavata-client/src/main/java/org/apache/airavata/client/tools/DocumentCreator.java
index 7b07874..eac7214 100644
--- a/modules/airavata-client/src/main/java/org/apache/airavata/client/tools/DocumentCreator.java
+++ b/modules/airavata-client/src/main/java/org/apache/airavata/client/tools/DocumentCreator.java
@@ -208,7 +208,7 @@ public class DocumentCreator {
}
}
- public void createGSISSHDocs() {
+ public void createPBSDocs() {
HostDescription host = new HostDescription(GsisshHostType.type);
host.getType().setHostAddress(hpcHostAddress);
host.getType().setHostName(gsiSshHostName);
@@ -305,6 +305,106 @@ public class DocumentCreator {
}
}
+ public void createSlurmDocs() {
+ HostDescription host = new HostDescription(GsisshHostType.type);
+ host.getType().setHostAddress("stampede.tacc.xsede.org");
+ host.getType().setHostName("stampede-host");
+ ((GsisshHostType) host.getType()).setJobManager("slurm");
+ ((GsisshHostType) host.getType()).setPort(2222);
+
+
+ try {
+ airavataAPI.getApplicationManager().saveHostDescription(host);
+ } catch (AiravataAPIInvocationException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+ /*
+ * Service Description creation and saving
+ */
+ String serviceName = "SimpleEcho2";
+ ServiceDescription serv = new ServiceDescription();
+ serv.getType().setName(serviceName);
+
+ List<InputParameterType> inputList = new ArrayList<InputParameterType>();
+ List<OutputParameterType> outputList = new ArrayList<OutputParameterType>();
+
+
+ InputParameterType input = InputParameterType.Factory.newInstance();
+ input.setParameterName("echo_input");
+ ParameterType parameterType = input.addNewParameterType();
+ parameterType.setType(DataType.STRING);
+ parameterType.setName("String");
+
+ OutputParameterType output = OutputParameterType.Factory.newInstance();
+ output.setParameterName("echo_output");
+ ParameterType parameterType1 = output.addNewParameterType();
+ parameterType1.setType(DataType.STRING);
+ parameterType1.setName("String");
+
+ inputList.add(input);
+ outputList.add(output);
+
+ InputParameterType[] inputParamList = inputList.toArray(new InputParameterType[inputList.size()]);
+ OutputParameterType[] outputParamList = outputList.toArray(new OutputParameterType[outputList.size()]);
+
+ serv.getType().setInputParametersArray(inputParamList);
+ serv.getType().setOutputParametersArray(outputParamList);
+ try {
+ airavataAPI.getApplicationManager().saveServiceDescription(serv);
+ } catch (AiravataAPIInvocationException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+
+ /*
+ Application descriptor creation and saving
+ */
+ ApplicationDescription appDesc = new ApplicationDescription(HpcApplicationDeploymentType.type);
+ HpcApplicationDeploymentType app = (HpcApplicationDeploymentType) appDesc.getType();
+ ApplicationDeploymentDescriptionType.ApplicationName name = ApplicationDeploymentDescriptionType.ApplicationName.Factory.newInstance();
+ name.setStringValue("EchoLocal");
+ app.setApplicationName(name);
+ ProjectAccountType projectAccountType = app.addNewProjectAccount();
+ projectAccountType.setProjectAccountNumber("TG-STA110014S");
+
+ QueueType queueType = app.addNewQueue();
+ queueType.setQueueName("normal");
+
+ app.setCpuCount(1);
+ app.setJobType(JobTypeType.SERIAL);
+ app.setNodeCount(1);
+ app.setProcessorsPerNode(1);
+ app.setMaxWallTime(10);
+ /*
+ * Use bat file if it is compiled on Windows
+ */
+ app.setExecutableLocation("/bin/echo");
+
+ /*
+ * Default tmp location
+ */
+ String tempDir = "/home1/01437/ogce";
+ String date = (new Date()).toString();
+ date = date.replaceAll(" ", "_");
+ date = date.replaceAll(":", "_");
+
+ tempDir = tempDir + File.separator
+ + "SimpleEcho" + "_" + date + "_" + UUID.randomUUID();
+
+ app.setScratchWorkingDirectory(tempDir);
+ app.setStaticWorkingDirectory(tempDir);
+ app.setInputDataDirectory(tempDir + File.separator + "inputData");
+ app.setOutputDataDirectory(tempDir + File.separator + "outputData");
+ app.setStandardOutput(tempDir + File.separator + app.getApplicationName().getStringValue() + ".stdout");
+ app.setStandardError(tempDir + File.separator + app.getApplicationName().getStringValue() + ".stderr");
+ app.setInstalledParentPath("/usr/bin/");
+
+ try {
+ airavataAPI.getApplicationManager().saveApplicationDescription(serviceName, "stampede-host", appDesc);
+ } catch (AiravataAPIInvocationException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+ }
+
public AiravataAPI getAiravataAPI() {
return airavataAPI;
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/modules/airavata-job-monitor/pom.xml
----------------------------------------------------------------------
diff --git a/modules/airavata-job-monitor/pom.xml b/modules/airavata-job-monitor/pom.xml
index 2d1a042..520c9a6 100644
--- a/modules/airavata-job-monitor/pom.xml
+++ b/modules/airavata-job-monitor/pom.xml
@@ -58,6 +58,11 @@
<dependency>
<groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-gfac-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
<artifactId>airavata-registry-cpi</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/AiravataJobStatusUpdator.java
----------------------------------------------------------------------
diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/AiravataJobStatusUpdator.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/AiravataJobStatusUpdator.java
index 78f0621..8b458a5 100644
--- a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/AiravataJobStatusUpdator.java
+++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/AiravataJobStatusUpdator.java
@@ -108,14 +108,13 @@ public class AiravataJobStatusUpdator{
}
}
}
- public static void updateJobStatus(String jobID, JobState state) throws Exception {
- Registry registry = RegistryFactory.getDefaultRegistry();
+ public void updateJobStatus(String jobID, JobState state) throws Exception {
JobDetails details = new JobDetails();
org.apache.airavata.model.workspace.experiment.JobStatus status = new org.apache.airavata.model.workspace.experiment.JobStatus();
status.setJobState(state);
status.setTimeOfStateChange(Calendar.getInstance().getTimeInMillis());
details.setJobStatus(status);
details.setJobID(jobID);
- registry.update(org.apache.airavata.registry.cpi.DataType.JOB_DETAIL, details, jobID);
+ airavataRegistry.update(org.apache.airavata.registry.cpi.DataType.JOB_DETAIL, details, jobID);
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/pull/qstat/QstatMonitor.java
----------------------------------------------------------------------
diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/pull/qstat/QstatMonitor.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/pull/qstat/QstatMonitor.java
index c9f8331..fa18db8 100644
--- a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/pull/qstat/QstatMonitor.java
+++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/pull/qstat/QstatMonitor.java
@@ -97,42 +97,47 @@ public class QstatMonitor extends PullMonitor implements Runnable {
while (!this.queue.isEmpty()) {
try {
take = this.queue.take();
- long monitorDiff = 0;
- long startedDiff = 0;
- if (take.getLastMonitored() != null) {
- monitorDiff = (new Timestamp((new Date()).getTime())).getTime() - take.getLastMonitored().getTime();
- startedDiff = (new Timestamp((new Date()).getTime())).getTime() - take.getJobStartedTime().getTime();
- //todo implement an algorithm to delay the monitor based no start time, we have to delay monitoring
- //todo for long running jobs
+ if (take.getHost().getType() instanceof GsisshHostType) {
+ long monitorDiff = 0;
+ long startedDiff = 0;
+ if (take.getLastMonitored() != null) {
+ monitorDiff = (new Timestamp((new Date()).getTime())).getTime() - take.getLastMonitored().getTime();
+ startedDiff = (new Timestamp((new Date()).getTime())).getTime() - take.getJobStartedTime().getTime();
+ //todo implement an algorithm to delay the monitor based no start time, we have to delay monitoring
+ //todo for long running jobs
// System.out.println(monitorDiff + "-" + startedDiff);
- if ((monitorDiff / 1000) < 5) {
- // its too early to monitor this job, so we put it at the tail of the queue
- this.queue.put(take);
- }
- }
- if (take.getLastMonitored() == null || ((monitorDiff / 1000) >= 5)) {
- GsisshHostType gsisshHostType = (GsisshHostType) take.getHost().getType();
- String hostName = gsisshHostType.getHostAddress();
- ResourceConnection connection = null;
- if (connections.containsKey(hostName)) {
- logger.debug("We already have this connection so not going to create one");
- connection = connections.get(hostName);
- } else {
- if(gsisshHostType.getInstalledPath() == null){
- connection = new ResourceConnection(take, "/opt/torque/bin");
- }else{
- connection = new ResourceConnection(take, gsisshHostType.getInstalledPath());
+ if ((monitorDiff / 1000) < 5) {
+ // its too early to monitor this job, so we put it at the tail of the queue
+ this.queue.put(take);
}
- connections.put(hostName, connection);
}
- jobStatus.setMonitorID(take);
- jobStatus.setState(connection.getJobStatus(take));
- publisher.publish(jobStatus);
- // if the job is completed we do not have to put the job to the queue again
- if (!jobStatus.getState().equals(JobState.COMPLETE)) {
- take.setLastMonitored(new Timestamp((new Date()).getTime()));
- this.queue.put(take);
+ if (take.getLastMonitored() == null || ((monitorDiff / 1000) >= 5)) {
+ GsisshHostType gsisshHostType = (GsisshHostType) take.getHost().getType();
+ String hostName = gsisshHostType.getHostAddress();
+ ResourceConnection connection = null;
+ if (connections.containsKey(hostName)) {
+ logger.debug("We already have this connection so not going to create one");
+ connection = connections.get(hostName);
+ } else {
+ if (gsisshHostType.getInstalledPath() == null) {
+ connection = new ResourceConnection(take, gsisshHostType.getInstalledPath());
+ } else {
+ connection = new ResourceConnection(take, gsisshHostType.getInstalledPath());
+ }
+ connections.put(hostName, connection);
+ }
+ jobStatus.setMonitorID(take);
+ jobStatus.setState(connection.getJobStatus(take));
+ publisher.publish(jobStatus);
+ // if the job is completed we do not have to put the job to the queue again
+ if (!jobStatus.getState().equals(JobState.COMPLETE)) {
+ take.setLastMonitored(new Timestamp((new Date()).getTime()));
+ this.queue.put(take);
+ }
}
+ } else {
+ //Qstat doesn't handle other jobs eexcept GsisshHostTypes
+ this.queue.put(take);
}
} catch (InterruptedException e) {
if(!this.queue.contains(take)){
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/pull/qstat/ResourceConnection.java
----------------------------------------------------------------------
diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/pull/qstat/ResourceConnection.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/pull/qstat/ResourceConnection.java
index 1a2d04d..60d2b9b 100644
--- a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/pull/qstat/ResourceConnection.java
+++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/pull/qstat/ResourceConnection.java
@@ -20,217 +20,56 @@
*/
package org.apache.airavata.job.monitor.impl.pull.qstat;
-import com.jcraft.jsch.*;
-import org.apache.airavata.gsi.ssh.api.CommandExecutor;
import org.apache.airavata.gsi.ssh.api.SSHApiException;
+import org.apache.airavata.gsi.ssh.api.ServerInfo;
import org.apache.airavata.gsi.ssh.api.authentication.*;
-import org.apache.airavata.gsi.ssh.config.ConfigReader;
-import org.apache.airavata.gsi.ssh.impl.RawCommandInfo;
-import org.apache.airavata.gsi.ssh.impl.StandardOutReader;
-import org.apache.airavata.gsi.ssh.jsch.ExtendedJSch;
-import org.apache.airavata.gsi.ssh.util.SSHAPIUIKeyboardInteractive;
-import org.apache.airavata.gsi.ssh.util.SSHKeyPasswordHandler;
+import org.apache.airavata.gsi.ssh.api.job.JobManagerConfiguration;
+import org.apache.airavata.gsi.ssh.impl.PBSCluster;
+import org.apache.airavata.gsi.ssh.util.CommonUtils;
import org.apache.airavata.job.monitor.MonitorID;
import org.apache.airavata.model.workspace.experiment.JobState;
+import org.apache.airavata.schemas.gfac.GsisshHostType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
public class ResourceConnection {
- static {
- JSch.setConfig("gssapi-with-mic.x509", "org.apache.airavata.gsi.ssh.GSSContextX509");
- JSch.setConfig("userauth.gssapi-with-mic", "com.jcraft.jsch.UserAuthGSSAPIWithMICGSSCredentials");
-
- }
-
private static final Logger log = LoggerFactory.getLogger(ResourceConnection.class);
- public static final String X509_CERT_DIR = "X509_CERT_DIR";
- public static final String SSH_SESSION_TIMEOUT = "ssh.session.timeout";
-
- private Session session;
- private ConfigReader configReader;
-
- private String installedPath;
+ private PBSCluster cluster;
public ResourceConnection(MonitorID monitorID, String installedPath) throws SSHApiException {
AuthenticationInfo authenticationInfo = monitorID.getAuthenticationInfo();
String hostAddress = monitorID.getHost().getType().getHostAddress();
String userName = monitorID.getUserName();
int port = monitorID.getPort();
- if (authenticationInfo instanceof GSIAuthenticationInfo) {
- System.setProperty(X509_CERT_DIR, (String) ((GSIAuthenticationInfo) authenticationInfo).getProperties().
- get("X509_CERT_DIR"));
- }
- if (installedPath == null) {
- throw new SSHApiException("Installed path cannot be null !!");
- }
- if (installedPath.endsWith("/")) {
- this.installedPath = installedPath;
+ String jobManager = ((GsisshHostType)monitorID.getHost().getType()).getJobManager();
+ JobManagerConfiguration jConfig = null;
+ if (jobManager == null) {
+ log.error("No Job Manager is configured, so we are picking pbs as the default job manager");
+ jConfig = CommonUtils.getPBSJobManager(installedPath);
} else {
- this.installedPath = installedPath + "/";
- }
-
-
- try {
- this.configReader = new ConfigReader();
- } catch (IOException e) {
- throw new SSHApiException("Unable to load system configurations.", e);
- }
- JSch jSch = new ExtendedJSch();
-
- log.debug("Connecting to server - " + monitorID.getHost().getType().getHostName() + ":" + "22" + " with user name - "
- + userName);
-
- try {
- session = jSch.getSession(userName, hostAddress, 22);
- session.setTimeout(Integer.parseInt(configReader.getConfiguration(SSH_SESSION_TIMEOUT)));
- } catch (Exception e) {
- throw new SSHApiException("An exception occurred while creating SSH session." +
- "Connecting server - " + hostAddress + ":" + 22 +
- " connecting user name - "
- + userName, e);
- }
-
- java.util.Properties config = this.configReader.getProperties();
- session.setConfig(config);
-
-
- //=============================================================
- // Handling vanilla SSH pieces
- //=============================================================
- if (authenticationInfo instanceof SSHPasswordAuthentication) {
- String password = ((SSHPasswordAuthentication) authenticationInfo).
- getPassword(userName, hostAddress);
-
- session.setUserInfo(new SSHAPIUIKeyboardInteractive(password));
-
- // TODO figure out why we need to set password to session
- session.setPassword(password);
-
- } else if (authenticationInfo instanceof SSHPublicKeyFileAuthentication) {
- SSHPublicKeyFileAuthentication sshPublicKeyFileAuthentication
- = (SSHPublicKeyFileAuthentication) authenticationInfo;
-
- String privateKeyFile = sshPublicKeyFileAuthentication.
- getPrivateKeyFile(userName, hostAddress);
-
- log.debug("The private key file for vanilla SSH " + privateKeyFile);
-
- String publicKeyFile = sshPublicKeyFileAuthentication.
- getPrivateKeyFile(userName, hostAddress);
-
- log.debug("The public key file for vanilla SSH " + publicKeyFile);
-
- Identity identityFile;
-
- try {
- identityFile = GSISSHIdentityFile.newInstance(privateKeyFile, null, jSch);
- } catch (JSchException e) {
- throw new SSHApiException("An exception occurred while initializing keys using files. " +
- "(private key and public key)." +
- "Connecting server - " + hostAddress + ":" + port +
- " connecting user name - "
- + userName + " private key file - " + privateKeyFile + ", public key file - " +
- publicKeyFile, e);
- }
-
- // Add identity to identity repository
- GSISSHIdentityRepository identityRepository = new GSISSHIdentityRepository(jSch);
- identityRepository.add(identityFile);
-
- // Set repository to session
- session.setIdentityRepository(identityRepository);
-
- // Set the user info
- SSHKeyPasswordHandler sshKeyPasswordHandler
- = new SSHKeyPasswordHandler((SSHKeyAuthentication) authenticationInfo);
-
- session.setUserInfo(sshKeyPasswordHandler);
-
- } else if (authenticationInfo instanceof SSHPublicKeyAuthentication) {
-
- SSHPublicKeyAuthentication sshPublicKeyAuthentication
- = (SSHPublicKeyAuthentication) authenticationInfo;
-
- Identity identityFile;
-
- try {
- String name = userName + "_" + hostAddress;
- identityFile = GSISSHIdentityFile.newInstance(name,
- sshPublicKeyAuthentication.getPrivateKey(userName, hostAddress),
- sshPublicKeyAuthentication.getPublicKey(userName, hostAddress), jSch);
- } catch (JSchException e) {
- throw new SSHApiException("An exception occurred while initializing keys using byte arrays. " +
- "(private key and public key)." +
- "Connecting server - " + hostAddress + ":" + port +
- " connecting user name - "
- + userName, e);
+ if (org.apache.airavata.job.monitor.util.CommonUtils.isPBSHost(monitorID.getHost())) {
+ jConfig = CommonUtils.getPBSJobManager(installedPath);
+ } else if(org.apache.airavata.job.monitor.util.CommonUtils.isSlurm(monitorID.getHost())) {
+ jConfig = CommonUtils.getSLURMJobManager(installedPath);
}
-
- // Add identity to identity repository
- GSISSHIdentityRepository identityRepository = new GSISSHIdentityRepository(jSch);
- identityRepository.add(identityFile);
-
- // Set repository to session j
- session.setIdentityRepository(identityRepository);
-
- // Set the user info
- SSHKeyPasswordHandler sshKeyPasswordHandler
- = new SSHKeyPasswordHandler((SSHKeyAuthentication) authenticationInfo);
-
- session.setUserInfo(sshKeyPasswordHandler);
-
+ //todo support br2 etc
}
-
- // Not a good way, but we dont have any choice
- if (session instanceof ExtendedSession) {
- if (authenticationInfo instanceof GSIAuthenticationInfo) {
- ((ExtendedSession) session).setAuthenticationInfo((GSIAuthenticationInfo) authenticationInfo);
- }
- }
-
- try {
- session.connect();
- } catch (JSchException e) {
- throw new SSHApiException("An exception occurred while connecting to server." +
- "Connecting server - " + hostAddress + ":" + port +
- " connecting user name - "
- + userName, e);
- }
- System.out.println(session.isConnected());
+ ServerInfo serverInfo = new ServerInfo(monitorID.getUserName(), hostAddress, monitorID.getPort());
+ cluster = new PBSCluster(serverInfo, authenticationInfo, jConfig);
}
public JobState getJobStatus(MonitorID monitorID) throws SSHApiException {
String jobID = monitorID.getJobID();
//todo so currently we execute the qstat for each job but we can use user based monitoring
//todo or we should concatenate all the commands and execute them in one go and parse the response
- RawCommandInfo rawCommandInfo = new RawCommandInfo(this.installedPath + "qstat -f " + jobID);
-
- StandardOutReader stdOutReader = new StandardOutReader();
- CommandExecutor.executeCommand(rawCommandInfo, this.getSession(), stdOutReader);
-
-
- String result = getOutputifAvailable(stdOutReader, "Error getting job status with job ID: " + jobID);
- String[] info = result.split("\n");
- String[] line = null;
- for (String anInfo : info) {
- if (anInfo.contains("=")) {
- line = anInfo.split("=", 2);
- if (line.length != 0) {
- if (line[0].contains("job_state")) {
- return getStatusFromString(line[1].replaceAll(" ", ""));
- }
- }
- }
- }
- return null;
+ return getStatusFromString(cluster.getJobStatus(monitorID.getJobID()).toString());
}
private JobState getStatusFromString(String status) {
if(status != null){
- if("C".equals(status)){
+ if("C".equals(status) || "CD".equals(status)){
return JobState.COMPLETE;
}else if("E".equals(status)){
return JobState.COMPLETE;
@@ -238,43 +77,22 @@ public class ResourceConnection {
return JobState.HELD;
}else if("Q".equals(status)){
return JobState.QUEUED;
- }else if("R".equals(status)){
+ }else if("R".equals(status) || "CG".equals(status) || "CF".equals(status)){
return JobState.ACTIVE;
}else if ("T".equals(status)) {
return JobState.HELD;
- } else if ("W".equals(status)) {
+ } else if ("W".equals(status) || "PD".equals(status)) {
return JobState.QUEUED;
} else if ("S".equals(status)) {
return JobState.SUSPENDED;
+ }else if("CA".equals(status)){
+ return JobState.CANCELED;
+ }else if ("F".equals(status) || "NF".equals(status) || "TO".equals(status)) {
+ return JobState.FAILED;
+ }else if ("PR".equals(status)) {
+ return JobState.FAILED;
}
}
return null;
}
- public Session getSession() {
- return session;
- }
-
- public void setSession(Session session) {
- this.session = session;
- }
-
- /**
- * This method will read standard output and if there's any it will be parsed
- *
- * @param jobIDReaderCommandOutput
- * @param errorMsg
- * @return
- * @throws SSHApiException
- */
- private String getOutputifAvailable(StandardOutReader jobIDReaderCommandOutput, String errorMsg) throws SSHApiException {
- String stdOutputString = jobIDReaderCommandOutput.getStdOutputString();
- String stdErrorString = jobIDReaderCommandOutput.getStdErrorString();
-
- if (stdOutputString == null && "".equals(stdOutputString) ||
- ((stdErrorString != null) && !("".equals(stdErrorString)))) {
- log.error("Standard Error output : " + stdErrorString);
- throw new SSHApiException(errorMsg + stdErrorString);
- }
- return stdOutputString;
- }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/util/CommonUtils.java
----------------------------------------------------------------------
diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/util/CommonUtils.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/util/CommonUtils.java
index a2254df..a6cd465 100644
--- a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/util/CommonUtils.java
+++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/util/CommonUtils.java
@@ -20,12 +20,32 @@
*/
package org.apache.airavata.job.monitor.util;
+import org.apache.airavata.commons.gfac.type.HostDescription;
import org.apache.airavata.job.monitor.MonitorID;
+import org.apache.airavata.schemas.gfac.GsisshHostType;
public class CommonUtils {
+ public static boolean isPBSHost(HostDescription host){
+ if("pbs".equals(((GsisshHostType)host.getType()).getJobManager()) ||
+ "".equals(((GsisshHostType)host.getType()).getJobManager())){
+ return true;
+ }else{
+ // default is pbs so we return true
+ return false;
+ }
+ }
+ public static boolean isSlurm(HostDescription host){
+ if("slurm".equals(((GsisshHostType)host.getType()).getJobManager())){
+ return true;
+ }else{
+ // default is pbs so we return true
+ return false;
+ }
+ }
public static String getChannelID(MonitorID monitorID) {
return monitorID.getUserName() + "-" + monitorID.getHost().getType().getHostName();
}
+
public static String getRoutingKey(MonitorID monitorID) {
return "*." + monitorID.getUserName() + "." + monitorID.getHost().getType().getHostAddress();
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/modules/commons/gfac-schema/src/main/resources/schemas/HostDescription.xsd
----------------------------------------------------------------------
diff --git a/modules/commons/gfac-schema/src/main/resources/schemas/HostDescription.xsd b/modules/commons/gfac-schema/src/main/resources/schemas/HostDescription.xsd
index 27dbed1..4ec3a4a 100644
--- a/modules/commons/gfac-schema/src/main/resources/schemas/HostDescription.xsd
+++ b/modules/commons/gfac-schema/src/main/resources/schemas/HostDescription.xsd
@@ -106,10 +106,12 @@
<complexContent>
<extension base="gfac:hostDescriptionType">
<sequence>
+ <element name="port" type="xsd:int" minOccurs="0" maxOccurs="1"/>
<element name="exports" type="gfac:exportProperties" minOccurs="0" maxOccurs="1"/>
<element name="preJobCommands" type="xsd:string" minOccurs="0" maxOccurs="unbounded"/>
<element name="postJobCommands" type="xsd:string" minOccurs="0" maxOccurs="unbounded"/>
<element name="installedPath" type="xsd:string" minOccurs="0" maxOccurs="1" default="/opt/torque/bin"/>
+ <element name="jobManager" type="xsd:string" minOccurs="0" maxOccurs="1"/>
</sequence>
</extension>
</complexContent>
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/cpi/GFacImpl.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/cpi/GFacImpl.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/cpi/GFacImpl.java
index a60b200..8988554 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/cpi/GFacImpl.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/cpi/GFacImpl.java
@@ -48,10 +48,12 @@ import org.apache.airavata.gsi.ssh.api.SSHApiException;
import org.apache.airavata.gsi.ssh.api.ServerInfo;
import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
import org.apache.airavata.gsi.ssh.api.authentication.GSIAuthenticationInfo;
+import org.apache.airavata.gsi.ssh.api.job.JobManagerConfiguration;
import org.apache.airavata.gsi.ssh.impl.PBSCluster;
import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPasswordAuthenticationInfo;
import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPublicKeyFileAuthentication;
import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
+import org.apache.airavata.gsi.ssh.util.CommonUtils;
import org.apache.airavata.model.workspace.experiment.DataObjectType;
import org.apache.airavata.model.workspace.experiment.TaskDetails;
import org.apache.airavata.registry.api.AiravataRegistry2;
@@ -61,6 +63,7 @@ import org.apache.airavata.schemas.gfac.*;
import org.apache.airavata.schemas.wec.ContextHeaderDocument;
import org.apache.airavata.schemas.wec.SecurityContextDocument;
import org.apache.airavata.workflow.model.exceptions.WorkflowException;
+import org.apache.xmlbeans.XmlObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -376,8 +379,21 @@ public class GFacImpl implements GFac {
Cluster pbsCluster = null;
try {
- pbsCluster = new PBSCluster(serverInfo, authenticationInfo,
- (((HpcApplicationDeploymentType) jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType()).getInstalledParentPath()));
+ JobManagerConfiguration jConfig = null;
+ String installedParentPath = ((HpcApplicationDeploymentType)
+ jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType()).getInstalledParentPath();
+ String jobManager = ((GsisshHostType) registeredHost.getType()).getJobManager();
+ if(jobManager == null){
+ log.error("No Job Manager is configured, so we are picking pbs as the default job manager");
+ jConfig = CommonUtils.getPBSJobManager(installedParentPath);
+ }else{
+ if("pbs".equalsIgnoreCase(jobManager)){
+ jConfig = CommonUtils.getPBSJobManager(installedParentPath);
+ }else if("slurm".equalsIgnoreCase(jobManager)){
+ jConfig = CommonUtils.getSLURMJobManager(installedParentPath);
+ }
+ }
+ pbsCluster = new PBSCluster(serverInfo, authenticationInfo,jConfig);
} catch (SSHApiException e) {
e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
}
@@ -409,8 +425,10 @@ public class GFacImpl implements GFac {
Cluster pbsCluster = null;
try {
+ String installedParentPath = ((HpcApplicationDeploymentType)
+ jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType()).getInstalledParentPath();
pbsCluster = new PBSCluster(serverInfo, authenticationInfo,
- (((HpcApplicationDeploymentType) jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType()).getInstalledParentPath()));
+ CommonUtils.getPBSJobManager(installedParentPath));
} catch (SSHApiException e) {
e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/provider/impl/GSISSHProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/provider/impl/GSISSHProvider.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/provider/impl/GSISSHProvider.java
index 8d3fcbf..1996500 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/provider/impl/GSISSHProvider.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/provider/impl/GSISSHProvider.java
@@ -163,7 +163,8 @@ public class GSISSHProvider extends AbstractProvider implements GFacProvider{
String jobID = cluster.submitBatchJob(jobDescriptor);
jobDetails.setJobID(jobID);
jobExecutionContext.setJobDetails(jobDetails);
- GFacUtils.saveJobStatus(jobDetails,JobState.QUEUED,taskID);
+ GFacUtils.saveJobStatus(jobDetails, JobState.SUBMITTED, taskID);
+
} catch (SSHApiException e) {
String error = "Error submitting the job to host " + host.getHostAddress() + e.getMessage();
log.error(error);
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/modules/orchestrator/airavata-orchestrator-service/src/test/java/org/apache/airavata/orchestrator/client/OrchestratorClientFactoryTest.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/airavata-orchestrator-service/src/test/java/org/apache/airavata/orchestrator/client/OrchestratorClientFactoryTest.java b/modules/orchestrator/airavata-orchestrator-service/src/test/java/org/apache/airavata/orchestrator/client/OrchestratorClientFactoryTest.java
index 829f25b..acaf785 100644
--- a/modules/orchestrator/airavata-orchestrator-service/src/test/java/org/apache/airavata/orchestrator/client/OrchestratorClientFactoryTest.java
+++ b/modules/orchestrator/airavata-orchestrator-service/src/test/java/org/apache/airavata/orchestrator/client/OrchestratorClientFactoryTest.java
@@ -58,7 +58,8 @@ public class OrchestratorClientFactoryTest {
documentCreator = new DocumentCreator(getAiravataAPI());
documentCreator.createLocalHostDocs();
documentCreator.createGramDocs();
- documentCreator.createGSISSHDocs();
+ documentCreator.createPBSDocs();
+ documentCreator.createSlurmDocs();
}
private AiravataAPI getAiravataAPI() {
@@ -103,8 +104,8 @@ public class OrchestratorClientFactoryTest {
"admin", "echoExperiment", "SimpleEcho2", "SimpleEcho2", exInputs);
simpleExperiment.setExperimentOutputs(exOut);
- ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling("trestles.sdsc.edu", 1, 1, 1, "normal", 0, 0, 1, "sds128");
- scheduling.setResourceHostId("gsissh-trestles");
+ ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling("stampede.tacc.xsede.org", 1, 1, 1, "normal", 0, 0, 1, "TG-STA110014S");
+ scheduling.setResourceHostId("stampede-host");
UserConfigurationData userConfigurationData = new UserConfigurationData();
userConfigurationData.setComputationalResourceScheduling(scheduling);
simpleExperiment.setUserConfigurationData(userConfigurationData);
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/modules/orchestrator/orchestrator-client-sdks/src/main/java/org/apache/airavata/orchestrator/client/sample/OrchestratorClientSample.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-client-sdks/src/main/java/org/apache/airavata/orchestrator/client/sample/OrchestratorClientSample.java b/modules/orchestrator/orchestrator-client-sdks/src/main/java/org/apache/airavata/orchestrator/client/sample/OrchestratorClientSample.java
index f1753f4..3678934 100644
--- a/modules/orchestrator/orchestrator-client-sdks/src/main/java/org/apache/airavata/orchestrator/client/sample/OrchestratorClientSample.java
+++ b/modules/orchestrator/orchestrator-client-sdks/src/main/java/org/apache/airavata/orchestrator/client/sample/OrchestratorClientSample.java
@@ -67,7 +67,7 @@ public class OrchestratorClientSample {
documentCreator = new DocumentCreator(getAiravataAPI());
documentCreator.createLocalHostDocs();
documentCreator.createGramDocs();
- documentCreator.createGSISSHDocs();
+ documentCreator.createPBSDocs();
storeExperimentDetail();
} catch (ApplicationSettingsException e) {
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/BaseOrchestratorTest.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/BaseOrchestratorTest.java b/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/BaseOrchestratorTest.java
index 1b20e19..57940b6 100644
--- a/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/BaseOrchestratorTest.java
+++ b/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/BaseOrchestratorTest.java
@@ -28,7 +28,6 @@ import org.apache.airavata.client.tools.DocumentCreator;
import org.apache.airavata.common.exception.ApplicationSettingsException;
import org.apache.airavata.common.utils.ServerSettings;
import org.apache.airavata.orchestrator.core.util.Initialize;
-import org.apache.airavata.persistance.registry.jpa.ResourceType;
import org.apache.airavata.persistance.registry.jpa.ResourceUtils;
import org.apache.airavata.persistance.registry.jpa.resources.*;
@@ -52,7 +51,7 @@ public class BaseOrchestratorTest {
documentCreator = new DocumentCreator(getAiravataAPI());
documentCreator.createLocalHostDocs();
documentCreator.createGramDocs();
- documentCreator.createGSISSHDocs();
+ documentCreator.createPBSDocs();
}
public void tearDown() throws Exception {
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/NewOrchestratorTest.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/NewOrchestratorTest.java b/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/NewOrchestratorTest.java
index b035e2b..522e3b2 100644
--- a/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/NewOrchestratorTest.java
+++ b/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/NewOrchestratorTest.java
@@ -101,16 +101,14 @@ public class NewOrchestratorTest extends BaseOrchestratorTest {
private AiravataAPI getAiravataAPI() {
AiravataAPI airavataAPI = null;
- if (airavataAPI == null) {
- try {
- String systemUserName = ServerSettings.getSystemUser();
- String gateway = ServerSettings.getSystemUserGateway();
- airavataAPI = AiravataAPIFactory.getAPI(gateway, systemUserName);
- } catch (ApplicationSettingsException e) {
- e.printStackTrace();
- } catch (AiravataAPIInvocationException e) {
- e.printStackTrace();
- }
+ try {
+ String systemUserName = ServerSettings.getSystemUser();
+ String gateway = ServerSettings.getSystemUserGateway();
+ airavataAPI = AiravataAPIFactory.getAPI(gateway, systemUserName);
+ } catch (ApplicationSettingsException e) {
+ e.printStackTrace();
+ } catch (AiravataAPIInvocationException e) {
+ e.printStackTrace();
}
return airavataAPI;
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/modules/xbaya-gui/src/main/java/org/apache/airavata/xbaya/invoker/EmbeddedGFacInvoker.java
----------------------------------------------------------------------
diff --git a/modules/xbaya-gui/src/main/java/org/apache/airavata/xbaya/invoker/EmbeddedGFacInvoker.java b/modules/xbaya-gui/src/main/java/org/apache/airavata/xbaya/invoker/EmbeddedGFacInvoker.java
index 564eb2b..267b838 100644
--- a/modules/xbaya-gui/src/main/java/org/apache/airavata/xbaya/invoker/EmbeddedGFacInvoker.java
+++ b/modules/xbaya-gui/src/main/java/org/apache/airavata/xbaya/invoker/EmbeddedGFacInvoker.java
@@ -61,6 +61,7 @@ import org.apache.airavata.gsi.ssh.impl.PBSCluster;
import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPasswordAuthenticationInfo;
import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPublicKeyFileAuthentication;
import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
+import org.apache.airavata.gsi.ssh.util.CommonUtils;
import org.apache.airavata.registry.api.exception.RegistryException;
import org.apache.airavata.schemas.gfac.*;
import org.apache.airavata.schemas.wec.ContextHeaderDocument;
@@ -461,8 +462,10 @@ public class EmbeddedGFacInvoker implements Invoker {
Cluster pbsCluster = null;
try {
+ String installedParentPath = ((HpcApplicationDeploymentType)
+ jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType()).getInstalledParentPath();
pbsCluster = new PBSCluster(serverInfo, authenticationInfo,
- (((HpcApplicationDeploymentType) jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType()).getInstalledParentPath()));
+ (CommonUtils.getPBSJobManager(installedParentPath)));
} catch (SSHApiException e) {
e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
}
@@ -495,8 +498,10 @@ public class EmbeddedGFacInvoker implements Invoker {
Cluster pbsCluster = null;
try {
+ String installedParentPath = ((HpcApplicationDeploymentType)
+ jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType()).getInstalledParentPath();
pbsCluster = new PBSCluster(serverInfo, authenticationInfo,
- (((HpcApplicationDeploymentType) jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType()).getInstalledParentPath()));
+ (CommonUtils.getPBSJobManager(installedParentPath)));
} catch (SSHApiException e) {
e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
index 5f00458..edea8aa 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
@@ -38,7 +38,7 @@ public interface Cluster {
* @return jobId after successful job submission
* @throws SSHApiException throws exception during error
*/
- public String submitBatchJobWithPBS(String pbsFilePath, String workingDirectory) throws SSHApiException;
+ public String submitBatchJobWithScript(String pbsFilePath, String workingDirectory) throws SSHApiException;
/**
* This will submit the given job and not performing any monitoring
@@ -50,15 +50,6 @@ public interface Cluster {
public String submitBatchJob(JobDescriptor jobDescriptor) throws SSHApiException;
/**
- * This will get all the information about the cluster and store them as parameters
- * So that api user can extract required information about the cluster
- *
- * @return return a cluster which consists of information about nodes etc.
- * @throws SSHApiException throws exception during error
- */
- public Cluster loadCluster() throws SSHApiException;
-
- /**
* This will copy the localFile to remoteFile location in configured cluster
*
* @param remoteFile remote file location, this can be a directory too
@@ -103,9 +94,13 @@ public interface Cluster {
/**
* This will get the job status of the the job associated with this jobId
+ *
* @param jobID jobId of the job user want to get the status
* @return job status of the given jobID
* @throws SSHApiException throws exception during error
*/
public JobStatus getJobStatus(String jobID) throws SSHApiException;
+
+
+
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java
index ce3b92d..d5a0837 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java
@@ -220,10 +220,22 @@ public class JobDescriptor {
this.getJobDescriptorDocument().getJobDescriptor().getPostJobCommands().addCommand(command);
}
- public void setJobSubmitter(String jobSubmitter){
- this.getJobDescriptorDocument().getJobDescriptor().setJobSubmitterCommand(jobSubmitter);
+ public void setPartition(String partition){
+ this.getJobDescriptorDocument().getJobDescriptor().setPartition(partition);
}
+ public void setUserName(String userName){
+ this.getJobDescriptorDocument().getJobDescriptor().setUserName(userName);
+ }
+ public void setNodeList(String nodeList){
+ this.getJobDescriptorDocument().getJobDescriptor().setNodeList(nodeList);
+ }
+ public void setJobSubmitter(String jobSubmitter){
+ this.getJobDescriptorDocument().getJobDescriptor().setJobSubmitterCommand(jobSubmitter);
+ }
+ public String getNodeList(){
+ return this.getJobDescriptorDocument().getJobDescriptor().getNodeList();
+ }
public String getExecutablePath() {
return this.getJobDescriptorDocument().getJobDescriptor().getExecutablePath();
}
@@ -351,5 +363,14 @@ public class JobDescriptor {
public String getJobSubmitterCommand(){
return this.getJobDescriptorDocument().getJobDescriptor().getJobSubmitterCommand();
}
+
+ public String getPartition(){
+ return this.getJobDescriptorDocument().getJobDescriptor().getPartition();
+ }
+
+ public String getUserName(){
+ return this.getJobDescriptorDocument().getJobDescriptor().getUserName();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobManagerConfiguration.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobManagerConfiguration.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobManagerConfiguration.java
new file mode 100644
index 0000000..c5d1585
--- /dev/null
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobManagerConfiguration.java
@@ -0,0 +1,38 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gsi.ssh.api.job;
+
+import org.apache.airavata.gsi.ssh.impl.RawCommandInfo;
+
+public interface JobManagerConfiguration {
+
+ public RawCommandInfo getCancelCommand(String jobID);
+
+ public String getJobDescriptionTemplateName();
+
+ public RawCommandInfo getMonitorCommand(String jobID);
+
+ public String getScriptExtension();
+
+ public RawCommandInfo getSubmitCommand(String workingDirectory,String pbsFilePath);
+
+ public OutputParser getParser();
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/OutputParser.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/OutputParser.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/OutputParser.java
new file mode 100644
index 0000000..a74746c
--- /dev/null
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/OutputParser.java
@@ -0,0 +1,58 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gsi.ssh.api.job;
+
+import org.apache.airavata.gsi.ssh.impl.JobStatus;
+
+import java.util.List;
+import java.util.Map;
+
+public interface OutputParser {
+
+ /**
+ * Tihs can be used to fill a jobdescriptor based on the output
+ * @param descriptor
+ * @return
+ */
+ public void parse(JobDescriptor descriptor,String rawOutput);
+
+ /**
+ * This can be used to parse the result of a job submission to get the JobID
+ * @param rawOutput
+ * @return
+ */
+ public String parse(String rawOutput);
+
+
+ /**
+ * This can be used to get the job status from the putput
+ * @param jobID
+ * @param rawOutput
+ */
+ public JobStatus parse(String jobID, String rawOutput);
+
+ /**
+ * This can be used to parse a big output and get multipleJob statuses
+ * @param statusMap list of status map will return and key will be the job ID
+ * @param rawOutput
+ */
+ public void parse(Map<String,JobStatus> statusMap, String rawOutput);
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/PBSJobConfiguration.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/PBSJobConfiguration.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/PBSJobConfiguration.java
new file mode 100644
index 0000000..9e07df5
--- /dev/null
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/PBSJobConfiguration.java
@@ -0,0 +1,93 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gsi.ssh.api.job;
+
+import org.apache.airavata.gsi.ssh.impl.RawCommandInfo;
+import org.apache.commons.io.FilenameUtils;
+
+import java.io.File;
+
+public class PBSJobConfiguration implements JobManagerConfiguration{
+
+ private String jobDescriptionTemplateName;
+
+ private String scriptExtension;
+
+ private String installedPath;
+
+ private OutputParser parser;
+
+ public PBSJobConfiguration(){
+ // this can be used to construct and use setter methods to set all the params in order
+ }
+ public PBSJobConfiguration(String jobDescriptionTemplateName,
+ String scriptExtension,String installedPath,OutputParser parser) {
+ this.jobDescriptionTemplateName = jobDescriptionTemplateName;
+ this.scriptExtension = scriptExtension;
+ this.parser = parser;
+ if (installedPath.endsWith("/")) {
+ this.installedPath = installedPath;
+ } else {
+ this.installedPath = installedPath + "/";
+ }
+ }
+
+ public RawCommandInfo getCancelCommand(String jobID) {
+ return new RawCommandInfo(this.installedPath + "qdel " + jobID);
+ }
+
+ public String getJobDescriptionTemplateName() {
+ return jobDescriptionTemplateName;
+ }
+
+ public void setJobDescriptionTemplateName(String jobDescriptionTemplateName) {
+ this.jobDescriptionTemplateName = jobDescriptionTemplateName;
+ }
+
+ public RawCommandInfo getMonitorCommand(String jobID) {
+ return new RawCommandInfo(this.installedPath + "qstat -f " + jobID);
+ }
+
+ public String getScriptExtension() {
+ return scriptExtension;
+ }
+
+ public RawCommandInfo getSubmitCommand(String workingDirectory,String pbsFilePath) {
+ return new RawCommandInfo(this.installedPath + "qsub " +
+ workingDirectory + File.separator + FilenameUtils.getName(pbsFilePath));
+ }
+
+ public String getInstalledPath() {
+ return installedPath;
+ }
+
+ public void setInstalledPath(String installedPath) {
+ this.installedPath = installedPath;
+ }
+
+ public OutputParser getParser() {
+ return parser;
+ }
+
+ public void setParser(OutputParser parser) {
+ this.parser = parser;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/PBSOutputParser.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/PBSOutputParser.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/PBSOutputParser.java
new file mode 100644
index 0000000..7c0a261
--- /dev/null
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/PBSOutputParser.java
@@ -0,0 +1,141 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gsi.ssh.api.job;
+
+import org.apache.airavata.gsi.ssh.impl.JobStatus;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.validation.constraints.Null;
+import java.util.Map;
+
+public class PBSOutputParser implements OutputParser {
+ private static final Logger log = LoggerFactory.getLogger(PBSOutputParser.class);
+
+ public void parse(JobDescriptor jobDescriptor, String rawOutput) {
+ String[] info = rawOutput.split("\n");
+ String[] line;
+ for (int i = 0; i < info.length; i++) {
+ if (info[i].contains("=")) {
+ line = info[i].split("=", 2);
+ } else {
+ line = info[i].split(":", 2);
+ }
+ if (line.length >= 2) {
+ String header = line[0].trim();
+ log.debug("Header = " + header);
+ String value = line[1].trim();
+ log.debug("value = " + value);
+
+ if (header.equals("Variable_List")) {
+ while (info[i + 1].startsWith("\t")) {
+ value += info[i + 1];
+ i++;
+ }
+ value = value.replaceAll("\t", "");
+ jobDescriptor.setVariableList(value);
+ } else if ("Job Id".equals(header)) {
+ jobDescriptor.setJobID(value);
+ } else if ("Job_Name".equals(header)) {
+ jobDescriptor.setJobName(value);
+ } else if ("Account_Name".equals(header)) {
+ jobDescriptor.setAcountString(value);
+ } else if ("job_state".equals(header)) {
+ jobDescriptor.setStatus(value);
+ } else if ("Job_Owner".equals(header)) {
+ jobDescriptor.setOwner(value);
+ } else if ("resources_used.cput".equals(header)) {
+ jobDescriptor.setUsedCPUTime(value);
+ } else if ("resources_used.mem".equals(header)) {
+ jobDescriptor.setUsedMemory(value);
+ } else if ("resources_used.walltime".equals(header)) {
+ jobDescriptor.setEllapsedTime(value);
+ } else if ("job_state".equals(header)) {
+ jobDescriptor.setStatus(value);
+ } else if ("queue".equals(header))
+ jobDescriptor.setQueueName(value);
+ else if ("ctime".equals(header)) {
+ jobDescriptor.setCTime(value);
+ } else if ("qtime".equals(header)) {
+ jobDescriptor.setQTime(value);
+ } else if ("mtime".equals(header)) {
+ jobDescriptor.setMTime(value);
+ } else if ("start_time".equals(header)) {
+ jobDescriptor.setSTime(value);
+ } else if ("comp_time".equals(header)) {
+ jobDescriptor.setCompTime(value);
+ } else if ("exec_host".equals(header)) {
+ jobDescriptor.setExecuteNode(value);
+ } else if ("Output_Path".equals(header)) {
+ if (info[i + 1].contains("=") || info[i + 1].contains(":"))
+ jobDescriptor.setStandardOutFile(value);
+ else {
+ jobDescriptor.setStandardOutFile(value + info[i + 1].trim());
+ i++;
+ }
+ } else if ("Error_Path".equals(header)) {
+ if (info[i + 1].contains("=") || info[i + 1].contains(":"))
+ jobDescriptor.setStandardErrorFile(value);
+ else {
+ String st = info[i + 1].trim();
+ jobDescriptor.setStandardErrorFile(value + st);
+ i++;
+ }
+
+ } else if ("submit_args".equals(header)) {
+ while (i + 1 < info.length) {
+ if (info[i + 1].startsWith("\t")) {
+ value += info[i + 1];
+ i++;
+ } else
+ break;
+ }
+ value = value.replaceAll("\t", "");
+ jobDescriptor.setSubmitArgs(value);
+ }
+ }
+ }
+ }
+
+ public String parse(String rawOutput) {
+ return rawOutput; //In PBS stdoutu is giong to be directly the jobID
+ }
+
+ public JobStatus parse(String jobID, String rawOutput) {
+ String[] info = rawOutput.split("\n");
+ String[] line = null;
+ for (String anInfo : info) {
+ if (anInfo.contains("=")) {
+ line = anInfo.split("=", 2);
+ if (line.length != 0) {
+ if (line[0].contains("job_state")) {
+ return JobStatus.valueOf(line[1].replaceAll(" ", ""));
+ }
+ }
+ }
+ }
+ return null;
+ }
+
+ public void parse(Map<String, JobStatus> statusMap, String rawOutput) {
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/SlurmJobConfiguration.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/SlurmJobConfiguration.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/SlurmJobConfiguration.java
new file mode 100644
index 0000000..e5a08b7
--- /dev/null
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/SlurmJobConfiguration.java
@@ -0,0 +1,93 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gsi.ssh.api.job;
+
+import org.apache.airavata.gsi.ssh.impl.RawCommandInfo;
+import org.apache.commons.io.FilenameUtils;
+
+import java.io.File;
+
+public class SlurmJobConfiguration implements JobManagerConfiguration{
+
+ private String jobDescriptionTemplateName;
+
+ private String scriptExtension;
+
+ private String installedPath;
+
+ private OutputParser parser;
+
+ public SlurmJobConfiguration(){
+ // this can be used to construct and use setter methods to set all the params in order
+ }
+ public SlurmJobConfiguration(String jobDescriptionTemplateName,
+ String scriptExtension,String installedPath,OutputParser parser) {
+ this.jobDescriptionTemplateName = jobDescriptionTemplateName;
+ this.scriptExtension = scriptExtension;
+ this.parser = parser;
+ if (installedPath.endsWith("/")) {
+ this.installedPath = installedPath;
+ } else {
+ this.installedPath = installedPath + "/";
+ }
+ }
+
+ public RawCommandInfo getCancelCommand(String jobID) {
+ return new RawCommandInfo(this.installedPath + "qcancel " + jobID);
+ }
+
+ public String getJobDescriptionTemplateName() {
+ return jobDescriptionTemplateName;
+ }
+
+ public void setJobDescriptionTemplateName(String jobDescriptionTemplateName) {
+ this.jobDescriptionTemplateName = jobDescriptionTemplateName;
+ }
+
+ public RawCommandInfo getMonitorCommand(String jobID) {
+ return new RawCommandInfo(this.installedPath + "squeue -v -j " + jobID);
+ }
+
+ public String getScriptExtension() {
+ return scriptExtension;
+ }
+
+ public RawCommandInfo getSubmitCommand(String workingDirectory,String pbsFilePath) {
+ return new RawCommandInfo(this.installedPath + "sbatch " +
+ workingDirectory + File.separator + FilenameUtils.getName(pbsFilePath));
+ }
+
+ public String getInstalledPath() {
+ return installedPath;
+ }
+
+ public void setInstalledPath(String installedPath) {
+ this.installedPath = installedPath;
+ }
+
+ public OutputParser getParser() {
+ return parser;
+ }
+
+ public void setParser(OutputParser parser) {
+ this.parser = parser;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b3581061/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/SlurmOutputParser.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/SlurmOutputParser.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/SlurmOutputParser.java
new file mode 100644
index 0000000..1948954
--- /dev/null
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/SlurmOutputParser.java
@@ -0,0 +1,106 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gsi.ssh.api.job;
+
+import org.apache.airavata.gsi.ssh.impl.JobStatus;
+
+import java.util.Map;
+
+public class SlurmOutputParser implements OutputParser {
+ public void parse(JobDescriptor descriptor, String rawOutput) {
+ System.out.println(rawOutput);
+ String[] info = rawOutput.split("\n");
+ String lastString = info[info.length -1];
+ if (lastString.contains("JOB ID")) {
+ // because there's no state
+ descriptor.setStatus("E");
+ }else{
+ int column = 0;
+ System.out.println(lastString);
+ for(String each:lastString.split(" ")){
+ if(each.trim().isEmpty()){
+ continue;
+ }else{
+ switch (column){
+ case 0:
+ descriptor.setJobID(each);
+ column++;
+ break;
+ case 1:
+ descriptor.setPartition(each);
+ column++;
+ break;
+ case 2:
+ descriptor.setJobName(each);
+ column++;
+ break;
+ case 3:
+ descriptor.setUserName(each);
+ column++;
+ break;
+ case 4:
+ descriptor.setStatus(each);
+ column++;
+ break;
+ case 5:
+ descriptor.setUsedCPUTime(each);
+ column++;
+ break;
+ case 6:
+ descriptor.setNodes(Integer.parseInt(each));
+ column++;
+ break;
+ case 7:
+ descriptor.setNodeList(each);
+ column++;
+ break;
+ }
+ }
+ }
+ }
+
+ }
+
+ /**
+ * This can be used to parse the outpu of sbatch and extrac the jobID from the content
+ *
+ * @param rawOutput
+ * @return
+ */
+ public String parse(String rawOutput) {
+ String[] info = rawOutput.split("\n");
+ for (String anInfo : info) {
+ if (anInfo.contains("Submitted batch job")) {
+ String[] split = anInfo.split("Submitted batch job");
+ return split[1].trim();
+ }
+ }
+ return null; //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+ public JobStatus parse(String jobID, String rawOutput) {
+ return null; //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+ public void parse(Map<String, JobStatus> statusMap, String rawOutput) {
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+}