airavata git commit: adding queueSpecific macro support

[sc...@apache.org]

Repository: airavata
Updated Branches:
  refs/heads/develop a9d277167 -> cd230e275


adding queueSpecific macro support


Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/cd230e27
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/cd230e27
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/cd230e27

Branch: refs/heads/develop
Commit: cd230e275fe69b0ff936324ef658f6eedc796a49
Parents: a9d2771
Author: scnakandala <su...@gmail.com>
Authored: Mon Jun 26 13:11:10 2017 -0400
Committer: scnakandala <su...@gmail.com>
Committed: Mon Jun 26 13:11:10 2017 -0400

----------------------------------------------------------------------
 .../src/main/resources/PBS_Groovy.template      |  1 +
 .../src/main/resources/SLURM_Groovy.template    |  1 +
 .../apache/airavata/gfac/core/GFacUtils.java    | 75 +++-----------------
 .../org/apache/airavata/gfac/core/Script.java   |  1 +
 .../gfac/core/context/ProcessContext.java       | 26 ++++---
 5 files changed, 29 insertions(+), 75 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/airavata/blob/cd230e27/modules/configuration/server/src/main/resources/PBS_Groovy.template
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/PBS_Groovy.template b/modules/configuration/server/src/main/resources/PBS_Groovy.template
index 97c49eb..8e3f277 100644
--- a/modules/configuration/server/src/main/resources/PBS_Groovy.template
+++ b/modules/configuration/server/src/main/resources/PBS_Groovy.template
@@ -14,6 +14,7 @@
    if (usedMem != null && usedMem != "") out.print '#PBS -l vmem=' + usedMem + 'M\n'
    if (nodes != null && nodes != "" && processPerNode != null && processPerNode != "")
         out.print '#PBS -l nodes=' + nodes + ':ppn=' + processPerNode + '\n'
+   if (queueSpecificMacros != null) for(queueMacro in queueSpecificMacros)  out.print queueMacro +'\n'
 %>
 #PBS -m abe
 

http://git-wip-us.apache.org/repos/asf/airavata/blob/cd230e27/modules/configuration/server/src/main/resources/SLURM_Groovy.template
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/SLURM_Groovy.template b/modules/configuration/server/src/main/resources/SLURM_Groovy.template
index bae646e..95596c7 100644
--- a/modules/configuration/server/src/main/resources/SLURM_Groovy.template
+++ b/modules/configuration/server/src/main/resources/SLURM_Groovy.template
@@ -13,6 +13,7 @@ if (queueName != null && queueName != "") out.print '#SBATCH -p ' + queueName +
    if (standardErrorFile != null && standardErrorFile != "") out.print '#SBATCH -e ' + standardErrorFile + '\n'
    if (qualityOfService != null && qualityOfService != "") out.print '#SBATCH --qos=' + qualityOfService + '\n'
    if (reservation != null && reservation != "") out.print '#SBATCH --reservation=' + reservation + '\n'
+   if (queueSpecificMacros != null) for(queueMacro in queueSpecificMacros)  out.print queueMacro +'\n'
 %>
 #SBATCH --mail-type=ALL
 

http://git-wip-us.apache.org/repos/asf/airavata/blob/cd230e27/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
index 358d6a1..1bb9e6f 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
@@ -23,12 +23,7 @@ import groovy.lang.Writable;
 import groovy.text.GStringTemplateEngine;
 import groovy.text.TemplateEngine;
 import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.AiravataUtils;
-import org.apache.airavata.common.utils.AiravataZKUtils;
-import org.apache.airavata.common.utils.ApplicationSettings;
-import org.apache.airavata.common.utils.DBUtil;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.common.utils.ZkConstants;
+import org.apache.airavata.common.utils.*;
 import org.apache.airavata.credential.store.store.CredentialReader;
 import org.apache.airavata.credential.store.store.impl.CredentialReaderImpl;
 import org.apache.airavata.gfac.core.context.ProcessContext;
@@ -36,54 +31,23 @@ import org.apache.airavata.gfac.core.context.TaskContext;
 import org.apache.airavata.messaging.core.MessageContext;
 import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
 import org.apache.airavata.model.appcatalog.appdeployment.CommandObject;
-import org.apache.airavata.model.appcatalog.computeresource.CloudJobSubmission;
-import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
-import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionInterface;
-import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
-import org.apache.airavata.model.appcatalog.computeresource.LOCALSubmission;
-import org.apache.airavata.model.appcatalog.computeresource.MonitorMode;
-import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManager;
-import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManagerType;
-import org.apache.airavata.model.appcatalog.computeresource.SSHJobSubmission;
-import org.apache.airavata.model.appcatalog.computeresource.UnicoreJobSubmission;
+import org.apache.airavata.model.appcatalog.computeresource.*;
 import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference;
 import org.apache.airavata.model.application.io.DataType;
 import org.apache.airavata.model.application.io.InputDataObjectType;
 import org.apache.airavata.model.application.io.OutputDataObjectType;
 import org.apache.airavata.model.commons.ErrorModel;
-import org.apache.airavata.model.data.replica.DataProductModel;
-import org.apache.airavata.model.data.replica.DataProductType;
-import org.apache.airavata.model.data.replica.DataReplicaLocationModel;
-import org.apache.airavata.model.data.replica.ReplicaLocationCategory;
-import org.apache.airavata.model.data.replica.ReplicaPersistentType;
+import org.apache.airavata.model.data.replica.*;
 import org.apache.airavata.model.experiment.ExperimentModel;
 import org.apache.airavata.model.job.JobModel;
-import org.apache.airavata.model.messaging.event.JobIdentifier;
-import org.apache.airavata.model.messaging.event.JobStatusChangeEvent;
-import org.apache.airavata.model.messaging.event.MessageType;
-import org.apache.airavata.model.messaging.event.ProcessIdentifier;
-import org.apache.airavata.model.messaging.event.ProcessStatusChangeEvent;
-import org.apache.airavata.model.messaging.event.TaskIdentifier;
-import org.apache.airavata.model.messaging.event.TaskStatusChangeEvent;
+import org.apache.airavata.model.messaging.event.*;
 import org.apache.airavata.model.parallelism.ApplicationParallelismType;
 import org.apache.airavata.model.process.ProcessModel;
 import org.apache.airavata.model.scheduling.ComputationalResourceSchedulingModel;
-import org.apache.airavata.model.status.JobStatus;
-import org.apache.airavata.model.status.ProcessState;
-import org.apache.airavata.model.status.ProcessStatus;
-import org.apache.airavata.model.status.TaskState;
-import org.apache.airavata.model.status.TaskStatus;
+import org.apache.airavata.model.status.*;
 import org.apache.airavata.model.task.JobSubmissionTaskModel;
 import org.apache.airavata.registry.core.experiment.catalog.impl.RegistryFactory;
-import org.apache.airavata.registry.cpi.AppCatalog;
-import org.apache.airavata.registry.cpi.AppCatalogException;
-import org.apache.airavata.registry.cpi.CompositeIdentifier;
-import org.apache.airavata.registry.cpi.ExpCatChildDataType;
-import org.apache.airavata.registry.cpi.ExperimentCatalog;
-import org.apache.airavata.registry.cpi.ExperimentCatalogModelType;
-import org.apache.airavata.registry.cpi.GwyResourceProfile;
-import org.apache.airavata.registry.cpi.RegistryException;
-import org.apache.airavata.registry.cpi.ReplicaCatalog;
+import org.apache.airavata.registry.cpi.*;
 import org.apache.airavata.registry.cpi.utils.Constants;
 import org.apache.commons.io.FileUtils;
 import org.apache.curator.framework.CuratorFramework;
@@ -100,35 +64,15 @@ import org.w3c.dom.Element;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 
-import javax.xml.xpath.XPath;
-import javax.xml.xpath.XPathConstants;
-import javax.xml.xpath.XPathExpression;
-import javax.xml.xpath.XPathExpressionException;
-import javax.xml.xpath.XPathFactory;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStream;
+import javax.xml.xpath.*;
+import java.io.*;
 import java.net.InetAddress;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
 import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Calendar;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.TreeSet;
+import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -501,6 +445,7 @@ public class GFacUtils {
         groovyMap.add(Script.GATEWAY_ID, processContext.getGatewayId());
         groovyMap.add(Script.GATEWAY_USER_NAME, processContext.getProcessModel().getUserName());
         groovyMap.add(Script.APPLICATION_NAME, processContext.getApplicationInterfaceDescription().getApplicationName());
+        groovyMap.add(Script.QUEUE_SPECIFIC_MACROS, processContext.getQueueSpecificMacros());
 
         groovyMap.add(Script.ACCOUNT_STRING, processContext.getAllocationProjectNumber());
         groovyMap.add(Script.RESERVATION, processContext.getReservation());

http://git-wip-us.apache.org/repos/asf/airavata/blob/cd230e27/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java
index 3ac5797..0937c7e 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java
@@ -70,6 +70,7 @@ public enum Script {
     GATEWAY_ID("gatewayId"),
     GATEWAY_USER_NAME("gatewayUserName"),
     APPLICATION_NAME("applicationName"),
+    QUEUE_SPECIFIC_MACROS("queueSpecificMacros")
     ;
 
     String name;

http://git-wip-us.apache.org/repos/asf/airavata/blob/cd230e27/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
index 5dd06db..6ca29e3 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
@@ -27,10 +27,7 @@ import org.apache.airavata.gfac.core.cluster.ServerInfo;
 import org.apache.airavata.messaging.core.Publisher;
 import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
 import org.apache.airavata.model.appcatalog.appinterface.ApplicationInterfaceDescription;
-import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
-import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
-import org.apache.airavata.model.appcatalog.computeresource.MonitorMode;
-import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManager;
+import org.apache.airavata.model.appcatalog.computeresource.*;
 import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference;
 import org.apache.airavata.model.appcatalog.gatewayprofile.GatewayResourceProfile;
 import org.apache.airavata.model.appcatalog.gatewayprofile.StoragePreference;
@@ -51,11 +48,7 @@ import org.apache.curator.framework.CuratorFramework;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 public class ProcessContext {
 
@@ -108,6 +101,7 @@ public class ProcessContext {
 	private SSHKeyAuthentication sshKeyAuthentication;
 	private boolean recoveryWithCancel = false;
 	private String usageReportingGatewayId;
+	private List<String> queueSpecificMacros;
 
 	/**
 	 * Note: process context property use lazy loading approach. In runtime you will see some properties as null
@@ -696,7 +690,19 @@ public class ProcessContext {
 		}
 	}
 
-	public static class ProcessContextBuilder{
+    public List<String> getQueueSpecificMacros() {
+		String queueName = getProcessCRSchedule().getQueueName();
+		Optional<BatchQueue> queue = getComputeResourceDescription().getBatchQueues().stream()
+				.filter(x->x.getQueueName().equals(queueName)).findFirst();
+		if(queue.isPresent()){
+			if(queue.get().getQueueSpecificMacros() != null && !queue.get().getQueueSpecificMacros().equals("")){
+				return Arrays.asList(queue.get().getQueueSpecificMacros().split(","));
+			}
+		}
+        return null;
+    }
+
+    public static class ProcessContextBuilder{
 		private final String processId;
 		private final String gatewayId;
 		private final String tokenId;