You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sh...@apache.org on 2014/09/26 17:19:54 UTC
[08/15] git commit: Added job count validation enable/disable
capability
Added job count validation enable/disable capability
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/a78137b7
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/a78137b7
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/a78137b7
Branch: refs/heads/orchestratorJobThrottleFeature
Commit: a78137b789e4573e57fd614ea015fd18ca43ecbb
Parents: d899c1c
Author: shamrath <sh...@gmail.com>
Authored: Thu Sep 18 14:55:04 2014 -0400
Committer: shamrath <sh...@gmail.com>
Committed: Thu Sep 18 14:55:04 2014 -0400
----------------------------------------------------------------------
.../airavata/common/utils/ServerSettings.java | 5 ++++
.../main/resources/airavata-server.properties | 3 ++-
.../handlers/GridPullMonitorHandler.java | 4 ++-
.../monitor/impl/pull/qstat/HPCPullMonitor.java | 20 +++++++-------
.../core/validator/impl/JobCountValidator.java | 28 +++++++++++++++-----
5 files changed, 42 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/a78137b7/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
----------------------------------------------------------------------
diff --git a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
index e607cdc..6c7bb3b 100644
--- a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
+++ b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
@@ -71,6 +71,7 @@ public class ServerSettings extends ApplicationSettings {
private static final String MY_PROXY_LIFETIME = "myproxy.life";
private static final String ACTIVITY_LISTENERS = "activity.listeners";
+ private static final String ENABLE_JOB_RESTRICTION_VALIDATION = "enable.job.restriction.validation";
private static final String GLOBAL_MAX_JOB_COUNT = "global.max.job.count";
private static boolean stopAllThreads = false;
@@ -107,6 +108,10 @@ public class ServerSettings extends ApplicationSettings {
return getSetting(GLOBAL_MAX_JOB_COUNT);
}
+ public static String getEnableJobRestrictionValidation() {
+ return getSetting(ENABLE_JOB_RESTRICTION_VALIDATION, "false");
+ }
+
public static String getCredentialStoreDBUser() throws ApplicationSettingsException {
try {
return getSetting(CREDENTIAL_STORE_DB_USER);
http://git-wip-us.apache.org/repos/asf/airavata/blob/a78137b7/modules/configuration/server/src/main/resources/airavata-server.properties
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/airavata-server.properties b/modules/configuration/server/src/main/resources/airavata-server.properties
index cd4a2b4..42b1bc8 100644
--- a/modules/configuration/server/src/main/resources/airavata-server.properties
+++ b/modules/configuration/server/src/main/resources/airavata-server.properties
@@ -193,12 +193,13 @@ activity.listeners=org.apache.airavata.gfac.core.monitor.AiravataJobStatusUpdato
###---------------------------Orchestrator module Configurations---------------------------###
#job.submitter=org.apache.airavata.orchestrator.core.impl.GFACEmbeddedJobSubmitter
job.submitter=org.apache.airavata.orchestrator.core.impl.GFACServiceJobSubmitter
-job.validators=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator,org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator
+job.validators=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator,org.apache.airavata.orchestrator.core.validator.impl.JobCountValidator
submitter.interval=10000
threadpool.size=10
start.submitter=true
embedded.mode=true
enable.validation=true
+enable.job.restriction.validation=true
global.max.job.count=50
orchestrator=org.apache.airavata.orchestrator.server.OrchestratorServer
http://git-wip-us.apache.org/repos/asf/airavata/blob/a78137b7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java
index 6428112..451466d 100644
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java
+++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java
@@ -100,7 +100,9 @@ public class GridPullMonitorHandler extends ThreadedHandler implements Watcher{
e.printStackTrace();
}
CommonUtils.addMonitortoQueue(hpcPullMonitor.getQueue(), monitorID);
- CommonUtils.increaseZkJobCount(monitorID); // update change job count to zookeeper
+ if (ServerSettings.getEnableJobRestrictionValidation().equals("true")) {
+ CommonUtils.increaseZkJobCount(monitorID); // update change job count to zookeeper
+ }
} catch (AiravataMonitorException e) {
logger.error("Error adding monitorID object to the queue with experiment ", monitorID.getExperimentID());
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/a78137b7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
index b0fa11a..25a1ab2 100644
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
+++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
@@ -272,17 +272,19 @@ public class HPCPullMonitor extends PullMonitor {
ZooKeeper zk = null;
for (MonitorID completedJob : completedJobs) {
CommonUtils.removeMonitorFromQueue(queue, completedJob);
- if (zk == null) {
- zk = completedJob.getJobExecutionContext().getZk();
- }
- String key = CommonUtils.getJobCountUpdatePath(completedJob);
- int i = 0;
- if (jobRemoveCountMap.containsKey(key)) {
- i = Integer.valueOf(jobRemoveCountMap.get(key));
+ if (ServerSettings.getEnableJobRestrictionValidation().equals("true")) { // is job restriction available?
+ if (zk == null) {
+ zk = completedJob.getJobExecutionContext().getZk();
+ }
+ String key = CommonUtils.getJobCountUpdatePath(completedJob);
+ int i = 0;
+ if (jobRemoveCountMap.containsKey(key)) {
+ i = Integer.valueOf(jobRemoveCountMap.get(key));
+ }
+ jobRemoveCountMap.put(key, ++i);
}
- jobRemoveCountMap.put(key, ++i);
}
- if (completedJobs.size() > 0) {
+ if (ServerSettings.getEnableJobRestrictionValidation().equals("true") && completedJobs.size() > 0) {
// reduce completed job count from zookeeper
CommonUtils.updateZkWithJobCount(zk, jobRemoveCountMap, false);
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/a78137b7/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator.java
index 37890ec..2c66fa2 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator.java
@@ -1,3 +1,24 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+
package org.apache.airavata.orchestrator.core.validator.impl;
import org.airavata.appcatalog.cpi.AppCatalog;
@@ -10,11 +31,6 @@ import org.apache.airavata.common.utils.AiravataUtils;
import org.apache.airavata.common.utils.Constants;
import org.apache.airavata.common.utils.RequestData;
import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.credential.store.credential.AuditInfo;
-import org.apache.airavata.credential.store.store.CredentialReader;
-import org.apache.airavata.credential.store.store.CredentialReaderFactory;
-import org.apache.airavata.credential.store.store.CredentialStoreException;
-import org.apache.airavata.credential.store.util.TokenizedMyProxyAuthInfo;
import org.apache.airavata.gfac.core.scheduler.HostScheduler;
import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
import org.apache.airavata.model.appcatalog.appinterface.ApplicationInterfaceDescription;
@@ -28,12 +44,10 @@ import org.apache.airavata.model.workspace.experiment.WorkflowNodeDetails;
import org.apache.airavata.orchestrator.core.context.OrchestratorContext;
import org.apache.airavata.orchestrator.core.validator.JobMetadataValidator;
import org.apache.airavata.persistance.registry.jpa.model.TaskDetail;
-import org.apache.airavata.registry.cpi.RegistryException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;