[ambari] branch branch-2.6 updated: [AMBARI-23127] Allow default value of number of parallel tasks run during Upgrade to be customizable. (swagle) (#523)

[sw...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

swagle pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/ambari.git


The following commit(s) were added to refs/heads/branch-2.6 by this push:
     new 5bc23e6  [AMBARI-23127] Allow default value of number of parallel tasks run during Upgrade to be customizable. (swagle) (#523)
5bc23e6 is described below

commit 5bc23e6c54a30d9584b3dc927a078aeaef0059e5
Author: Siddharth <sw...@apache.org>
AuthorDate: Fri Mar 2 08:10:59 2018 -0800

    [AMBARI-23127] Allow default value of number of parallel tasks run during Upgrade to be customizable. (swagle) (#523)
---
 ambari-server/docs/configuration/index.md              | 18 ++++++++++++++++--
 .../ambari/server/configuration/Configuration.java     | 13 +++++++++++++
 .../org/apache/ambari/server/state/UpgradeContext.java | 14 ++++++++++++++
 .../ambari/server/state/stack/upgrade/Grouping.java    |  9 +++++++--
 .../server/state/stack/upgrade/ParallelScheduler.java  |  3 ++-
 pom.xml                                                |  2 +-
 6 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/ambari-server/docs/configuration/index.md b/ambari-server/docs/configuration/index.md
index e1a6899..743557d 100644
--- a/ambari-server/docs/configuration/index.md
+++ b/ambari-server/docs/configuration/index.md
@@ -114,9 +114,9 @@ The following are the properties which can be used to configure Ambari.
 | auto.group.creation | The auto group creation by Ambari |`false` | 
 | bootstrap.dir | The directory on the Ambari Server file system used for storing Ambari Agent bootstrap information such as request responses. |`/var/run/ambari-server/bootstrap` | 
 | bootstrap.master_host_name | The host name of the Ambari Server which will be used by the Ambari Agents for communication. | | 
-| bootstrap.script | The location and name of the Python script used to bootstrap new Ambari Agent hosts. |`/usr/lib/python2.6/site-packages/ambari_server/bootstrap.py` | 
+| bootstrap.script | The location and name of the Python script used to bootstrap new Ambari Agent hosts. |`/usr/lib/ambari-server/lib/ambari_server/bootstrap.py` | 
 | bootstrap.setup_agent.password | The password to set on the `AMBARI_PASSPHRASE` environment variable before invoking the bootstrap script. |`password` | 
-| bootstrap.setup_agent.script | The location and name of the Python script executed on the Ambari Agent host during the bootstrap process. |`/usr/lib/python2.6/site-packages/ambari_server/setupAgent.py` | 
+| bootstrap.setup_agent.script | The location and name of the Python script executed on the Ambari Agent host during the bootstrap process. |`/usr/lib/ambari-server/lib/ambari_server/setupAgent.py` | 
 | client.api.acceptor.count | Count of acceptors to configure for the jetty connector used for Ambari API. | | 
 | client.api.port | The port that client connections will use with the REST API. The Ambari Web client runs on this port. |`8080` | 
 | client.api.ssl.cert_pass_file | The filename which contains the password for the keystores, truststores, and certificates for the REST API when it's protected by SSL. |`https.pass.txt` | 
@@ -135,7 +135,11 @@ The following are the properties which can be used to configure Ambari.
 | db.oracle.jdbc.name | The name of the Oracle JDBC JAR connector. |`ojdbc6.jar` | 
 | default.kdcserver.port | The port used to communicate with the Kerberos Key Distribution Center. |`88` | 
 | extensions.path | The location on the Ambari Server where stack extensions exist.<br/><br/>The following are examples of valid values:<ul><li>`/var/lib/ambari-server/resources/extensions`</ul> | | 
+| gpl.license.accepted | Whether user accepted GPL license. |`false` | 
+| http.cache-control | The value that will be used to set the `Cache-Control` HTTP response header. |`no-store` | 
+| http.pragma | The value that will be used to set the `PRAGMA` HTTP response header. |`no-cache` | 
 | http.strict-transport-security | When using SSL, this will be used to set the `Strict-Transport-Security` response header. |`max-age=31536000` | 
+| http.x-content-type-options | The value that will be used to set the `X-CONTENT-TYPE` HTTP response header. |`nosniff` | 
 | http.x-frame-options | The value that will be used to set the `X-Frame-Options` HTTP response header. |`DENY` | 
 | http.x-xss-protection | The value that will be used to set the `X-XSS-Protection` HTTP response header. |`1; mode=block` | 
 | java.home | The location of the JDK on the Ambari Agent hosts.<br/><br/>The following are examples of valid values:<ul><li>`/usr/jdk64/jdk1.7.0_45`</ul> | | 
@@ -172,6 +176,7 @@ The following are the properties which can be used to configure Ambari.
 | recovery.window_in_minutes | The length of a recovery window, in minutes, in which recovery attempts can be retried.<br/><br/> This property is related to `recovery.max_count`. | | 
 | repo.validation.suffixes.default | The suffixes to use when validating most types of repositories. |`/repodata/repomd.xml` | 
 | repo.validation.suffixes.ubuntu | The suffixes to use when validating Ubuntu repositories. |`/dists/%s/Release` | 
+| repositories.legacy-override.enabled | This property is used in specific testing circumstances only. Its use otherwise will lead to very unpredictable results with repository management and package installation |`false` | 
 | resources.dir | The location on the Ambari Server where all resources exist, including common services, stacks, and scripts. |`/var/lib/ambari-server/resources/` | 
 | rolling.upgrade.skip.packages.prefixes | A comma-separated list of packages which will be skipped during a stack upgrade. | | 
 | security.agent.hostname.validate | Determines whether the Ambari Agent host names should be validated against a regular expression to ensure that they are well-formed.<br><br>WARNING: By setting this value to false, host names will not be validated, allowing a possible security vulnerability as described in CVE-2014-3582. See https://cwiki.apache.org/confluence/display/AMBARI/Ambari+Vulnerabilities for more information. |`true` | 
@@ -204,6 +209,7 @@ The following are the properties which can be used to configure Ambari.
 | server.ecCacheSize | The size of the cache which is used to hold current operations in memory until they complete. |`10000` | 
 | server.execution.scheduler.isClustered | Determines whether Quartz will use a clustered job scheduled when performing scheduled actions like rolling restarts. |`false` | 
 | server.execution.scheduler.maxDbConnections | The number of concurrent database connections that the Quartz job scheduler can use. |`5` | 
+| server.execution.scheduler.maxStatementsPerConnection | The maximum number of prepared statements cached per database connection. |`120` | 
 | server.execution.scheduler.maxThreads | The number of threads that the Quartz job scheduler will use when executing scheduled jobs. |`5` | 
 | server.execution.scheduler.misfire.toleration.minutes | The time, in minutes, that a scheduled job can be run after its missed scheduled execution time. |`480` | 
 | server.execution.scheduler.start.delay.seconds | The delay, in seconds, that a Quartz job must wait before it starts. |`120` | 
@@ -248,6 +254,9 @@ The following are the properties which can be used to configure Ambari.
 | server.property-provider.threadpool.size.core | The core number of threads that will be used to retrieve data from federated datasources, such as remote JMX endpoints. |`16` | 
 | server.property-provider.threadpool.size.max | The maximum number of threads that will be used to retrieve data from federated datasources, such as remote JMX endpoints. |`32` | 
 | server.property-provider.threadpool.worker.size | The maximum size of pending federated datasource requests, such as those to JMX endpoints, which can be queued before rejecting new requests. |`2147483647` | 
+| server.requestlogs.namepattern | The pattern of request log file name |`ambari-access-yyyy_mm_dd.log` | 
+| server.requestlogs.path | The location on the Ambari Server where request logs can be created. | | 
+| server.requestlogs.retaindays | The number of days that request log would be retained. |`15` | 
 | server.script.threads | The number of threads that should be allocated to run external script. |`20` | 
 | server.script.timeout | The time, in milliseconds, until an external script is killed. |`10000` | 
 | server.stage.command.execution_type | How to execute commands in one stage |`STAGE` | 
@@ -277,6 +286,7 @@ The following are the properties which can be used to configure Ambari.
 | stack.upgrade.auto.retry.command.names.to.ignore | A comma-separate list of upgrade tasks names to skip when retrying failed commands automatically. |`"ComponentVersionCheckAction","FinalizeUpgradeAction"` | 
 | stack.upgrade.auto.retry.timeout.mins | The amount of time to wait in order to retry a command during a stack upgrade when an agent loses communication. This value must be greater than the `agent.task.timeout` value. |`0` | 
 | stack.upgrade.bypass.prechecks | Determines whether pre-upgrade checks will be skipped when performing a rolling or express stack upgrade. |`false` | 
+| stack.upgrade.default.parallelism | Default value of max number of tasks to schedule in parallel for upgrades. Upgrade packs can override this value. |`100` | 
 | stackadvisor.script | The location and name of the Python stack advisor script executed when configuring services. |`/var/lib/ambari-server/resources/scripts/stack_advisor.py` | 
 | task.query.parameterlist.size | The maximum number of tasks which can be queried by ID from the database. |`999` | 
 | topology.task.creation.parallel | Indicates whether parallel topology task creation is enabled |`false` | 
@@ -290,7 +300,11 @@ The following are the properties which can be used to configure Ambari.
 | views.ambari.request.connect.timeout.millis | The amount of time, in milliseconds, that a view will wait when trying to connect on HTTP(S) operations to the Ambari REST API. |`30000` | 
 | views.ambari.request.read.timeout.millis | The amount of time, in milliseconds, that a view will wait before terminating an HTTP(S) read request to the Ambari REST API. |`45000` | 
 | views.dir | The directory on the Ambari Server file system used for expanding Views and storing webapp work. |`/var/lib/ambari-server/resources/views` | 
+| views.directory.watcher.disable | Determines whether the view directory watcher service should be disabled. |`false` | 
+| views.http.cache-control | The value that will be used to set the `Cache-Control` HTTP response header for Ambari View requests. |`no-store` | 
+| views.http.pragma | The value that will be used to set the `PRAGMA` HTTP response header for Ambari View requests. |`no-cache` | 
 | views.http.strict-transport-security | The value that will be used to set the `Strict-Transport-Security` HTTP response header for Ambari View requests. |`max-age=31536000` | 
+| views.http.x-content-type-options | The value that will be used to set the `X-CONTENT-TYPE` HTTP response header for Ambari View requests. |`nosniff` | 
 | views.http.x-frame-options | The value that will be used to set the `X-Frame-Options` HTTP response header for Ambari View requests. |`SAMEORIGIN` | 
 | views.http.x-xss-protection | The value that will be used to set the `X-XSS-Protection` HTTP response header for Ambari View requests. |`1; mode=block` | 
 | views.remove.undeployed | Determines whether remove undeployed views from the Ambari database. |`false` | 
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index f146c41..a7961bc 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -2596,6 +2596,12 @@ public class Configuration {
   // Ambari server log4j file name
   public static final String AMBARI_LOG_FILE = "log4j.properties";
 
+  /**
+   * Default value of Max number of tasks to schedule in parallel for upgrades.
+   */
+  @Markdown(description = "Default value of max number of tasks to schedule in parallel for upgrades. Upgrade packs can override this value.")
+  public static final ConfigurationProperty<Integer> DEFAULT_MAX_DEGREE_OF_PARALLELISM_FOR_UPGRADES = new ConfigurationProperty<>(
+    "stack.upgrade.default.parallelism", 100);
 
   /**
    * The number of tasks that can be queried from the database at once In the
@@ -5569,6 +5575,13 @@ public class Configuration {
   }
 
   /**
+   * @return default value of number of tasks to run in parallel during upgrades
+   */
+  public int getDefaultMaxParallelismForUpgrades() {
+    return Integer.parseInt(getProperty(DEFAULT_MAX_DEGREE_OF_PARALLELISM_FOR_UPGRADES));
+  }
+
+  /**
    * Generates a markdown table which includes:
    * <ul>
    * <li>Property key name</li>
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
index dbdc091..9368f80 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
@@ -47,6 +47,7 @@ import org.apache.ambari.server.ServiceNotFoundException;
 import org.apache.ambari.server.actionmanager.HostRoleCommandFactory;
 import org.apache.ambari.server.agent.ExecutionCommand.KeyNames;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.internal.AbstractControllerResourceProvider;
 import org.apache.ambari.server.controller.internal.PreUpgradeCheckResourceProvider;
 import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
@@ -262,6 +263,12 @@ public class UpgradeContext {
   private RepositoryType m_orchestration = RepositoryType.STANDARD;
 
   /**
+   * Used to lookup overridable settings like default task parallelism
+   */
+  @Inject
+  private Configuration configuration;
+
+  /**
    * Reading upgrade type from provided request  or if nothing were provided,
    * from previous upgrade for downgrade direction.
    *
@@ -922,6 +929,13 @@ public class UpgradeContext {
   }
 
   /**
+   * @return default value of number of tasks to run in parallel during upgrades
+   */
+  public int getDefaultMaxDegreeOfParallelism() {
+    return configuration.getDefaultMaxParallelismForUpgrades();
+  }
+
+  /**
    * Gets a POJO of the upgrade suitable to serialize.
    *
    * @return the upgrade summary as a POJO.
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
index 291891d..d7dfc0f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
@@ -225,8 +225,13 @@ public class Grouping {
       // Expand some of the TaskWrappers into multiple based on the batch size.
       for (TaskWrapper tw : tasks) {
         List<Set<String>> hostSets = null;
-        if (m_grouping.parallelScheduler != null && m_grouping.parallelScheduler.maxDegreeOfParallelism > 0) {
-          hostSets = SetUtils.split(tw.getHosts(), m_grouping.parallelScheduler.maxDegreeOfParallelism);
+
+        if (m_grouping.parallelScheduler != null) {
+          int taskParallelism = m_grouping.parallelScheduler.maxDegreeOfParallelism;
+          if (taskParallelism == Integer.MAX_VALUE) {
+            taskParallelism = ctx.getDefaultMaxDegreeOfParallelism();
+          }
+          hostSets = SetUtils.split(tw.getHosts(), taskParallelism);
         } else {
           hostSets = SetUtils.split(tw.getHosts(), 1);
         }
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ParallelScheduler.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ParallelScheduler.java
index 6004a12..bddc105 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ParallelScheduler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ParallelScheduler.java
@@ -24,7 +24,8 @@ import javax.xml.bind.annotation.XmlElement;
  */
 public class ParallelScheduler {
 
-  public static int DEFAULT_MAX_DEGREE_OF_PARALLELISM = 100;
+  // This setting can be overriden using ambari.properties file
+  public static int DEFAULT_MAX_DEGREE_OF_PARALLELISM = Integer.MAX_VALUE;
 
   @XmlElement(name="max-degree-of-parallelism")
   public int maxDegreeOfParallelism = DEFAULT_MAX_DEGREE_OF_PARALLELISM;
diff --git a/pom.xml b/pom.xml
index 9020b20..f1483b8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
   <url>http://ambari.apache.org/</url>
   <scm>
     <url>https://github.com/apache/ambari</url>
-    <connection>https://git-wip-us.apache.org/repos/asf/ambari.git</connection>
+    <connection>scm:git:https://git-wip-us.apache.org/repos/asf/ambari.git</connection>
   </scm>
   <licenses>
     <license>

-- 
To stop receiving notification emails like this one, please contact
swagle@apache.org.