You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2016/12/01 08:14:30 UTC
[47/50] ambari git commit: AMBARI-18755. Deployment failing at
creating principal [addendum] (rlevas)
AMBARI-18755. Deployment failing at creating principal [addendum] (rlevas)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a5ce8230
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a5ce8230
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a5ce8230
Branch: refs/heads/branch-feature-AMBARI-18901
Commit: a5ce8230e59d515e6bcbe0db949ef6bfcd447c7a
Parents: 707f07f
Author: Robert Levas <rl...@hortonworks.com>
Authored: Wed Nov 30 19:57:28 2016 -0500
Committer: Robert Levas <rl...@hortonworks.com>
Committed: Wed Nov 30 19:57:28 2016 -0500
----------------------------------------------------------------------
ambari-server/docs/configuration/index.md | 25 +++++++++++++-------
.../server/configuration/Configuration.java | 7 ++++++
.../kerberos/MITKerberosOperationHandler.java | 22 ++++++++++++-----
.../ambari/server/utils/ShellCommandUtil.java | 11 ++++++++-
.../MITKerberosOperationHandlerTest.java | 23 ++++++++++++++++++
.../server/utils/TestShellCommandUtil.java | 13 ++++++----
6 files changed, 81 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5ce8230/ambari-server/docs/configuration/index.md
----------------------------------------------------------------------
diff --git a/ambari-server/docs/configuration/index.md b/ambari-server/docs/configuration/index.md
index 77d3a4d..34dafc2 100644
--- a/ambari-server/docs/configuration/index.md
+++ b/ambari-server/docs/configuration/index.md
@@ -49,15 +49,17 @@ The following are the properties which can be used to configure Ambari.
| agent.stack.retry.tries | The number of times an Ambari Agent should retry package installation when it fails due to a repository error. <br/><br/> This property is related to `agent.stack.retry.on_repo_unavailability`. |`5` |
| agent.task.timeout | The time, in seconds, before agent commands are killed. This does not include package installation commands. |`900` |
| agent.threadpool.size.max | The size of the Jetty connection pool used for handling incoming Ambari Agent requests. |`25` |
+| alerts.ambari.snmp.dispatcher.udp.port | The UDP port to use when binding the Ambari SNMP dispatcher on Ambari Server startup. If no port is specified, then a random port will be used. | |
| alerts.cache.enabled | Determines whether current alerts should be cached. Enabling this can increase performance on large cluster, but can also result in lost alert data if the cache is not flushed frequently. |`false` |
| alerts.cache.flush.interval | The time, in minutes, after which cached alert information is flushed to the database<br/><br/> This property is related to `alerts.cache.enabled`. |`10` |
| alerts.cache.size | The size of the alert cache.<br/><br/> This property is related to `alerts.cache.enabled`. |`50000` |
| alerts.execution.scheduler.maxThreads | The number of threads used to handle alerts received from the Ambari Agents. The value should be increased as the size of the cluster increases. |`2` |
| alerts.snmp.dispatcher.udp.port | The UDP port to use when binding the SNMP dispatcher on Ambari Server startup. If no port is specified, then a random port will be used. | |
-| alerts.ambari.snmp.dispatcher.udp.port | The UDP port to use when binding the SNMP dispatcher on Ambari Server startup. If no port is specified, then a random port will be used. | |
-| alerts.template.file | The full path to the XML file that describes the different alert templates. | |
+| alerts.template.file | The full path to the XML file that describes the different alert templates. | |
| ambari.display.url | The URL to use when creating messages which should include the Ambari Server URL.<br/><br/>The following are examples of valid values:<ul><li>`http://ambari.apache.org:8080`</ul> | |
| ambari.ldap.isConfigured | An internal property used for unit testing and development purposes. |`false` |
+| ambari.post.user.creation.hook | The location of the post user creation hook on the ambari server hosting machine. |`/var/lib/ambari-server/resources/scripts/post-user-creation-hook.sh` |
+| ambari.post.user.creation.hook.enabled | Indicates whether the post user creation is enabled or not. By default is false. |`false` |
| ambari.python.wrap | The name of the shell script used to wrap all invocations of Python by Ambari. |`ambari-python-wrap` |
| anonymous.audit.name | The name of the user given to requests which are executed without any credentials. |`_anonymous` |
| api.authenticated.user | The username of the default user assumed to be executing API calls. When set, authentication is not required in order to login to Ambari or use the REST APIs. | |
@@ -105,6 +107,7 @@ The following are the properties which can be used to configure Ambari.
| authentication.ldap.usernameAttribute | The attribute used for determining the user name, such as `uid`. |`uid` |
| authorization.ldap.adminGroupMappingRules | A comma-separate list of groups which would give a user administrative access to Ambari when syncing from LDAP. This is only used when `authorization.ldap.groupSearchFilter` is blank.<br/><br/>The following are examples of valid values:<ul><li>`administrators`<li>`Hadoop Admins,Hadoop Admins.*,DC Admins,.*Hadoop Operators`</ul> |`Ambari Administrators` |
| authorization.ldap.groupSearchFilter | The DN to use when searching for LDAP groups. | |
+| auto.group.creation | The auto group creation by Ambari |`false` |
| bootstrap.dir | The directory on the Ambari Server file system used for storing Ambari Agent bootstrap information such as request responses. |`/var/run/ambari-server/bootstrap` |
| bootstrap.master_host_name | The host name of the Ambari Server which will be used by the Ambari Agents for communication. | |
| bootstrap.script | The location and name of the Python script used to bootstrap new Ambari Agent hosts. |`/usr/lib/python2.6/site-packages/ambari_server/bootstrap.py` |
@@ -138,6 +141,7 @@ The following are the properties which can be used to configure Ambari.
| kerberos.check.jaas.configuration | Determines whether Kerberos-enabled Ambari deployments should use JAAS to validate login credentials. |`false` |
| kerberos.keytab.cache.dir | The location on the Ambari Server where Kerberos keytabs are cached. |`/var/lib/ambari-server/data/cache` |
| kerberos.operation.retries | The number of times failed kerberos operations should be retried to execute. |`3` |
+| kerberos.operation.retry.timeout | The time to wait (in seconds) between failed kerberos operations retries. |`10` |
| ldap.sync.username.collision.behavior | Determines how to handle username collision while updating from LDAP.<br/><br/>The following are examples of valid values:<ul><li>`skip`<li>`convert`</ul> |`convert` |
| log4j.monitor.delay | Indicates the delay, in milliseconds, for the log4j monitor to check for changes |`300000` |
| metadata.path | The location on the Ambari Server where the stack resources exist.<br/><br/>The following are examples of valid values:<ul><li>`/var/lib/ambari-server/resources/stacks`</ul> | |
@@ -146,8 +150,10 @@ The following are the properties which can be used to configure Ambari.
| metrics.retrieval-service.request.ttl.enabled | Enables throttling requests to the same endpoint within a fixed amount of time. This property will prevent Ambari from making new metric requests to update the cache for URLs which have been recently retrieved.<br/><br/> This property is related to `metrics.retrieval-service.request.ttl`. |`true` |
| mpacks.staging.path | The Ambari Management Pack staging directory on the Ambari Server.<br/><br/>The following are examples of valid values:<ul><li>`/var/lib/ambari-server/resources/mpacks`</ul> | |
| packages.pre.installed | Determines whether Ambari Agent instances have already have the necessary stack software installed |`false` |
+| pam.configuration | The PAM configuration file. | |
| proxy.allowed.hostports | A comma-separated whitelist of host and port values which Ambari Server can use to determine if a proxy value is valid. |`*:*` |
| recommendations.artifacts.lifetime | The amount of time that Recommendation API data is kept on the Ambari Server file system. This is specified using a `hdwmy` syntax for pairing the value with a time unit (hours, days, weeks, months, years)<br/><br/>The following are examples of valid values:<ul><li>`8h`<li>`2w`<li>`1m`</ul> |`1w` |
+| recommendations.artifacts.rollover.max | Maximum number of recommendations artifacts at a given time<br/><br/>The following are examples of valid values:<ul><li>`50`<li>`10`<li>`100`</ul> |`100` |
| recommendations.dir | The directory on the Ambari Server file system used for storing Recommendation API artifacts. |`/var/run/ambari-server/stack-recommendations` |
| recovery.disabled_components | A comma-separated list of component names which are not included in automatic recovery attempts.<br/><br/>The following are examples of valid values:<ul><li>`NAMENODE,ZOOKEEPER_SERVER`</ul> | |
| recovery.enabled_components | A comma-separated list of component names which are included in automatic recovery attempts.<br/><br/>The following are examples of valid values:<ul><li>`NAMENODE,ZOOKEEPER_SERVER`</ul> | |
@@ -222,16 +228,16 @@ The following are the properties which can be used to configure Ambari.
| server.jdbc.user.passwd | The password for the user when logging into the database. |`bigdata` |
| server.locks.profiling | Enable the profiling of internal locks. |`false` |
| server.metrics.retrieval-service.thread.priority | The priority of threads used by the service which retrieves JMX and REST metrics directly from their respective endpoints. |`5` |
-| server.metrics.retrieval-service.threadpool.size.core | The core number of threads used to retrieve JMX and REST metrics directly from their respective endpoints. |`16` |
-| server.metrics.retrieval-service.threadpool.size.max | The maximum number of threads used to retrieve JMX and REST metrics directly from their respective endpoints. |`32` |
-| server.metrics.retrieval-service.threadpool.worker.size | The number of queued requests allowed for JMX and REST metrics before discarding old requests which have not been fullfilled. |`320` |
+| server.metrics.retrieval-service.threadpool.size.core | The core number of threads used to retrieve JMX and REST metrics directly from their respective endpoints. |`8` |
+| server.metrics.retrieval-service.threadpool.size.max | The maximum number of threads used to retrieve JMX and REST metrics directly from their respective endpoints. |`16` |
+| server.metrics.retrieval-service.threadpool.worker.size | The number of queued requests allowed for JMX and REST metrics before discarding old requests which have not been fullfilled. |`160` |
| server.operations.retry-attempts | The number of retry attempts for failed API and blueprint operations. |`0` |
| server.os_family | The operating system family for all hosts in the cluster. This is used when bootstrapping agents and when enabling Kerberos.<br/><br/>The following are examples of valid values:<ul><li>`redhat`<li>`ubuntu`</ul> | |
| server.os_type | The operating system version for all hosts in the cluster. This is used when bootstrapping agents and when enabling Kerberos.<br/><br/>The following are examples of valid values:<ul><li>`6`<li>`7`</ul> | |
| server.persistence.type | The type of database connection being used. Unless using an embedded PostgresSQL server, then this should be `remote`.<br/><br/>The following are examples of valid values:<ul><li>`local`<li>`remote`</ul> |`local` |
| server.property-provider.threadpool.completion.timeout | The maximum time, in milliseconds, that federated requests for data can execute before being terminated. Increasing this value could result in degraded performanc from the REST APIs. |`5000` |
-| server.property-provider.threadpool.size.core | The core number of threads that will be used to retrieve data from federated datasources, such as remote JMX endpoints. |`16` |
-| server.property-provider.threadpool.size.max | The maximum number of threads that will be used to retrieve data from federated datasources, such as remote JMX endpoints. |`32` |
+| server.property-provider.threadpool.size.core | The core number of threads that will be used to retrieve data from federated datasources, such as remote JMX endpoints. |`8` |
+| server.property-provider.threadpool.size.max | The maximum number of threads that will be used to retrieve data from federated datasources, such as remote JMX endpoints. |`16` |
| server.property-provider.threadpool.worker.size | The maximum size of pending federated datasource requests, such as those to JMX endpoints, which can be queued before rejecting new requests. |`2147483647` |
| server.script.timeout | The time, in milliseconds, until an external script is killed. |`5000` |
| server.stage.command.execution_type | How to execute commands in one stage |`STAGE` |
@@ -262,6 +268,9 @@ The following are the properties which can be used to configure Ambari.
| stack.upgrade.bypass.prechecks | Determines whether pre-upgrade checks will be skipped when performing a rolling or express stack upgrade. |`false` |
| stackadvisor.script | The location and name of the Python stack advisor script executed when configuring services. |`/var/lib/ambari-server/resources/scripts/stack_advisor.py` |
| task.query.parameterlist.size | The maximum number of tasks which can be queried by ID from the database. |`999` |
+| topology.task.creation.parallel | Indicates whether parallel topology task creation is enabled |`false` |
+| topology.task.creation.parallel.threads | The number of threads to use for parallel topology task creation if enabled |`10` |
+| view.extract-after-cluster-config | Drives view extraction in case of blueprint deployments; non-system views are deployed when cluster configuration is successful |`false` |
| view.extraction.threadpool.size.core | The number of threads used to extract Ambari Views when Ambari Server is starting up. |`10` |
| view.extraction.threadpool.size.max | The maximum number of threads used to extract Ambari Views when Ambari Server is starting up. |`20` |
| view.extraction.threadpool.timeout | The time, in milliseconds, that non-core threads will live when extraction views on Ambari Server startup. |`100000` |
@@ -316,4 +325,4 @@ EclipseLink properties can also be configured using a prefix of `server.persiste
```
server.persistence.properties.eclipselink.jdbc.batch-writing.size=25
server.persistence.properties.eclipselink.profiler=QueryMonitor
-```
+```
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5ce8230/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index ef35ce8..8f10023 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -1407,6 +1407,9 @@ public class Configuration {
public static final ConfigurationProperty<Integer> KERBEROS_OPERATION_RETRIES = new ConfigurationProperty<>(
"kerberos.operation.retries", 3);
+ @Markdown(description = "The time to wait (in seconds) between failed kerberos operations retries.")
+ public static final ConfigurationProperty<Integer> KERBEROS_OPERATION_RETRY_TIMEOUT = new ConfigurationProperty<>(
+ "kerberos.operation.retry.timeout", 10);
/**
* The type of connection pool to use with JDBC connections to the database.
*/
@@ -5611,6 +5614,10 @@ public class Configuration {
return Integer.valueOf(getProperty(KERBEROS_OPERATION_RETRIES));
}
+ public int getKerberosOperationRetryTimeout() {
+ return Integer.valueOf(getProperty(KERBEROS_OPERATION_RETRY_TIMEOUT));
+ }
+
/**
* Return configured acceptors for agent api connector. Default = null
*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5ce8230/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandler.java
index 2f756fb..4b20973 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandler.java
@@ -473,9 +473,11 @@ public class MITKerberosOperationHandler extends KerberosOperationHandler {
}
tries++;
- try { Thread.sleep(3000); } catch (InterruptedException e) {}
+ try { Thread.sleep(1000 * configuration.getKerberosOperationRetryTimeout()); } catch (InterruptedException e) {}
- String message = String.format("Retrying to execute kadmin after a wait of 3 seconds :\n\tCommand: %s", command);
+ String message = String.format("Retrying to execute kadmin after a wait of %d seconds :\n\tCommand: %s",
+ configuration.getKerberosOperationRetryTimeout(),
+ command);
LOG.warn(message);
}
@@ -521,8 +523,8 @@ public class MITKerberosOperationHandler extends KerberosOperationHandler {
/**
* The queue of responses to return
*/
- private final Queue<String> responses = new LinkedList<String>();
-
+ private LinkedList<String> responses;
+ private Queue<String> currentResponses;
/**
* Constructor.
@@ -531,6 +533,7 @@ public class MITKerberosOperationHandler extends KerberosOperationHandler {
* @param userPassword the user's password (optional)
*/
public InteractivePasswordHandler(String adminPassword, String userPassword) {
+ responses = new LinkedList<String>();
if (adminPassword != null) {
responses.offer(adminPassword);
@@ -540,16 +543,23 @@ public class MITKerberosOperationHandler extends KerberosOperationHandler {
responses.offer(userPassword);
responses.offer(userPassword); // Add a 2nd time for the password "confirmation" request
}
+
+ currentResponses = new LinkedList<String>(responses);
}
@Override
public boolean done() {
- return responses.size() == 0;
+ return currentResponses.size() == 0;
}
@Override
public String getResponse(String query) {
- return responses.poll();
+ return currentResponses.poll();
+ }
+
+ @Override
+ public void start() {
+ currentResponses = new LinkedList<String>(responses);
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5ce8230/ambari-server/src/main/java/org/apache/ambari/server/utils/ShellCommandUtil.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/ShellCommandUtil.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/ShellCommandUtil.java
index 99f47c5..3152a0c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/utils/ShellCommandUtil.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/ShellCommandUtil.java
@@ -434,6 +434,8 @@ public class ShellCommandUtil {
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(process.getOutputStream()));
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
+ interactiveHandler.start();
+
while (!interactiveHandler.done()) {
StringBuilder query = new StringBuilder();
@@ -546,11 +548,18 @@ public class ShellCommandUtil {
boolean done();
/**
- * Gnven a query, returns the relative response to send to the shell command (via stdin)
+ * Given a query, returns the relative response to send to the shell command (via stdin)
*
* @param query a string containing the query that needs a response
* @return a string or null if no response is needed
*/
String getResponse(String query);
+
+ /**
+ * Starts or resets this handler.
+ * <p>
+ * It is expected that the caller calls this before using handler.
+ */
+ void start();
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5ce8230/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandlerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandlerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandlerTest.java
index 4c40a5d..39f4201 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandlerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandlerTest.java
@@ -70,6 +70,7 @@ public class MITKerberosOperationHandlerTest extends KerberosOperationHandlerTes
protected void configure() {
Configuration configuration = EasyMock.createNiceMock(Configuration.class);
expect(configuration.getServerOsFamily()).andReturn("redhat6").anyTimes();
+ expect(configuration.getKerberosOperationRetryTimeout()).andReturn(1).anyTimes();
replay(configuration);
bind(Clusters.class).toInstance(EasyMock.createNiceMock(Clusters.class));
@@ -547,6 +548,28 @@ public class MITKerberosOperationHandlerTest extends KerberosOperationHandlerTes
handler.close();
}
+ @Test
+ public void testInteractivePasswordHandler() {
+ MITKerberosOperationHandler.InteractivePasswordHandler handler = new MITKerberosOperationHandler.InteractivePasswordHandler("admin_password", "user_password");
+
+ handler.start();
+ Assert.assertEquals("admin_password", handler.getResponse("password"));
+ Assert.assertFalse(handler.done());
+ Assert.assertEquals("user_password", handler.getResponse("password"));
+ Assert.assertFalse(handler.done());
+ Assert.assertEquals("user_password", handler.getResponse("password"));
+ Assert.assertTrue(handler.done());
+
+ // Test restarting
+ handler.start();
+ Assert.assertEquals("admin_password", handler.getResponse("password"));
+ Assert.assertFalse(handler.done());
+ Assert.assertEquals("user_password", handler.getResponse("password"));
+ Assert.assertFalse(handler.done());
+ Assert.assertEquals("user_password", handler.getResponse("password"));
+ Assert.assertTrue(handler.done());
+ }
+
private MITKerberosOperationHandler createMock(){
return createMock(false);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5ce8230/ambari-server/src/test/java/org/apache/ambari/server/utils/TestShellCommandUtil.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/utils/TestShellCommandUtil.java b/ambari-server/src/test/java/org/apache/ambari/server/utils/TestShellCommandUtil.java
index 34ac91f..24af9bd 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/utils/TestShellCommandUtil.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/utils/TestShellCommandUtil.java
@@ -126,17 +126,20 @@ public class TestShellCommandUtil {
@Override
public String getResponse(String query) {
- if(query.contains("Arg1")) {
+ if (query.contains("Arg1")) {
return "a1";
- }
- else if(query.contains("Arg2")) {
+ } else if (query.contains("Arg2")) {
done = true; // this is the last expected prompt
return "a2";
- }
- else {
+ } else {
return null;
}
}
+
+ @Override
+ public void start() {
+
+ }
};
ShellCommandUtil.Result result = ShellCommandUtil.runCommand(new String[]{"./src/test/resources/interactive_shell_test.sh"}, null, interactiveHandler, false);