You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/02/03 14:53:24 UTC
[20/50] [abbrv] ambari git commit: AMBARI-19806: After setting up
hadoop credential, cannot start Hive Metastore
AMBARI-19806: After setting up hadoop credential, cannot start Hive Metastore
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d340fe93
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d340fe93
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d340fe93
Branch: refs/heads/branch-dev-patch-upgrade
Commit: d340fe93b51b82dc380ad7783cd63b7e76b72224
Parents: f36af3d
Author: Nahappan Somasundaram <ns...@hortonworks.com>
Authored: Tue Jan 31 15:34:38 2017 -0800
Committer: Nahappan Somasundaram <ns...@hortonworks.com>
Committed: Wed Feb 1 15:09:44 2017 -0800
----------------------------------------------------------------------
.../ambari_agent/CustomServiceOrchestrator.py | 4 +
ambari-server/pom.xml | 14 +
.../server/credentialapi/CredentialUtil.java | 580 +++++++++++++++++
.../0.12.0.2.0/package/scripts/params_linux.py | 54 +-
.../credentialapi/CredentialUtilTest.java | 644 +++++++++++++++++++
5 files changed, 1292 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d340fe93/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
index 8f1848c..9baaf08 100644
--- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
+++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
@@ -75,6 +75,9 @@ class CustomServiceOrchestrator():
# The property name used by the hadoop credential provider
CREDENTIAL_PROVIDER_PROPERTY_NAME = 'hadoop.security.credential.provider.path'
+ # Property name for credential store class path
+ CREDENTIAL_STORE_CLASS_PATH_NAME = 'credentialStoreClassPath'
+
def __init__(self, config, controller):
self.config = config
self.tmp_dir = config.get('agent', 'prefix')
@@ -286,6 +289,7 @@ class CustomServiceOrchestrator():
os.chmod(file_path, 0644) # group and others should have read access so that the service user can read
# Add JCEKS provider path instead
config[self.CREDENTIAL_PROVIDER_PROPERTY_NAME] = provider_path
+ config[self.CREDENTIAL_STORE_CLASS_PATH_NAME] = cs_lib_path
return cmd_result
http://git-wip-us.apache.org/repos/asf/ambari/blob/d340fe93/ambari-server/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml
index 0441867..89b734e 100644
--- a/ambari-server/pom.xml
+++ b/ambari-server/pom.xml
@@ -134,6 +134,15 @@
value="org.apache.ambari.server.DBConnectionVerification" />
</manifest>
</jar>
+ <jar destfile="target/CredentialUtil.jar">
+ <fileset dir="${basedir}/target/classes/">
+ <include name="**/CredentialUtil*.class" />
+ </fileset>
+ <manifest>
+ <attribute name="Main-Class"
+ value="org.apache.ambari.server.credentialapi.CredentialUtil" />
+ </manifest>
+ </jar>
</tasks>
</configuration>
<goals>
@@ -1469,6 +1478,11 @@
<version>${hadoop.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
<version>${jetty.version}</version>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d340fe93/ambari-server/src/main/java/org/apache/ambari/server/credentialapi/CredentialUtil.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/credentialapi/CredentialUtil.java b/ambari-server/src/main/java/org/apache/ambari/server/credentialapi/CredentialUtil.java
new file mode 100644
index 0000000..e6d7a37
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/credentialapi/CredentialUtil.java
@@ -0,0 +1,580 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ambari.server.credentialapi;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.security.alias.CredentialShell;
+import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Command line utility that wraps over CredentialShell. Extends the
+ * create command to overwrite a credential if it exists. Also
+ * provides the ability to get the decrypted password.
+ *
+ * The CLI structure and code is the same as CredentialShell.
+ */
+public class CredentialUtil extends Configured implements Tool {
+ /**
+ * List of supported commands.
+ */
+ final static private String COMMANDS =
+ " [--help]\n" +
+ " [" + CreateCommand.USAGE + "]\n" +
+ " [" + DeleteCommand.USAGE + "]\n" +
+ " [" + ListCommand.USAGE + "]\n" +
+ " [" + GetCommand.USAGE + "]\n";
+
+ /**
+ * JCEKS provider prefix.
+ */
+ public static final String jceksPrefix = JavaKeyStoreProvider.SCHEME_NAME + "://file";
+
+ /**
+ * Local JCEKS provider prefix.
+ */
+ public static final String localJceksPrefix = "localjceks://file";
+
+ /**
+ * Password alias
+ */
+ private String alias = null;
+
+ /**
+ * Password specified using the -value option
+ */
+ private String value = null;
+
+ /**
+ * Provider specified using the -provider option
+ */
+ protected CredentialProvider provider;
+
+ /**
+ * When creating a credential, overwrite the credential if it exists.
+ * If -n option is specified, this will be set to false.
+ */
+ private boolean overwrite = true;
+
+ /**
+ * Prompt for user confirmation before deleting/overwriting a credential.
+ * If -f option is specified, it will be set to false. In the case
+ * of a create command, it will be set to false if -n or -f is specified.
+ */
+ private boolean interactive = true;
+
+ /**
+ * One of the supported credential commands.
+ */
+ private Command command = null;
+
+ /**
+ * Main program.
+ *
+ * @param args Command line arguments
+ * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new CredentialUtil(), args);
+ System.exit(res);
+ }
+
+ /**
+ * Called by ToolRunner.run(). This is the entry point to the tool.
+ * Parses the command line arguments and executes the appropriate command.
+ *
+ * @param args - Arguments supplied by the user.
+ * @return - 0 if successful. 1 in case of a failure.
+ * @throws Exception - If something goes wrong during command execution.
+ */
+ @Override
+ public int run(String[] args) throws Exception {
+ int exitCode = 1;
+
+ for (int i = 0; i < args.length; ++i) {
+ if (args[i].equals("create")) {
+ if (i == args.length - 1) {
+ return 1;
+ }
+ command = new CreateCommand();
+ alias = args[++i];
+ if (alias.equals("-h") || alias.equals("-help")) {
+ printUsage();
+ return 0;
+ }
+ } else if (args[i].equals("get")) {
+ if (i == args.length - 1) {
+ return 1;
+ }
+ command = new GetCommand();
+ alias = args[++i];
+ if (alias.equals("-h") || alias.equals("-help")) {
+ printUsage();
+ return 0;
+ }
+ } else if (args[i].equals("delete")) {
+ if (i == args.length - 1) {
+ printUsage();
+ return 1;
+ }
+ command = new DeleteCommand();
+ alias = args[++i];
+ if (alias.equals("-help")) {
+ printUsage();
+ return 0;
+ }
+ } else if (args[i].equals("list")) {
+ if (i < args.length - 1) {
+ alias = args[i + 1];
+ }
+ command = new ListCommand();
+ if (alias.equals("-h") || alias.equals("-help")) {
+ printUsage();
+ return 0;
+ }
+ alias = "not required";
+ } else if (args[i].equals("-provider")) {
+ if (i == args.length - 1) {
+ return 1;
+ }
+ String providerPath = getNormalizedPath(args[++i]);
+ getConf().set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerPath);
+ provider = getCredentialProvider();
+ } else if (args[i].equals("-f") || args[i].equals("-force")) {
+ interactive = false;
+ overwrite = true;
+ } else if (args[i].equals("-n")) {
+ interactive = false;
+ overwrite = false;
+ } else if (args[i].equals("-v") || args[i].equals("-value")) {
+ value = args[++i];
+ } else if (args[i].equals("-h") || args[i].equals("-help")) {
+ printUsage();
+ return 0;
+ } else {
+ printUsage();
+ ToolRunner.printGenericCommandUsage(System.err);
+ return 1;
+ }
+ }
+
+ if (command == null) {
+ printUsage();
+ }
+ else if (command.validate()) {
+ exitCode = command.execute();
+ }
+
+ return exitCode;
+ }
+
+ /**
+ * Prints a command specific usage or overall tool usage.
+ */
+ protected void printUsage() {
+ System.out.println(getUsagePrefix() + COMMANDS);
+ if (command != null) {
+ System.out.println(command.getUsage());
+ }
+ else {
+ System.out.println("=========================================================" +
+ "======");
+ System.out.println(CreateCommand.USAGE + ":\n\n" + CreateCommand.DESC);
+ System.out.println("=========================================================" +
+ "======");
+ System.out.println(DeleteCommand.USAGE + ":\n\n" + DeleteCommand.DESC);
+ System.out.println("=========================================================" +
+ "======");
+ System.out.println(ListCommand.USAGE + ":\n\n" + ListCommand.DESC);
+ System.out.println("=========================================================" +
+ "======");
+ System.out.println(GetCommand.USAGE + ":\n\n" + GetCommand.DESC);
+ }
+ }
+
+ /**
+ * Overridden by the command line driver to provide name of the tool.
+ *
+ * @return - CLI specific information, like tool name, year, copyright, etc.
+ */
+ protected String getUsagePrefix() {
+ return "Usage: ";
+ }
+
+ /*
+ * Normalize the providerPath to jceks://file/<file_path> or localjceks://file/<file_path>
+ */
+ private static String getNormalizedPath(String providerPath) {
+ if (providerPath != null) {
+ String jceksPath;
+
+ if (providerPath.startsWith("/")) {
+ providerPath = providerPath.substring(1);
+ }
+
+ jceksPath = StringUtils.lowerCase(providerPath.trim());
+
+ if (!jceksPath.startsWith(StringUtils.lowerCase(jceksPrefix)) &&
+ !jceksPath.startsWith(localJceksPrefix)) {
+ providerPath = jceksPrefix + "/" + providerPath;
+ }
+ }
+
+ return providerPath;
+ }
+
+ /**
+ * Gets the provider object for the user specified provider.
+ *
+ * @return - A credential provider.
+ */
+ private CredentialProvider getCredentialProvider() {
+ CredentialProvider provider = null;
+
+ List<CredentialProvider> providers;
+ try {
+ providers = CredentialProviderFactory.getProviders(getConf());
+ provider = providers.get(0);
+ } catch (IOException e) {
+ e.printStackTrace(System.err);
+ }
+
+ return provider;
+ }
+
+ /**
+ * CredentialCommand base class
+ */
+ private abstract class Command {
+ /**
+ * Validates the user input.
+ *
+ * @return - True if inputs are valid. False otherwise.
+ */
+ public boolean validate() {
+ boolean rc = true;
+
+ if (alias == null || alias.isEmpty()) {
+ System.out.println("There is no alias specified. Please provide the" +
+ "mandatory <alias>. See the usage description with -help.");
+ rc = false;
+ }
+
+ if (provider == null) {
+ System.out.println("There are no valid CredentialProviders configured." +
+ "\nCredential will not be created.\n"
+ + "Consider using the -provider option to indicate the provider" +
+ " to use.");
+ rc = false;
+ }
+
+ return rc;
+ }
+
+ /**
+ * Gets command usage and description.
+ *
+ * @return
+ */
+ public abstract String getUsage();
+
+ /**
+ * Called by run(). Implemented by the concrete command classes.
+ *
+ * @return - 0 if successful. 1 on failure.
+ * @throws Exception - If something goes wrong.
+ */
+ public abstract int execute() throws Exception;
+ }
+
+ /**
+ * Gets the credential for the specified alias from the
+ * specified provider.
+ */
+ private class GetCommand extends Command {
+ public static final String USAGE = "get <alias> [-provider provider-path]";
+ public static final String DESC =
+ "The get subcommand gets the credential for the specified alias\n" +
+ "from the provider specified through the -provider argument.\n";
+
+ /**
+ * Executes the get command. Prints the clear text password on the command line.
+ *
+ * @return - 0 on success; 1 on failure.
+ * @throws IOException
+ */
+ @Override
+ public int execute() throws IOException {
+ int exitCode = 0;
+
+ try {
+ String credential = getCredential();
+ if (credential == null) {
+ exitCode = 1;
+ } else {
+ System.out.println(credential);
+ }
+ } catch (IOException ex) {
+ System.out.println("Cannot get the credential for the specified alias."
+ + ": " + ex.getMessage());
+ throw ex;
+ }
+
+ return exitCode;
+ }
+
+ /**
+ * Gets the clear text password from the credential provider.
+ *
+ * @return - Decrypted password for the specified alias.
+ * @throws IOException
+ */
+ private String getCredential() throws IOException {
+ String credential = null;
+ CredentialProvider.CredentialEntry credEntry = provider.getCredentialEntry(alias);
+
+ if (credEntry != null) {
+ char[] password = credEntry.getCredential();
+ if (password != null) {
+ credential = String.valueOf(password);
+ }
+ }
+
+ return credential;
+ }
+
+ /**
+ * Usage and description.
+ *
+ * @return
+ */
+ @Override
+ public String getUsage() {
+ return USAGE + ":\n\n" + DESC;
+ }
+ }
+
+ /**
+ * Creates a new credential for the alias specified or overwrites an
+ * existing credential
+ */
+ private class CreateCommand extends Command {
+ /**
+ * Usage summary
+ */
+ public static final String USAGE =
+ "create <alias> [-value credential] [-provider provider-path] [-f | -n]";
+
+ /**
+ * Command description
+ */
+ public static final String DESC =
+ "The create subcommand creates a new credential or overwrites\n" +
+ "an existing credential for the name specified\n" +
+ "as the <alias> argument within the provider indicated through\n" +
+ "the -provider argument. The command asks for confirmation to\n" +
+ "overwrite the existing credential unless the -f option is specified.\n" +
+ "Specify -n to not overwrite if the credential exists.\nThe option specified last wins.";
+
+ /**
+ * Creates or updates the specified credential.
+ *
+ * @return - 0 on success; 1 on failure.
+ * @throws Exception
+ */
+ @Override
+ public int execute() throws Exception {
+ int exitCode = 0;
+ CredentialProvider.CredentialEntry credEntry = provider.getCredentialEntry(alias);
+
+ if (credEntry != null) {
+ /*
+ * If credential already exists, overwrite if -f flag was specified.
+ * overwrite is true if -f was specified.
+ * overwrite is false if -n was specified.
+ * if neither options were specified, prompt the user.
+ */
+ if (interactive) {
+ // prompt the user to confirm or reject the overwrite
+ overwrite = ToolRunner
+ .confirmPrompt("You are about to OVERWRITE the credential " +
+ alias + " from CredentialProvider " + provider.toString() +
+ ". Continue? ");
+ }
+
+ if (overwrite) {
+ // delete the existing credential
+ DeleteCommand deleteCommand = new DeleteCommand();
+ exitCode = deleteCommand.execute();
+ } else {
+ // nothing to do
+ return 0;
+ }
+ }
+
+ // create new or overwrite existing credential if delete succeeded
+ if (exitCode == 0) {
+ exitCode = createCredential();
+ }
+
+ return exitCode;
+ }
+
+ /**
+ * Usage and description.
+ * @return
+ */
+ @Override
+ public String getUsage() {
+ return USAGE + ":\n\n" + DESC;
+ }
+
+ /**
+ * Creates the specified credential. A credential with the same alias
+ * should not exist. It must be deleted before this method is called.
+ *
+ * @return - 0 on success; 1 on failure.
+ * @throws Exception - If the alias already exists.
+ */
+ private int createCredential() throws Exception {
+ int exitCode;
+ List<String> args = new ArrayList<>();
+
+ args.add("create");
+ args.add(alias);
+ if (value != null) {
+ args.add("-value");
+ args.add(value);
+ }
+
+ String[] toolArgs = args.toArray(new String[args.size()]);
+
+ exitCode = ToolRunner.run(getConf(), new CredentialShell(), toolArgs);
+
+ return exitCode;
+ }
+ }
+
+ /**
+ * Deletes the credential specified by the alias from the
+ * specified provider.
+ */
+ private class DeleteCommand extends Command {
+ public static final String USAGE =
+ "delete <alias> [-f] [-provider provider-path]";
+ public static final String DESC =
+ "The delete subcommand deletes the credential specified\n" +
+ "as the <alias> argument from within the provider indicated\n" +
+ "through the -provider argument. The command asks for\n" +
+ "confirmation unless the -f option is specified.";
+
+ /**
+ * Deletes the specified alias. Prompts for user confirmation
+ * if -f option is not specified.
+ * @return
+ * @throws Exception
+ */
+ @Override
+ public int execute() throws Exception {
+ int exitCode;
+ List<String> args = new ArrayList<>();
+
+ args.add("delete");
+ args.add(alias);
+ if (!interactive) {
+ args.add("-f");
+ }
+
+ String[] toolArgs = args.toArray(new String[args.size()]);
+
+ exitCode = ToolRunner.run(getConf(), new CredentialShell(), toolArgs);
+
+ return exitCode;
+ }
+
+ /**
+ * Usage and description.
+ *
+ * @return
+ */
+ @Override
+ public String getUsage() {
+ return USAGE + ":\n\n" + DESC;
+ }
+ }
+
+
+ /**
+ * Lists all the aliases contained in the specified provider.
+ */
+ private class ListCommand extends Command {
+ /**
+ * Command usage
+ */
+ public static final String USAGE = "list [-provider provider-path]";
+
+ /**
+ * Command description
+ */
+ public static final String DESC =
+ "The list subcommand displays the aliases contained within \n" +
+ "a particular provider - as configured in core-site.xml or\n " +
+ "indicated through the -provider argument.";
+
+ /**
+ * Executes the list command.
+ *
+ * @return - 0 if successful; 1 otherwise.
+ * @throws Exception - If something goes wrong.
+ */
+ @Override
+ public int execute() throws Exception {
+ int exitCode;
+ List<String> args = new ArrayList<>();
+
+ args.add("list");
+
+ String[] toolArgs = args.toArray(new String[args.size()]);
+
+ exitCode = ToolRunner.run(getConf(), new CredentialShell(), toolArgs);
+
+ return exitCode;
+ }
+
+ /**
+ * Usage and description.
+ *
+ * @return
+ */
+ @Override
+ public String getUsage() {
+ return USAGE + ":\n\n" + DESC;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/d340fe93/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index 8451de1..211fe0a 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -46,6 +46,11 @@ from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_f
from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs
from resource_management.libraries.functions.get_architecture import get_architecture
+from resource_management.core.utils import PasswordString
+from resource_management.core.shell import checked_call
+from resource_management.core.logger import Logger
+from ambari_commons.inet_utils import download_file
+
# Default log4j version; put config files under /etc/hive/conf
log4j_version = '1'
@@ -55,6 +60,10 @@ tmp_dir = Script.get_tmp_dir()
architecture = get_architecture()
sudo = AMBARI_SUDO_BINARY
+credential_store_enabled = False
+if 'credentialStoreEnabled' in config:
+ credential_store_enabled = config['credentialStoreEnabled']
+
stack_root = status_params.stack_root
stack_name = status_params.stack_name
stack_name_uppercase = stack_name.upper()
@@ -218,7 +227,46 @@ execute_path = os.environ['PATH'] + os.pathsep + hive_bin + os.pathsep + hadoop_
hive_metastore_user_name = config['configurations']['hive-site']['javax.jdo.option.ConnectionUserName']
hive_jdbc_connection_url = config['configurations']['hive-site']['javax.jdo.option.ConnectionURL']
-hive_metastore_user_passwd = config['configurations']['hive-site']['javax.jdo.option.ConnectionPassword']
+jdk_location = config['hostLevelParams']['jdk_location']
+
+credential_util_cmd = 'org.apache.ambari.server.credentialapi.CredentialUtil'
+credential_util_jar = 'CredentialUtil.jar'
+
+# Gets the hive metastore password from its JCEKS provider, if available.
+def getHiveMetastorePassword():
+ passwd = ''
+ if 'hadoop.security.credential.provider.path' in config['configurations']['hive-site']:
+ # Try to download CredentialUtil.jar from ambari-server resources
+ cs_lib_path = config['configurations']['hive-site']['credentialStoreClassPath']
+ credential_util_dir = cs_lib_path.split('*')[0] # Remove the trailing '*'
+ credential_util_path = os.path.join(credential_util_dir, credential_util_jar)
+ credential_util_url = jdk_location + credential_util_jar
+ try:
+ download_file(credential_util_url, credential_util_path)
+ except Exception, e:
+ message = 'Error downloading {0} from Ambari Server resources. {1}'.format(credential_util_url, str(e))
+ Logger.error(message)
+ raise
+
+ # Execute a get command on the CredentialUtil CLI to get the password for the specified alias
+ java_home = config['hostLevelParams']['java_home']
+ java_bin = '{java_home}/bin/java'.format(java_home=java_home)
+ alias = 'javax.jdo.option.ConnectionPassword'
+ provider_path = config['configurations']['hive-site']['hadoop.security.credential.provider.path']
+ cmd = (java_bin, '-cp', cs_lib_path, credential_util_cmd, 'get', alias, '-provider', provider_path)
+ cmd_result, std_out_msg = checked_call(cmd)
+ if cmd_result != 0:
+ message = 'The following error occurred while executing {0}: {1}'.format(' '.join(cmd), std_out_msg)
+ Logger.error(message)
+ raise
+ std_out_lines = std_out_msg.split('\n')
+ passwd = std_out_lines[-1] # Get the last line of the output, to skip warnings if any.
+ return passwd
+
+if credential_store_enabled:
+ hive_metastore_user_passwd = PasswordString(getHiveMetastorePassword())
+else:
+ hive_metastore_user_passwd = config['configurations']['hive-site']['javax.jdo.option.ConnectionPassword']
hive_metastore_user_passwd = unicode(hive_metastore_user_passwd) if not is_empty(hive_metastore_user_passwd) else hive_metastore_user_passwd
hive_metastore_db_type = config['configurations']['hive-env']['hive_database_type']
@@ -237,8 +285,6 @@ if 'roleCommand' in config and 'CUSTOM_COMMAND' == config['roleCommand']:
#JDBC driver jar name
hive_jdbc_driver = config['configurations']['hive-site']['javax.jdo.option.ConnectionDriverName']
-jdk_location = config['hostLevelParams']['jdk_location']
-
java_share_dir = '/usr/share/java'
hive_database_name = config['configurations']['hive-env']['hive_database_name']
hive_database = config['configurations']['hive-env']['hive_database']
@@ -796,4 +842,4 @@ if enable_ranger_hive:
if has_ranger_admin and stack_supports_ranger_audit_db and xa_audit_db_flavor.lower() == 'sqla':
xa_audit_db_is_enabled = False
-# ranger hive plugin section end
+# ranger hive plugin section end
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/d340fe93/ambari-server/src/test/java/org/apache/ambari/server/credentialapi/CredentialUtilTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/credentialapi/CredentialUtilTest.java b/ambari-server/src/test/java/org/apache/ambari/server/credentialapi/CredentialUtilTest.java
new file mode 100644
index 0000000..abc7cfe
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/credentialapi/CredentialUtilTest.java
@@ -0,0 +1,644 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.credentialapi;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ToolRunner;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+
+/**
+ * Tests CredentialUtilTest.
+ */
+public class CredentialUtilTest {
+ /**
+ * Cleans up itself after a test method is run.
+ */
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ /**
+ * Redirect System.out to a stream. CredentialShell() writes to System.out.
+ * We want to capture that.
+ */
+ private final ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+ /**
+ * Redirect System.err to a stream.
+ */
+ private final ByteArrayOutputStream err = new ByteArrayOutputStream();
+
+ /**
+ * CRUD command verbs
+ */
+ private static final String CREATE_VERB = "create";
+ private static final String DELETE_VERB = "delete";
+ private static final String LIST_VERB = "list";
+ private static final String GET_VERB = "get";
+
+
+ /**
+ * Gets a temporary folder name.
+ *
+ * @return
+ */
+ private String getTempFolderName() {
+ return temporaryFolder.getRoot().getAbsolutePath();
+ }
+
+ /**
+ * Constructs a valid JCEKS provider path from the specified file name.
+ *
+ * @param fileName
+ * @return
+ */
+ private String getProviderPath(String fileName) {
+ return CredentialUtil.jceksPrefix + getTempFolderName() + "/" + fileName;
+ }
+
+ /**
+ * Constructs the create arguments to create a new credential.
+ *
+ * @param alias
+ * @param credential
+ * @param providerPath
+ * @return
+ */
+ private String[] getCreateArgs(String alias, String credential, String providerPath) {
+ List<String> args = new ArrayList<String>();
+
+ args.add(CREATE_VERB);
+ args.add(alias);
+ args.add("-value");
+ args.add(credential);
+ args.add("-provider");
+ args.add(providerPath);
+
+ return args.toArray(new String[args.size()]);
+ }
+
+ /**
+ * Constructs the create arguments with no overwrite existing option.
+ *
+ * @param alias
+ * @param credential
+ * @param providerPath
+ * @return
+ */
+ private String[] getSafeCreateArgs(String alias, String credential, String providerPath) {
+ List<String> args = new ArrayList<String>();
+
+ args.add(CREATE_VERB);
+ args.add(alias);
+ args.add("-value");
+ args.add(credential);
+ args.add("-provider");
+ args.add(providerPath);
+ args.add("-n"); /* do not overwrite if credential exists */
+
+ return args.toArray(new String[args.size()]);
+ }
+
+ /**
+ * Constructs the create arguments with overwrite.
+ *
+ * @param alias
+ * @param credential
+ * @param providerPath
+ * @return
+ */
+ private String[] getUpdateArgs(String alias, String credential, String providerPath) {
+ List<String> args = new ArrayList<String>();
+
+ args.add(CREATE_VERB);
+ args.add(alias);
+ args.add("-value");
+ args.add(credential);
+ args.add("-provider");
+ args.add(providerPath);
+ args.add("-f"); /* overwrite */
+
+ return args.toArray(new String[args.size()]);
+ }
+
+ /**
+ * Constructs the delete arguments.
+ *
+ * @param alias
+ * @param providerPath
+ * @return
+ */
+ private String[] getDeleteArgs(String alias, String providerPath) {
+ List<String> args = new ArrayList<String>();
+
+ args.add(DELETE_VERB);
+ args.add(alias);
+ args.add("-provider");
+ args.add(providerPath);
+ args.add("-f"); /* suppress prompt to confirm */
+
+ return args.toArray(new String[args.size()]);
+ }
+
+ /**
+ * List arguments.
+ *
+ * @param providerPath
+ * @return
+ */
+ private String[] getListArgs(String providerPath) {
+ List<String> args = new ArrayList<String>();
+
+ args.add(LIST_VERB);
+ args.add("-provider");
+ args.add(providerPath);
+
+ return args.toArray(new String[args.size()]);
+ }
+
+ /**
+ * Get arguments.
+ *
+ * @param alias
+ * @param providerPath
+ * @return
+ */
+ private String[] getGetArgs(String alias, String providerPath) {
+ List<String> args = new ArrayList<String>();
+
+ args.add(GET_VERB);
+ args.add(alias);
+ args.add("-provider");
+ args.add(providerPath);
+
+ return args.toArray(new String[args.size()]);
+ }
+
+ /**
+ * Executes the given arguments.
+ *
+ * @param args
+ * @return
+ * @throws Exception
+ */
+ private int executeCommand(String[] args) throws Exception {
+ return ToolRunner.run(new Configuration(), new CredentialUtil(), args);
+ }
+
+ /**
+ * Set up the streams before each test.
+ */
+ @Before
+ public void setupStreams() {
+ System.setOut(new PrintStream(out));
+ System.setErr(new PrintStream(err));
+ }
+
+ /**
+ * Clean up the stream after each test.
+ */
+ @After
+ public void teardownStreams() {
+ System.setOut(null);
+ System.setErr(null);
+ }
+
+ /**
+ * Creates a non-existing credential
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testCreateCommand() throws Exception {
+ String alias = "javax.jdo.option.ConnectionPassword";
+ String credential = "MyTopSecretPassword";
+ String providerPath = getProviderPath("CreateCommandTest.jceks");
+ String[] args = getCreateArgs(alias, credential, providerPath);
+
+ int exitCode = executeCommand(args);
+
+ Assert.assertEquals(exitCode, 0);
+ }
+
+ /**
+ * Overwrites an existing credential.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testCreateCommandOverwriteExisting() throws Exception {
+ String alias = "javax.jdo.option.ConnectionPassword";
+ String credential = "MyTopSecretPassword";
+ String providerPath = getProviderPath("CreateCommandTest.jceks");
+ String[] args;
+ int exitCode;
+
+ /*
+ * Create a new credential
+ */
+ args = getCreateArgs(alias, credential, providerPath);
+ exitCode = ToolRunner.run(new Configuration(), new CredentialUtil(), args);
+ Assert.assertEquals(exitCode, 0);
+
+ /*
+ * Update the created credential
+ */
+ credential = "MyUpdatedTopSecretPassword";
+ args = getUpdateArgs(alias, credential, providerPath);
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 0);
+ }
+
+ /**
+ * Updates a non-existing credential. Should create it.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testCreateCommandOverwriteNonExisting() throws Exception {
+ String alias = "javax.jdo.option.ConnectionPassword";
+ String credential = "MyTopSecretPassword";
+ String providerPath = getProviderPath("CreateCommandTest.jceks");
+ String[] args;
+ int exitCode;
+
+ /*
+ * Update a non-existing credential. Should create it.
+ */
+ args = getUpdateArgs(alias, credential, providerPath);
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 0);
+ }
+
+ /**
+ * Safely creates a credential. If credential already exists, nothing is done.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testSafeCreateCommandExisting() throws Exception {
+ String alias = "javax.jdo.option.ConnectionPassword";
+ String credential = "MyTopSecretPassword";
+ String providerPath = getProviderPath("CreateCommandTest.jceks");
+ String[] args;
+ int exitCode;
+
+ /*
+ * Create a new credential
+ */
+ args = getCreateArgs(alias, credential, providerPath);
+ exitCode = ToolRunner.run(new Configuration(), new CredentialUtil(), args);
+ Assert.assertEquals(exitCode, 0);
+
+ /*
+ * Safely update the previously created credential. Nothing is done.
+ */
+ credential = "AnotherTopSecretPassword";
+ args = getSafeCreateArgs(alias, credential, providerPath);
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 0);
+ }
+
+ /**
+ * Safely creates a credential. If it does not exist, it will be created.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testSafeCreateCommandNotExisting() throws Exception {
+ String alias = "javax.jdo.option.ConnectionPassword";
+ String credential = "MyTopSecretPassword";
+ String providerPath = getProviderPath("CreateCommandTest.jceks");
+ String[] args;
+ int exitCode;
+
+ /*
+ * Safely update a non-existing credential. Should create the credential.
+ */
+ credential = "AnotherTopSecretPassword";
+ args = getSafeCreateArgs(alias, credential, providerPath);
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 0);
+ }
+
+ /**
+ * Delete an existing credential
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testDeleteCommandExisting() throws Exception {
+ String alias = "javax.jdo.option.ConnectionPassword";
+ String credential = "MyTopSecretPassword";
+ String providerPath = getProviderPath("CreateCommandTest.jceks");
+ String[] args;
+ int exitCode;
+
+ /*
+ * Create a new credential
+ */
+ args = getCreateArgs(alias, credential, providerPath);
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 0);
+
+ /*
+ * Delete the above credential
+ */
+ args = getDeleteArgs(alias, providerPath);
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 0);
+ }
+
+ /**
+ * Delete a non-existing credential
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testDeleteCommandNonExisting() throws Exception {
+ String alias = "javax.jdo.option.ConnectionPassword";
+ String providerPath = getProviderPath("CreateCommandTest.jceks");
+ String[] args;
+ int exitCode;
+
+ /*
+ * Delete a non-existing credential. Should fail with exit code 1.
+ */
+ args = getDeleteArgs(alias, providerPath);
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 1);
+ }
+
+ /**
+ * Retrieve an existing credential.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testGetCommandExisting() throws Exception {
+ String alias = "javax.jdo.option.ConnectionPassword";
+ String credential = "MyTopSecretPassword";
+ String providerPath = getProviderPath("CreateCommandTest.jceks");
+ String[] args;
+ int exitCode;
+
+ /*
+ * Create a new credential
+ */
+ args = getCreateArgs(alias, credential, providerPath);
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 0);
+ out.reset();
+
+ /*
+ * Get the existing credential.
+ */
+ args = getGetArgs(alias, providerPath);
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 0);
+
+ String retrievedCredential = out.toString().trim();
+ Assert.assertEquals(credential, retrievedCredential);
+ }
+
+ /**
+ * Retrieve a non-existing credential.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testGetCommandNonExisting() throws Exception {
+ String alias = "javax.jdo.option.ConnectionPassword";
+ String providerPath = getProviderPath("CreateCommandTest.jceks");
+ String[] args;
+ int exitCode;
+
+ /*
+ * Get a non-existing credential. Should fail with exit code 1.
+ */
+ args = getGetArgs(alias, providerPath);
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 1);
+ }
+
+ /**
+ * Create, delete and attempt to get the alias.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testGetCommandAfterDeletion() throws Exception {
+ String alias = "javax.jdo.option.ConnectionPassword";
+ String credential = "MyTopSecretPassword";
+ String providerPath = getProviderPath("CreateCommandTest.jceks");
+ String[] args;
+ int exitCode;
+
+ /*
+ * Create a new credential
+ */
+ args = getCreateArgs(alias, credential, providerPath);
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 0);
+
+ /*
+ * Delete the above credential
+ */
+ args = getDeleteArgs(alias, providerPath);
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 0);
+
+ /*
+ * Get the existing credential. Should not be there.
+ */
+ args = getGetArgs(alias, providerPath);
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 1);
+ }
+
+ /**
+ * Execute get on an invalid provider path.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testGetCommandWithNonExistingProvider() throws Exception {
+ String alias = "javax.jdo.option.ConnectionPassword";
+ String credential = "MyTopSecretPassword";
+ String providerPath = getProviderPath("CreateCommandTest.jceks");
+ String[] args;
+ int exitCode;
+
+ /*
+ * Create a new credential
+ */
+ args = getCreateArgs(alias, credential, providerPath);
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 0);
+
+ /*
+ * Get a credential. Should not be there.
+ */
+ args = getGetArgs(alias, "BadProvider.jceks");
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 1);
+ }
+
+ /**
+ * List all aliases.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testListCommand() throws Exception {
+ String providerPath = getProviderPath("CreateCommandTest.jceks");
+ String[] args;
+ int exitCode;
+ final int numEntries = 5;
+ Properties properties = new Properties();
+
+ /*
+ * Create some alias password entries.
+ */
+ for (int i = 0; i < numEntries; ++i) {
+ String alias = String.format("alias_%d", i + 1);
+ String credential = String.format("credential_%d", i + 1);
+ properties.setProperty(alias, credential);
+ args = getCreateArgs(alias, credential, providerPath);
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 0);
+ }
+
+ out.reset();
+
+ /*
+ * List all aliases.
+ */
+ args = getListArgs(providerPath);
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 0);
+ List<String> aliases = Arrays.asList(out.toString().split(System.getProperty("line.separator")));
+ Enumeration enumeration = properties.propertyNames();
+ while (enumeration.hasMoreElements()) {
+ String alias = (String)enumeration.nextElement();
+ Assert.assertTrue(aliases.contains(alias));
+ }
+ }
+
+ /**
+ * Prints the tool usage.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testToolUsage() throws Exception {
+ String[] args = new String[1];
+ int exitCode;
+
+ /*
+ * Invoke tool help
+ */
+ args[0] = "-help";
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 0);
+ Assert.assertTrue(!out.toString().isEmpty());
+ }
+
+ /**
+ * Invoke Create command help
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testCreateCommandUsage() throws Exception {
+ String[] args = new String[2];
+ int exitCode;
+
+ args[0] = CREATE_VERB;
+ args[1] = "-help";
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 0);
+ Assert.assertTrue(!out.toString().isEmpty());
+ }
+
+ /*
+ * Invoke Delete command help
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testDeleteCommandUsage() throws Exception {
+ String[] args = new String[2];
+ int exitCode;
+
+ args[0] = DELETE_VERB;
+ args[1] = "-help";
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 0);
+ Assert.assertTrue(!out.toString().isEmpty());
+ }
+
+ /*
+ * Invoke List command help
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testListCommandUsage() throws Exception {
+ String[] args = new String[2];
+ int exitCode;
+
+ args[0] = LIST_VERB;
+ args[1] = "-help";
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 0);
+ Assert.assertTrue(!out.toString().isEmpty());
+ }
+
+ /*
+ * Invoke Get command help
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testGetCommandUsage() throws Exception {
+ String[] args = new String[2];
+ int exitCode;
+
+ args[0] = GET_VERB;
+ args[1] = "-help";
+ exitCode = executeCommand(args);
+ Assert.assertEquals(exitCode, 0);
+ Assert.assertTrue(!out.toString().isEmpty());
+ }
+}
\ No newline at end of file