You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2015/05/20 19:31:13 UTC
[1/2] ambari git commit: AMBARI-11202. templeton.hive.properties
property value substitution should be done by ambari-server (rlevas)
Repository: ambari
Updated Branches:
refs/heads/trunk a40b2b691 -> dac2f08c6
AMBARI-11202. templeton.hive.properties property value substitution should be done by ambari-server (rlevas)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/dac2f08c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/dac2f08c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/dac2f08c
Branch: refs/heads/trunk
Commit: dac2f08c655991ec04160875ed94f37be2bf5fca
Parents: 64ed815
Author: Robert Levas <rl...@hortonworks.com>
Authored: Wed May 20 13:30:37 2015 -0400
Committer: Robert Levas <rl...@hortonworks.com>
Committed: Wed May 20 13:31:06 2015 -0400
----------------------------------------------------------------------
.../server/controller/KerberosHelper.java | 56 ++---
.../kerberos/AbstractKerberosDescriptor.java | 87 -------
.../kerberos/KerberosKeytabDescriptor.java | 4 +-
.../kerberos/KerberosPrincipalDescriptor.java | 4 +-
.../kerberos/VariableReplacementHelper.java | 228 +++++++++++++++++++
.../HIVE/0.12.0.2.0/kerberos.json | 2 +-
.../state/kerberos/KerberosDescriptorTest.java | 123 +---------
.../kerberos/VariableReplacementHelperTest.java | 174 ++++++++++++++
8 files changed, 438 insertions(+), 240 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/dac2f08c/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
index e083b0e..141803b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
@@ -92,6 +92,7 @@ import org.apache.ambari.server.state.kerberos.KerberosIdentityDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosKeytabDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosPrincipalDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor;
+import org.apache.ambari.server.state.kerberos.VariableReplacementHelper;
import org.apache.ambari.server.state.svccomphost.ServiceComponentHostServerActionEvent;
import org.apache.ambari.server.utils.StageUtils;
import org.apache.commons.io.FileUtils;
@@ -180,6 +181,9 @@ public class KerberosHelper {
@Inject
private KerberosConfigDataFileWriterFactory kerberosConfigDataFileWriterFactory;
+ @Inject
+ private VariableReplacementHelper variableReplacementHelper;
+
/**
* Used to get kerberos descriptors associated with the cluster or stack.
* Currently not available via injection.
@@ -1085,9 +1089,9 @@ public class KerberosHelper {
// Add the relevant principal name and keytab file data to the command params state
if (!commandParameters.containsKey("principal_name") || !commandParameters.containsKey("keytab_file")) {
commandParameters.put("principal_name",
- KerberosDescriptor.replaceVariables(identity.getPrincipalDescriptor().getValue(), configurations));
+ variableReplacementHelper.replaceVariables(identity.getPrincipalDescriptor().getValue(), configurations));
commandParameters.put("keytab_file",
- KerberosDescriptor.replaceVariables(identity.getKeytabDescriptor().getFile(), configurations));
+ variableReplacementHelper.replaceVariables(identity.getKeytabDescriptor().getFile(), configurations));
}
serviceComponentHostsToProcess.add(sch);
@@ -1400,7 +1404,7 @@ public class KerberosHelper {
* Merges configuration from a Map of configuration updates into a main configurations Map. Each
* property in the updates Map is processed to replace variables using the replacement Map.
* <p/>
- * See {@link org.apache.ambari.server.state.kerberos.KerberosDescriptor#replaceVariables(String, java.util.Map)}
+ * See {@link VariableReplacementHelper#replaceVariables(String, Map)}
* for information on variable replacement.
*
* @param configurations a Map of configurations
@@ -1437,7 +1441,7 @@ public class KerberosHelper {
* Merges the specified configuration property in a map of configuration types.
* The supplied property is processed to replace variables using the replacement Map.
* <p/>
- * See {@link org.apache.ambari.server.state.kerberos.KerberosDescriptor#replaceVariables(String, java.util.Map)}
+ * See {@link VariableReplacementHelper#replaceVariables(String, Map)}
* for information on variable replacement.
*
* @param configurations the Map of configuration types to update
@@ -1466,7 +1470,7 @@ public class KerberosHelper {
* Merges configuration from a Map of configuration updates into a main configurations Map. Each
* property in the updates Map is processed to replace variables using the replacement Map.
* <p/>
- * See {@link org.apache.ambari.server.state.kerberos.KerberosDescriptor#replaceVariables(String, java.util.Map)}
+ * See {@link VariableReplacementHelper#replaceVariables(String, Map)}
* for information on variable replacement.
*
* @param configurations a Map of configurations
@@ -1487,8 +1491,8 @@ public class KerberosHelper {
for (Map.Entry<String, String> property : updates.entrySet()) {
existingProperties.put(
- KerberosDescriptor.replaceVariables(property.getKey(), replacements),
- KerberosDescriptor.replaceVariables(property.getValue(), replacements)
+ variableReplacementHelper.replaceVariables(property.getKey(), replacements),
+ variableReplacementHelper.replaceVariables(property.getValue(), replacements)
);
}
}
@@ -1532,9 +1536,9 @@ public class KerberosHelper {
String principalConfiguration = null;
if (principalDescriptor != null) {
- principal = KerberosDescriptor.replaceVariables(principalDescriptor.getValue(), configurations);
+ principal = variableReplacementHelper.replaceVariables(principalDescriptor.getValue(), configurations);
principalType = principalDescriptor.getType().name().toLowerCase();
- principalConfiguration = KerberosDescriptor.replaceVariables(principalDescriptor.getConfiguration(), configurations);
+ principalConfiguration = variableReplacementHelper.replaceVariables(principalDescriptor.getConfiguration(), configurations);
}
if (principal != null) {
@@ -1548,12 +1552,12 @@ public class KerberosHelper {
boolean keytabIsCachable = false;
if (keytabDescriptor != null) {
- keytabFilePath = KerberosDescriptor.replaceVariables(keytabDescriptor.getFile(), configurations);
- keytabFileOwnerName = KerberosDescriptor.replaceVariables(keytabDescriptor.getOwnerName(), configurations);
- keytabFileOwnerAccess = KerberosDescriptor.replaceVariables(keytabDescriptor.getOwnerAccess(), configurations);
- keytabFileGroupName = KerberosDescriptor.replaceVariables(keytabDescriptor.getGroupName(), configurations);
- keytabFileGroupAccess = KerberosDescriptor.replaceVariables(keytabDescriptor.getGroupAccess(), configurations);
- keytabFileConfiguration = KerberosDescriptor.replaceVariables(keytabDescriptor.getConfiguration(), configurations);
+ keytabFilePath = variableReplacementHelper.replaceVariables(keytabDescriptor.getFile(), configurations);
+ keytabFileOwnerName = variableReplacementHelper.replaceVariables(keytabDescriptor.getOwnerName(), configurations);
+ keytabFileOwnerAccess = variableReplacementHelper.replaceVariables(keytabDescriptor.getOwnerAccess(), configurations);
+ keytabFileGroupName = variableReplacementHelper.replaceVariables(keytabDescriptor.getGroupName(), configurations);
+ keytabFileGroupAccess = variableReplacementHelper.replaceVariables(keytabDescriptor.getGroupAccess(), configurations);
+ keytabFileConfiguration = variableReplacementHelper.replaceVariables(keytabDescriptor.getConfiguration(), configurations);
keytabIsCachable = keytabDescriptor.isCachable();
}
@@ -1608,8 +1612,8 @@ public class KerberosHelper {
KerberosPrincipalDescriptor principalDescriptor = identity.getPrincipalDescriptor();
if (principalDescriptor != null) {
authToLocalBuilder.addRule(
- KerberosDescriptor.replaceVariables(principalDescriptor.getValue(), configurations),
- KerberosDescriptor.replaceVariables(principalDescriptor.getLocalUsername(), configurations));
+ variableReplacementHelper.replaceVariables(principalDescriptor.getValue(), configurations),
+ variableReplacementHelper.replaceVariables(principalDescriptor.getLocalUsername(), configurations));
}
}
}
@@ -2073,7 +2077,7 @@ public class KerberosHelper {
String principal = null;
if (principalDescriptor != null) {
- principal = KerberosDescriptor.replaceVariables(principalDescriptor.getValue(), configurations);
+ principal = variableReplacementHelper.replaceVariables(principalDescriptor.getValue(), configurations);
}
if (principal != null) {
@@ -2081,7 +2085,7 @@ public class KerberosHelper {
String keytabFile = null;
if (keytabDescriptor != null) {
- keytabFile = KerberosDescriptor.replaceVariables(keytabDescriptor.getFile(), configurations);
+ keytabFile = variableReplacementHelper.replaceVariables(keytabDescriptor.getFile(), configurations);
}
if (replaceHostNames) {
@@ -2094,8 +2098,8 @@ public class KerberosHelper {
KerberosPrincipalDescriptor resolvedPrincipalDescriptor =
new KerberosPrincipalDescriptor(principal,
principalDescriptor.getType(),
- KerberosDescriptor.replaceVariables(principalDescriptor.getConfiguration(), configurations),
- KerberosDescriptor.replaceVariables(principalDescriptor.getLocalUsername(), configurations));
+ variableReplacementHelper.replaceVariables(principalDescriptor.getConfiguration(), configurations),
+ variableReplacementHelper.replaceVariables(principalDescriptor.getLocalUsername(), configurations));
KerberosKeytabDescriptor resolvedKeytabDescriptor;
@@ -2105,11 +2109,11 @@ public class KerberosHelper {
resolvedKeytabDescriptor =
new KerberosKeytabDescriptor(
keytabFile,
- KerberosDescriptor.replaceVariables(keytabDescriptor.getOwnerName(), configurations),
- KerberosDescriptor.replaceVariables(keytabDescriptor.getOwnerAccess(), configurations),
- KerberosDescriptor.replaceVariables(keytabDescriptor.getGroupName(), configurations),
- KerberosDescriptor.replaceVariables(keytabDescriptor.getGroupAccess(), configurations),
- KerberosDescriptor.replaceVariables(keytabDescriptor.getConfiguration(), configurations),
+ variableReplacementHelper.replaceVariables(keytabDescriptor.getOwnerName(), configurations),
+ variableReplacementHelper.replaceVariables(keytabDescriptor.getOwnerAccess(), configurations),
+ variableReplacementHelper.replaceVariables(keytabDescriptor.getGroupName(), configurations),
+ variableReplacementHelper.replaceVariables(keytabDescriptor.getGroupAccess(), configurations),
+ variableReplacementHelper.replaceVariables(keytabDescriptor.getConfiguration(), configurations),
keytabDescriptor.isCachable());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/dac2f08c/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java
index b49fec1..79b9a55 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java
@@ -18,20 +18,8 @@
package org.apache.ambari.server.state.kerberos;
-import com.google.gson.Gson;
-import com.google.gson.JsonSyntaxException;
-import com.google.gson.reflect.TypeToken;
-import org.apache.ambari.server.AmbariException;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
/**
* AbstractKerberosDescriptor is the base class for all Kerberos*Descriptor and associated classes.
@@ -42,11 +30,6 @@ import java.util.regex.Pattern;
public abstract class AbstractKerberosDescriptor {
/**
- * a regular expression Pattern used to find "variable" placeholders in strings
- */
- private static final Pattern PATTERN_VARIABLE = Pattern.compile("\\$\\{(?:([\\w\\-\\.]+)/)?([\\w\\-\\.]+)\\}");
-
- /**
* An AbstractKerberosDescriptor serving as the parent (or container) for this
* AbstractKerberosDescriptor.
* <p/>
@@ -62,76 +45,6 @@ public abstract class AbstractKerberosDescriptor {
private String name = null;
/**
- * Performs variable replacement on the supplied String value using values from the replacementsMap.
- * <p/>
- * The value is a String containing one or more "variables" in the form of ${variable_name}, such
- * that "variable_name" may indicate a group identifier; else "" is used as the group.
- * For example:
- * <p/>
- * variable_name: group: ""; property: "variable_name"
- * group1/variable_name: group: "group1"; property: "variable_name"
- * root/group1/variable_name: Not Supported
- * <p/>
- * The replacementsMap is a Map of Maps creating a (small) hierarchy of data to traverse in order
- * to resolve the variable.
- * <p/>
- * If a variable resolves to one or more variables, that new variable(s) will be processed and replaced.
- * If variable exists after a set number of iterations it is assumed that a cycle has been created
- * and the process will abort returning a String in a possibly unexpected state.
- *
- * @param value a String containing zero or more variables to be replaced
- * @param replacementsMap a Map of data used to perform the variable replacements
- * @return a new String
- */
- public static String replaceVariables(String value, Map<String, Map<String, String>> replacementsMap) throws AmbariException {
- if ((value != null) && (replacementsMap != null) && !replacementsMap.isEmpty()) {
- int count = 0; // Used to help prevent an infinite loop...
- boolean replacementPerformed;
-
- do {
- if (++count > 1000) {
- throw new AmbariException(String.format("Circular reference found while replacing variables in %s", value));
- }
-
- Matcher matcher = PATTERN_VARIABLE.matcher(value);
- StringBuffer sb = new StringBuffer();
-
- replacementPerformed = false;
-
- while (matcher.find()) {
- String type = matcher.group(1);
- String name = matcher.group(2);
- Map<String, String> replacements;
-
- if ((name != null) && !name.isEmpty()) {
- if (type == null) {
- replacements = replacementsMap.get("");
- } else {
- replacements = replacementsMap.get(type);
- }
-
- if (replacements != null) {
- String replacement = replacements.get(name);
-
- if (replacement != null) {
- // Escape '$' and '\' so they don't cause any issues.
- matcher.appendReplacement(sb, replacement.replace("\\", "\\\\").replace("$", "\\$"));
- replacementPerformed = true;
- }
- }
- }
- }
-
- matcher.appendTail(sb);
- value = sb.toString();
- }
- while (replacementPerformed); // Process the string again to make sure new variables were not introduced
- }
-
- return value;
- }
-
- /**
* Generates a Map of data that represents this AbstractKerberosDescriptor implementation.
* <p/>
* This method should be overwritten by AbstractKerberosDescriptor implementations to generate a
http://git-wip-us.apache.org/repos/asf/ambari/blob/dac2f08c/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java
index 404efa2..7ce1c9f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java
@@ -220,12 +220,12 @@ public class KerberosKeytabDescriptor extends AbstractKerberosDescriptor {
* <ul>
* <li>
* ${variable} placeholders are replaced on the server - see
- * {@link org.apache.ambari.server.state.kerberos.KerberosDescriptor#replaceVariables(String, java.util.Map)}
+ * {@link VariableReplacementHelper#replaceVariables(String, Map)}
* </li>
* </ul>
*
* @return a String declaring the keytab file's absolute path
- * @see org.apache.ambari.server.state.kerberos.KerberosDescriptor#replaceVariables(String, java.util.Map)
+ * @see VariableReplacementHelper#replaceVariables(String, Map)
*/
public String getFile() {
return getName();
http://git-wip-us.apache.org/repos/asf/ambari/blob/dac2f08c/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java
index 6e5ac5c..09f6872 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java
@@ -138,13 +138,13 @@ public class KerberosPrincipalDescriptor extends AbstractKerberosDescriptor {
* <ul>
* <li>
* ${variable} placeholders are replaced on the server - see
- * {@link org.apache.ambari.server.state.kerberos.KerberosDescriptor#replaceVariables(String, java.util.Map)}
+ * {@link VariableReplacementHelper#replaceVariables(String, Map)}
* </li>
* <li>the _HOST placeholder is replaced on the hosts to dynamically populate the relevant hostname</li>
* </ul>
*
* @return a String declaring this principal's value
- * @see org.apache.ambari.server.state.kerberos.KerberosDescriptor#replaceVariables(String, java.util.Map)
+ * @see VariableReplacementHelper#replaceVariables(String, Map)
*/
public String getValue() {
return getName();
http://git-wip-us.apache.org/repos/asf/ambari/blob/dac2f08c/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelper.java
new file mode 100644
index 0000000..42eea14
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelper.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.state.kerberos;
+
+import com.google.inject.Singleton;
+import org.apache.ambari.server.AmbariException;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Helper class to provide variable replacement services
+ */
+@Singleton
+public class VariableReplacementHelper {
+
+ /**
+ * a regular expression Pattern used to find "variable" placeholders in strings
+ */
+ private static final Pattern PATTERN_VARIABLE = Pattern.compile("\\$\\{(?:([\\w\\-\\.]+)/)?([\\w\\-\\.]+)(?:\\s*\\|\\s*(.+?))?\\}");
+
+ /**
+ * a regular expression Pattern used to parse "function" declarations: name(arg1, arg2, ...)
+ */
+ private static final Pattern PATTERN_FUNCTION = Pattern.compile("(\\w+)\\((.*?)\\)");
+
+ /**
+ * A map of "registered" functions
+ */
+ private static final Map<String, Function> FUNCTIONS = new HashMap<String, Function>() {
+ {
+ put("each", new EachFunction());
+ }
+ };
+
+ /**
+ * Performs variable replacement on the supplied String value using values from the replacementsMap.
+ * <p/>
+ * The value is a String containing one or more "variables" in the form of ${variable_name}, such
+ * that "variable_name" may indicate a group identifier; else "" is used as the group.
+ * For example:
+ * <p/>
+ * variable_name: group: ""; property: "variable_name"
+ * group1/variable_name: group: "group1"; property: "variable_name"
+ * root/group1/variable_name: Not Supported
+ * <p/>
+ * The replacementsMap is a Map of Maps creating a (small) hierarchy of data to traverse in order
+ * to resolve the variable.
+ * <p/>
+ * If a variable resolves to one or more variables, that new variable(s) will be processed and replaced.
+ * If variable exists after a set number of iterations it is assumed that a cycle has been created
+ * and the process will abort returning a String in a possibly unexpected state.
+ *
+ * @param value a String containing zero or more variables to be replaced
+ * @param replacementsMap a Map of data used to perform the variable replacements
+ * @return a new String
+ */
+ public String replaceVariables(String value, Map<String, Map<String, String>> replacementsMap) throws AmbariException {
+ if ((value != null) && (replacementsMap != null) && !replacementsMap.isEmpty()) {
+ int count = 0; // Used to help prevent an infinite loop...
+ boolean replacementPerformed;
+
+ do {
+ if (++count > 1000) {
+ throw new AmbariException(String.format("Circular reference found while replacing variables in %s", value));
+ }
+
+ Matcher matcher = PATTERN_VARIABLE.matcher(value);
+ StringBuffer sb = new StringBuffer();
+
+ replacementPerformed = false;
+
+ while (matcher.find()) {
+ String type = matcher.group(1);
+ String name = matcher.group(2);
+ String function = matcher.group(3);
+
+ Map<String, String> replacements;
+
+ if ((name != null) && !name.isEmpty()) {
+ if (type == null) {
+ replacements = replacementsMap.get("");
+ } else {
+ replacements = replacementsMap.get(type);
+ }
+
+ if (replacements != null) {
+ String replacement = replacements.get(name);
+
+ if (replacement != null) {
+ if (function != null) {
+ replacement = applyReplacementFunction(function, replacement);
+ }
+
+ // Escape '$' and '\' so they don't cause any issues.
+ matcher.appendReplacement(sb, replacement.replace("\\", "\\\\").replace("$", "\\$"));
+ replacementPerformed = true;
+ }
+ }
+ }
+ }
+
+ matcher.appendTail(sb);
+ value = sb.toString();
+ }
+ while (replacementPerformed); // Process the string again to make sure new variables were not introduced
+ }
+
+ return value;
+ }
+
+ /**
+ * Applies the specified replacement function to the supplied data.
+ * <p/>
+ * The function must be in the following format:
+ * <code>
+ * name(arg1,arg2,arg3...)
+ * </code>
+ * <p/>
+ * Commas in arguments should be escaped with a '/'.
+ *
+ * @param function the name and arguments of the function
+ * @param replacement the data to use in the function
+ * @return a new string generated by appling the function
+ */
+ private String applyReplacementFunction(String function, String replacement) {
+ if (function != null) {
+ Matcher matcher = PATTERN_FUNCTION.matcher(function);
+
+ if (matcher.matches()) {
+ String name = matcher.group(1);
+
+ if (name != null) {
+ Function f = FUNCTIONS.get(name);
+
+ if (f != null) {
+ String args = matcher.group(2);
+ String[] argsList = args.split("(?<!\\\\),");
+
+ // Remove escape character from '\,'
+ for (int i = 0; i < argsList.length; i++) {
+ argsList[i] = argsList[i].trim().replace("\\,", ",");
+ }
+
+ return f.perform(argsList, replacement);
+ }
+ }
+ }
+ }
+
+ return replacement;
+ }
+
+ /**
+ * Function is the interface to be implemented by replacement functions.
+ */
+ private interface Function {
+ /**
+ * Perform the function to generate a new string by applying the logic of this function to the
+ * supplied data.
+ *
+ * @param args an array of arguments, specific to the function
+ * @param data the data to apply the function logic to
+ * @return the resulting string
+ */
+ String perform(String[] args, String data);
+ }
+
+ /**
+ * EachFunction is a Function implementation that iterates over a list of values pulled from a
+ * delimited string to yield a new string.
+ * <p/>
+ * This function expects the following arguments (in order) within the args array:
+ * <ol>
+ * <li>pattern to use for each item, see {@link String#format(String, Object...)}</li>
+ * <li>delimiter to use when concatenating the resolved pattern per item</li>
+ * <li>regular expression used to split the original value</li>
+ * </ol>
+ */
+ private static class EachFunction implements Function {
+ @Override
+ public String perform(String[] args, String data) {
+ if ((args == null) || (args.length != 3)) {
+ throw new IllegalArgumentException("Invalid number of arguments encountered");
+ }
+
+ if (data != null) {
+ StringBuilder builder = new StringBuilder();
+
+ String pattern = args[0];
+ String concatDelimiter = args[1];
+ String dataDelimiter = args[2];
+
+ String[] items = data.split(dataDelimiter);
+
+ for (String item : items) {
+ if (builder.length() > 0) {
+ builder.append(concatDelimiter);
+ }
+
+ builder.append(String.format(pattern, item));
+ }
+
+ return builder.toString();
+ }
+
+ return "";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/dac2f08c/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json
index 932b71b..84821a5 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json
@@ -21,7 +21,7 @@
{
"webhcat-site": {
"templeton.kerberos.secret": "secret",
- "templeton.hive.properties": "hive.metastore.local=false,hive.metastore.uris=thrift://${clusterHostInfo/hive_metastore_host}:9083,hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}"
+ "templeton.hive.properties": "hive.metastore.local=false,hive.metastore.uris=${clusterHostInfo/hive_metastore_host|each(thrift://%s:9083, \\\\,, \\s*\\,\\s*)},hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}"
}
},
{
http://git-wip-us.apache.org/repos/asf/ambari/blob/dac2f08c/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorTest.java
index 637facc..d6a6f41 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorTest.java
@@ -318,128 +318,7 @@ public class KerberosDescriptorTest {
validateUpdatedData(descriptor);
}
- @Test
- public void testReplaceVariables() throws AmbariException {
- Map<String, Map<String, String>> configurations = new HashMap<String, Map<String, String>>() {
- {
- put("", new HashMap<String, String>() {{
- put("global_variable", "Hello World");
- put("variable-name", "dash");
- put("variable_name", "underscore");
- put("variable.name", "dot");
- }});
-
- put("config_type", new HashMap<String, String>() {{
- put("variable-name", "config_type_dash");
- put("variable_name", "config_type_underscore");
- put("variable.name", "config_type_dot");
- }});
-
- put("config.type", new HashMap<String, String>() {{
- put("variable-name", "config.type_dash");
- put("variable_name", "config.type_underscore");
- put("variable.name", "config.type_dot");
- }});
-
- put("config-type", new HashMap<String, String>() {{
- put("variable.name", "Replacement1");
- put("variable.name1", "${config-type2/variable.name}");
- put("variable.name2", "");
- }});
-
- put("config-type2", new HashMap<String, String>() {{
- put("variable.name", "Replacement2");
- put("self_reference", "${config-type2/self_reference}"); // This essentially references itself.
- put("${config-type/variable.name}_reference", "Replacement in the key");
- }});
- }
- };
-
- Assert.assertEquals("concrete",
- KerberosDescriptor.replaceVariables("concrete", configurations));
-
- Assert.assertEquals("Hello World",
- KerberosDescriptor.replaceVariables("${global_variable}", configurations));
-
- Assert.assertEquals("Replacement1",
- KerberosDescriptor.replaceVariables("${config-type/variable.name}", configurations));
-
- Assert.assertEquals("Replacement1|Replacement2",
- KerberosDescriptor.replaceVariables("${config-type/variable.name}|${config-type2/variable.name}", configurations));
-
- Assert.assertEquals("Replacement1|Replacement2|${config-type3/variable.name}",
- KerberosDescriptor.replaceVariables("${config-type/variable.name}|${config-type2/variable.name}|${config-type3/variable.name}", configurations));
-
- Assert.assertEquals("Replacement2|Replacement2",
- KerberosDescriptor.replaceVariables("${config-type/variable.name1}|${config-type2/variable.name}", configurations));
-
- Assert.assertEquals("Replacement1_reference",
- KerberosDescriptor.replaceVariables("${config-type/variable.name}_reference", configurations));
-
- Assert.assertEquals("dash",
- KerberosDescriptor.replaceVariables("${variable-name}", configurations));
-
- Assert.assertEquals("underscore",
- KerberosDescriptor.replaceVariables("${variable_name}", configurations));
-
- Assert.assertEquals("config_type_dot",
- KerberosDescriptor.replaceVariables("${config_type/variable.name}", configurations));
-
- Assert.assertEquals("config_type_dash",
- KerberosDescriptor.replaceVariables("${config_type/variable-name}", configurations));
-
- Assert.assertEquals("config_type_underscore",
- KerberosDescriptor.replaceVariables("${config_type/variable_name}", configurations));
-
- Assert.assertEquals("config.type_dot",
- KerberosDescriptor.replaceVariables("${config.type/variable.name}", configurations));
-
- Assert.assertEquals("config.type_dash",
- KerberosDescriptor.replaceVariables("${config.type/variable-name}", configurations));
-
- Assert.assertEquals("config.type_underscore",
- KerberosDescriptor.replaceVariables("${config.type/variable_name}", configurations));
-
- Assert.assertEquals("dot",
- KerberosDescriptor.replaceVariables("${variable.name}", configurations));
-
- // Replacement yields an empty string
- Assert.assertEquals("",
- KerberosDescriptor.replaceVariables("${config-type/variable.name2}", configurations));
-
-
- // This might cause an infinite loop... we assume protection is in place...
- try {
- Assert.assertEquals("${config-type2/self_reference}",
- KerberosDescriptor.replaceVariables("${config-type2/self_reference}", configurations));
- Assert.fail(String.format("%s expected to be thrown", AmbariException.class.getName()));
- } catch (AmbariException e) {
- // This is expected...
- }
- }
-
- @Test
- public void testReplaceComplicatedVariables() throws AmbariException {
- Map<String, Map<String, String>> configurations = new HashMap<String, Map<String, String>>() {
- {
- put("", new HashMap<String, String>() {{
- put("host", "c6401.ambari.apache.org");
- put("realm", "EXAMPLE.COM");
- }});
- }
- };
-
- Assert.assertEquals("hive.metastore.local=false,hive.metastore.uris=thrift://c6401.ambari.apache.org:9083,hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@EXAMPLE.COM",
- KerberosDescriptor.replaceVariables("hive.metastore.local=false,hive.metastore.uris=thrift://${host}:9083,hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}", configurations));
-
- Assert.assertEquals("Hello my realm is {EXAMPLE.COM}",
- KerberosDescriptor.replaceVariables("Hello my realm is {${realm}}", configurations));
-
- Assert.assertEquals("$c6401.ambari.apache.org",
- KerberosDescriptor.replaceVariables("$${host}", configurations));
- }
-
- @Test
+ @Test
public void testGetReferencedIdentityDescriptor() throws IOException {
URL systemResourceURL = ClassLoader.getSystemResource("kerberos/test_get_referenced_identity_descriptor.json");
Assert.assertNotNull(systemResourceURL);
http://git-wip-us.apache.org/repos/asf/ambari/blob/dac2f08c/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelperTest.java
new file mode 100644
index 0000000..cbfa4a3
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelperTest.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.state.kerberos;
+
+import junit.framework.Assert;
+import org.apache.ambari.server.AmbariException;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class VariableReplacementHelperTest {
+ VariableReplacementHelper helper = new VariableReplacementHelper();
+
+ @Test
+ public void testReplaceVariables() throws AmbariException {
+ Map<String, Map<String, String>> configurations = new HashMap<String, Map<String, String>>() {
+ {
+ put("", new HashMap<String, String>() {{
+ put("global_variable", "Hello World");
+ put("variable-name", "dash");
+ put("variable_name", "underscore");
+ put("variable.name", "dot");
+ }});
+
+ put("config_type", new HashMap<String, String>() {{
+ put("variable-name", "config_type_dash");
+ put("variable_name", "config_type_underscore");
+ put("variable.name", "config_type_dot");
+ }});
+
+ put("config.type", new HashMap<String, String>() {{
+ put("variable-name", "config.type_dash");
+ put("variable_name", "config.type_underscore");
+ put("variable.name", "config.type_dot");
+ }});
+
+ put("config-type", new HashMap<String, String>() {{
+ put("variable.name", "Replacement1");
+ put("variable.name1", "${config-type2/variable.name}");
+ put("variable.name2", "");
+ }});
+
+ put("config-type2", new HashMap<String, String>() {{
+ put("variable.name", "Replacement2");
+ put("self_reference", "${config-type2/self_reference}"); // This essentially references itself.
+ put("${config-type/variable.name}_reference", "Replacement in the key");
+ }});
+ }
+ };
+
+ Assert.assertEquals("concrete",
+ helper.replaceVariables("concrete", configurations));
+
+ Assert.assertEquals("Hello World",
+ helper.replaceVariables("${global_variable}", configurations));
+
+ Assert.assertEquals("Replacement1",
+ helper.replaceVariables("${config-type/variable.name}", configurations));
+
+ Assert.assertEquals("Replacement1|Replacement2",
+ helper.replaceVariables("${config-type/variable.name}|${config-type2/variable.name}", configurations));
+
+ Assert.assertEquals("Replacement1|Replacement2|${config-type3/variable.name}",
+ helper.replaceVariables("${config-type/variable.name}|${config-type2/variable.name}|${config-type3/variable.name}", configurations));
+
+ Assert.assertEquals("Replacement2|Replacement2",
+ helper.replaceVariables("${config-type/variable.name1}|${config-type2/variable.name}", configurations));
+
+ Assert.assertEquals("Replacement1_reference",
+ helper.replaceVariables("${config-type/variable.name}_reference", configurations));
+
+ Assert.assertEquals("dash",
+ helper.replaceVariables("${variable-name}", configurations));
+
+ Assert.assertEquals("underscore",
+ helper.replaceVariables("${variable_name}", configurations));
+
+ Assert.assertEquals("config_type_dot",
+ helper.replaceVariables("${config_type/variable.name}", configurations));
+
+ Assert.assertEquals("config_type_dash",
+ helper.replaceVariables("${config_type/variable-name}", configurations));
+
+ Assert.assertEquals("config_type_underscore",
+ helper.replaceVariables("${config_type/variable_name}", configurations));
+
+ Assert.assertEquals("config.type_dot",
+ helper.replaceVariables("${config.type/variable.name}", configurations));
+
+ Assert.assertEquals("config.type_dash",
+ helper.replaceVariables("${config.type/variable-name}", configurations));
+
+ Assert.assertEquals("config.type_underscore",
+ helper.replaceVariables("${config.type/variable_name}", configurations));
+
+ Assert.assertEquals("dot",
+ helper.replaceVariables("${variable.name}", configurations));
+
+ // Replacement yields an empty string
+ Assert.assertEquals("",
+ helper.replaceVariables("${config-type/variable.name2}", configurations));
+
+
+ // This might cause an infinite loop... we assume protection is in place...
+ try {
+ Assert.assertEquals("${config-type2/self_reference}",
+ helper.replaceVariables("${config-type2/self_reference}", configurations));
+ Assert.fail(String.format("%s expected to be thrown", AmbariException.class.getName()));
+ } catch (AmbariException e) {
+ // This is expected...
+ }
+ }
+
+ @Test
+ public void testReplaceComplicatedVariables() throws AmbariException {
+ Map<String, Map<String, String>> configurations = new HashMap<String, Map<String, String>>() {
+ {
+ put("", new HashMap<String, String>() {{
+ put("host", "c6401.ambari.apache.org");
+ put("realm", "EXAMPLE.COM");
+ }});
+ }
+ };
+
+ Assert.assertEquals("hive.metastore.local=false,hive.metastore.uris=thrift://c6401.ambari.apache.org:9083,hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@EXAMPLE.COM",
+ helper.replaceVariables("hive.metastore.local=false,hive.metastore.uris=thrift://${host}:9083,hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}", configurations));
+
+ Assert.assertEquals("Hello my realm is {EXAMPLE.COM}",
+ helper.replaceVariables("Hello my realm is {${realm}}", configurations));
+
+ Assert.assertEquals("$c6401.ambari.apache.org",
+ helper.replaceVariables("$${host}", configurations));
+ }
+
+ @Test
+ public void testReplaceVariablesWithFunctions() throws AmbariException {
+ Map<String, Map<String, String>> configurations = new HashMap<String, Map<String, String>>() {
+ {
+ put("", new HashMap<String, String>() {{
+ put("delimited.data", "one,two,three,four");
+ put("realm", "UNIT.TEST");
+ }});
+
+ put("clusterHostInfo", new HashMap<String, String>() {{
+ put("hive_metastore_host", "host1.unit.test, host2.unit.test , host3.unit.test"); // spaces are there on purpose.
+ }});
+ }
+ };
+
+ Assert.assertEquals("test=thrift://one:9083\\,thrift://two:9083\\,thrift://three:9083\\,thrift://four:9083",
+ helper.replaceVariables("test=${delimited.data|each(thrift://%s:9083, \\\\,, \\s*\\,\\s*)}", configurations));
+
+ Assert.assertEquals("hive.metastore.local=false,hive.metastore.uris=thrift://host1.unit.test:9083\\,thrift://host2.unit.test:9083\\,thrift://host3.unit.test:9083,hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@UNIT.TEST",
+ helper.replaceVariables("hive.metastore.local=false,hive.metastore.uris=${clusterHostInfo/hive_metastore_host | each(thrift://%s:9083, \\\\,, \\s*\\,\\s*)},hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}", configurations));
+ }
+
+}
\ No newline at end of file
[2/2] ambari git commit: Revert "AMBARI-11202.
templeton.hive.properties property value substitution should be done by
ambari-server (rlevas)"
Posted by rl...@apache.org.
Revert "AMBARI-11202. templeton.hive.properties property value substitution should be done by ambari-server (rlevas)"
This reverts commit 6c7f47c3601fca103c5ec551727a1a745272410c.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/64ed815a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/64ed815a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/64ed815a
Branch: refs/heads/trunk
Commit: 64ed815a844412a63c05eac692a762c1245e6652
Parents: a40b2b6
Author: Robert Levas <rl...@hortonworks.com>
Authored: Wed May 20 13:30:10 2015 -0400
Committer: Robert Levas <rl...@hortonworks.com>
Committed: Wed May 20 13:31:06 2015 -0400
----------------------------------------------------------------------
.../server/controller/KerberosHelper.java | 56 ++++-----
.../kerberos/AbstractKerberosDescriptor.java | 87 +++++++++++++
.../kerberos/KerberosKeytabDescriptor.java | 4 +-
.../kerberos/KerberosPrincipalDescriptor.java | 4 +-
.../HIVE/0.12.0.2.0/kerberos.json | 2 +-
.../state/kerberos/KerberosDescriptorTest.java | 123 ++++++++++++++++++-
.../main/admin/kerberos/step4_controller.js | 18 ++-
7 files changed, 256 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/64ed815a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
index 141803b..e083b0e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
@@ -92,7 +92,6 @@ import org.apache.ambari.server.state.kerberos.KerberosIdentityDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosKeytabDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosPrincipalDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor;
-import org.apache.ambari.server.state.kerberos.VariableReplacementHelper;
import org.apache.ambari.server.state.svccomphost.ServiceComponentHostServerActionEvent;
import org.apache.ambari.server.utils.StageUtils;
import org.apache.commons.io.FileUtils;
@@ -181,9 +180,6 @@ public class KerberosHelper {
@Inject
private KerberosConfigDataFileWriterFactory kerberosConfigDataFileWriterFactory;
- @Inject
- private VariableReplacementHelper variableReplacementHelper;
-
/**
* Used to get kerberos descriptors associated with the cluster or stack.
* Currently not available via injection.
@@ -1089,9 +1085,9 @@ public class KerberosHelper {
// Add the relevant principal name and keytab file data to the command params state
if (!commandParameters.containsKey("principal_name") || !commandParameters.containsKey("keytab_file")) {
commandParameters.put("principal_name",
- variableReplacementHelper.replaceVariables(identity.getPrincipalDescriptor().getValue(), configurations));
+ KerberosDescriptor.replaceVariables(identity.getPrincipalDescriptor().getValue(), configurations));
commandParameters.put("keytab_file",
- variableReplacementHelper.replaceVariables(identity.getKeytabDescriptor().getFile(), configurations));
+ KerberosDescriptor.replaceVariables(identity.getKeytabDescriptor().getFile(), configurations));
}
serviceComponentHostsToProcess.add(sch);
@@ -1404,7 +1400,7 @@ public class KerberosHelper {
* Merges configuration from a Map of configuration updates into a main configurations Map. Each
* property in the updates Map is processed to replace variables using the replacement Map.
* <p/>
- * See {@link VariableReplacementHelper#replaceVariables(String, Map)}
+ * See {@link org.apache.ambari.server.state.kerberos.KerberosDescriptor#replaceVariables(String, java.util.Map)}
* for information on variable replacement.
*
* @param configurations a Map of configurations
@@ -1441,7 +1437,7 @@ public class KerberosHelper {
* Merges the specified configuration property in a map of configuration types.
* The supplied property is processed to replace variables using the replacement Map.
* <p/>
- * See {@link VariableReplacementHelper#replaceVariables(String, Map)}
+ * See {@link org.apache.ambari.server.state.kerberos.KerberosDescriptor#replaceVariables(String, java.util.Map)}
* for information on variable replacement.
*
* @param configurations the Map of configuration types to update
@@ -1470,7 +1466,7 @@ public class KerberosHelper {
* Merges configuration from a Map of configuration updates into a main configurations Map. Each
* property in the updates Map is processed to replace variables using the replacement Map.
* <p/>
- * See {@link VariableReplacementHelper#replaceVariables(String, Map)}
+ * See {@link org.apache.ambari.server.state.kerberos.KerberosDescriptor#replaceVariables(String, java.util.Map)}
* for information on variable replacement.
*
* @param configurations a Map of configurations
@@ -1491,8 +1487,8 @@ public class KerberosHelper {
for (Map.Entry<String, String> property : updates.entrySet()) {
existingProperties.put(
- variableReplacementHelper.replaceVariables(property.getKey(), replacements),
- variableReplacementHelper.replaceVariables(property.getValue(), replacements)
+ KerberosDescriptor.replaceVariables(property.getKey(), replacements),
+ KerberosDescriptor.replaceVariables(property.getValue(), replacements)
);
}
}
@@ -1536,9 +1532,9 @@ public class KerberosHelper {
String principalConfiguration = null;
if (principalDescriptor != null) {
- principal = variableReplacementHelper.replaceVariables(principalDescriptor.getValue(), configurations);
+ principal = KerberosDescriptor.replaceVariables(principalDescriptor.getValue(), configurations);
principalType = principalDescriptor.getType().name().toLowerCase();
- principalConfiguration = variableReplacementHelper.replaceVariables(principalDescriptor.getConfiguration(), configurations);
+ principalConfiguration = KerberosDescriptor.replaceVariables(principalDescriptor.getConfiguration(), configurations);
}
if (principal != null) {
@@ -1552,12 +1548,12 @@ public class KerberosHelper {
boolean keytabIsCachable = false;
if (keytabDescriptor != null) {
- keytabFilePath = variableReplacementHelper.replaceVariables(keytabDescriptor.getFile(), configurations);
- keytabFileOwnerName = variableReplacementHelper.replaceVariables(keytabDescriptor.getOwnerName(), configurations);
- keytabFileOwnerAccess = variableReplacementHelper.replaceVariables(keytabDescriptor.getOwnerAccess(), configurations);
- keytabFileGroupName = variableReplacementHelper.replaceVariables(keytabDescriptor.getGroupName(), configurations);
- keytabFileGroupAccess = variableReplacementHelper.replaceVariables(keytabDescriptor.getGroupAccess(), configurations);
- keytabFileConfiguration = variableReplacementHelper.replaceVariables(keytabDescriptor.getConfiguration(), configurations);
+ keytabFilePath = KerberosDescriptor.replaceVariables(keytabDescriptor.getFile(), configurations);
+ keytabFileOwnerName = KerberosDescriptor.replaceVariables(keytabDescriptor.getOwnerName(), configurations);
+ keytabFileOwnerAccess = KerberosDescriptor.replaceVariables(keytabDescriptor.getOwnerAccess(), configurations);
+ keytabFileGroupName = KerberosDescriptor.replaceVariables(keytabDescriptor.getGroupName(), configurations);
+ keytabFileGroupAccess = KerberosDescriptor.replaceVariables(keytabDescriptor.getGroupAccess(), configurations);
+ keytabFileConfiguration = KerberosDescriptor.replaceVariables(keytabDescriptor.getConfiguration(), configurations);
keytabIsCachable = keytabDescriptor.isCachable();
}
@@ -1612,8 +1608,8 @@ public class KerberosHelper {
KerberosPrincipalDescriptor principalDescriptor = identity.getPrincipalDescriptor();
if (principalDescriptor != null) {
authToLocalBuilder.addRule(
- variableReplacementHelper.replaceVariables(principalDescriptor.getValue(), configurations),
- variableReplacementHelper.replaceVariables(principalDescriptor.getLocalUsername(), configurations));
+ KerberosDescriptor.replaceVariables(principalDescriptor.getValue(), configurations),
+ KerberosDescriptor.replaceVariables(principalDescriptor.getLocalUsername(), configurations));
}
}
}
@@ -2077,7 +2073,7 @@ public class KerberosHelper {
String principal = null;
if (principalDescriptor != null) {
- principal = variableReplacementHelper.replaceVariables(principalDescriptor.getValue(), configurations);
+ principal = KerberosDescriptor.replaceVariables(principalDescriptor.getValue(), configurations);
}
if (principal != null) {
@@ -2085,7 +2081,7 @@ public class KerberosHelper {
String keytabFile = null;
if (keytabDescriptor != null) {
- keytabFile = variableReplacementHelper.replaceVariables(keytabDescriptor.getFile(), configurations);
+ keytabFile = KerberosDescriptor.replaceVariables(keytabDescriptor.getFile(), configurations);
}
if (replaceHostNames) {
@@ -2098,8 +2094,8 @@ public class KerberosHelper {
KerberosPrincipalDescriptor resolvedPrincipalDescriptor =
new KerberosPrincipalDescriptor(principal,
principalDescriptor.getType(),
- variableReplacementHelper.replaceVariables(principalDescriptor.getConfiguration(), configurations),
- variableReplacementHelper.replaceVariables(principalDescriptor.getLocalUsername(), configurations));
+ KerberosDescriptor.replaceVariables(principalDescriptor.getConfiguration(), configurations),
+ KerberosDescriptor.replaceVariables(principalDescriptor.getLocalUsername(), configurations));
KerberosKeytabDescriptor resolvedKeytabDescriptor;
@@ -2109,11 +2105,11 @@ public class KerberosHelper {
resolvedKeytabDescriptor =
new KerberosKeytabDescriptor(
keytabFile,
- variableReplacementHelper.replaceVariables(keytabDescriptor.getOwnerName(), configurations),
- variableReplacementHelper.replaceVariables(keytabDescriptor.getOwnerAccess(), configurations),
- variableReplacementHelper.replaceVariables(keytabDescriptor.getGroupName(), configurations),
- variableReplacementHelper.replaceVariables(keytabDescriptor.getGroupAccess(), configurations),
- variableReplacementHelper.replaceVariables(keytabDescriptor.getConfiguration(), configurations),
+ KerberosDescriptor.replaceVariables(keytabDescriptor.getOwnerName(), configurations),
+ KerberosDescriptor.replaceVariables(keytabDescriptor.getOwnerAccess(), configurations),
+ KerberosDescriptor.replaceVariables(keytabDescriptor.getGroupName(), configurations),
+ KerberosDescriptor.replaceVariables(keytabDescriptor.getGroupAccess(), configurations),
+ KerberosDescriptor.replaceVariables(keytabDescriptor.getConfiguration(), configurations),
keytabDescriptor.isCachable());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/64ed815a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java
index 79b9a55..b49fec1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java
@@ -18,8 +18,20 @@
package org.apache.ambari.server.state.kerberos;
+import com.google.gson.Gson;
+import com.google.gson.JsonSyntaxException;
+import com.google.gson.reflect.TypeToken;
+import org.apache.ambari.server.AmbariException;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
/**
* AbstractKerberosDescriptor is the base class for all Kerberos*Descriptor and associated classes.
@@ -30,6 +42,11 @@ import java.util.Map;
public abstract class AbstractKerberosDescriptor {
/**
+ * a regular expression Pattern used to find "variable" placeholders in strings
+ */
+ private static final Pattern PATTERN_VARIABLE = Pattern.compile("\\$\\{(?:([\\w\\-\\.]+)/)?([\\w\\-\\.]+)\\}");
+
+ /**
* An AbstractKerberosDescriptor serving as the parent (or container) for this
* AbstractKerberosDescriptor.
* <p/>
@@ -45,6 +62,76 @@ public abstract class AbstractKerberosDescriptor {
private String name = null;
/**
+ * Performs variable replacement on the supplied String value using values from the replacementsMap.
+ * <p/>
+ * The value is a String containing one or more "variables" in the form of ${variable_name}, such
+ * that "variable_name" may indicate a group identifier; else "" is used as the group.
+ * For example:
+ * <p/>
+ * variable_name: group: ""; property: "variable_name"
+ * group1/variable_name: group: "group1"; property: "variable_name"
+ * root/group1/variable_name: Not Supported
+ * <p/>
+ * The replacementsMap is a Map of Maps creating a (small) hierarchy of data to traverse in order
+ * to resolve the variable.
+ * <p/>
+ * If a variable resolves to one or more variables, that new variable(s) will be processed and replaced.
+ * If variable exists after a set number of iterations it is assumed that a cycle has been created
+ * and the process will abort returning a String in a possibly unexpected state.
+ *
+ * @param value a String containing zero or more variables to be replaced
+ * @param replacementsMap a Map of data used to perform the variable replacements
+ * @return a new String
+ */
+ public static String replaceVariables(String value, Map<String, Map<String, String>> replacementsMap) throws AmbariException {
+ if ((value != null) && (replacementsMap != null) && !replacementsMap.isEmpty()) {
+ int count = 0; // Used to help prevent an infinite loop...
+ boolean replacementPerformed;
+
+ do {
+ if (++count > 1000) {
+ throw new AmbariException(String.format("Circular reference found while replacing variables in %s", value));
+ }
+
+ Matcher matcher = PATTERN_VARIABLE.matcher(value);
+ StringBuffer sb = new StringBuffer();
+
+ replacementPerformed = false;
+
+ while (matcher.find()) {
+ String type = matcher.group(1);
+ String name = matcher.group(2);
+ Map<String, String> replacements;
+
+ if ((name != null) && !name.isEmpty()) {
+ if (type == null) {
+ replacements = replacementsMap.get("");
+ } else {
+ replacements = replacementsMap.get(type);
+ }
+
+ if (replacements != null) {
+ String replacement = replacements.get(name);
+
+ if (replacement != null) {
+ // Escape '$' and '\' so they don't cause any issues.
+ matcher.appendReplacement(sb, replacement.replace("\\", "\\\\").replace("$", "\\$"));
+ replacementPerformed = true;
+ }
+ }
+ }
+ }
+
+ matcher.appendTail(sb);
+ value = sb.toString();
+ }
+ while (replacementPerformed); // Process the string again to make sure new variables were not introduced
+ }
+
+ return value;
+ }
+
+ /**
* Generates a Map of data that represents this AbstractKerberosDescriptor implementation.
* <p/>
* This method should be overwritten by AbstractKerberosDescriptor implementations to generate a
http://git-wip-us.apache.org/repos/asf/ambari/blob/64ed815a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java
index 7ce1c9f..404efa2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java
@@ -220,12 +220,12 @@ public class KerberosKeytabDescriptor extends AbstractKerberosDescriptor {
* <ul>
* <li>
* ${variable} placeholders are replaced on the server - see
- * {@link VariableReplacementHelper#replaceVariables(String, Map)}
+ * {@link org.apache.ambari.server.state.kerberos.KerberosDescriptor#replaceVariables(String, java.util.Map)}
* </li>
* </ul>
*
* @return a String declaring the keytab file's absolute path
- * @see VariableReplacementHelper#replaceVariables(String, Map)
+ * @see org.apache.ambari.server.state.kerberos.KerberosDescriptor#replaceVariables(String, java.util.Map)
*/
public String getFile() {
return getName();
http://git-wip-us.apache.org/repos/asf/ambari/blob/64ed815a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java
index 09f6872..6e5ac5c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java
@@ -138,13 +138,13 @@ public class KerberosPrincipalDescriptor extends AbstractKerberosDescriptor {
* <ul>
* <li>
* ${variable} placeholders are replaced on the server - see
- * {@link VariableReplacementHelper#replaceVariables(String, Map)}
+ * {@link org.apache.ambari.server.state.kerberos.KerberosDescriptor#replaceVariables(String, java.util.Map)}
* </li>
* <li>the _HOST placeholder is replaced on the hosts to dynamically populate the relevant hostname</li>
* </ul>
*
* @return a String declaring this principal's value
- * @see VariableReplacementHelper#replaceVariables(String, Map)
+ * @see org.apache.ambari.server.state.kerberos.KerberosDescriptor#replaceVariables(String, java.util.Map)
*/
public String getValue() {
return getName();
http://git-wip-us.apache.org/repos/asf/ambari/blob/64ed815a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json
index 84821a5..932b71b 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json
@@ -21,7 +21,7 @@
{
"webhcat-site": {
"templeton.kerberos.secret": "secret",
- "templeton.hive.properties": "hive.metastore.local=false,hive.metastore.uris=${clusterHostInfo/hive_metastore_host|each(thrift://%s:9083, \\\\,, \\s*\\,\\s*)},hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}"
+ "templeton.hive.properties": "hive.metastore.local=false,hive.metastore.uris=thrift://${clusterHostInfo/hive_metastore_host}:9083,hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}"
}
},
{
http://git-wip-us.apache.org/repos/asf/ambari/blob/64ed815a/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorTest.java
index d6a6f41..637facc 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorTest.java
@@ -318,7 +318,128 @@ public class KerberosDescriptorTest {
validateUpdatedData(descriptor);
}
- @Test
+ @Test
+ public void testReplaceVariables() throws AmbariException {
+ Map<String, Map<String, String>> configurations = new HashMap<String, Map<String, String>>() {
+ {
+ put("", new HashMap<String, String>() {{
+ put("global_variable", "Hello World");
+ put("variable-name", "dash");
+ put("variable_name", "underscore");
+ put("variable.name", "dot");
+ }});
+
+ put("config_type", new HashMap<String, String>() {{
+ put("variable-name", "config_type_dash");
+ put("variable_name", "config_type_underscore");
+ put("variable.name", "config_type_dot");
+ }});
+
+ put("config.type", new HashMap<String, String>() {{
+ put("variable-name", "config.type_dash");
+ put("variable_name", "config.type_underscore");
+ put("variable.name", "config.type_dot");
+ }});
+
+ put("config-type", new HashMap<String, String>() {{
+ put("variable.name", "Replacement1");
+ put("variable.name1", "${config-type2/variable.name}");
+ put("variable.name2", "");
+ }});
+
+ put("config-type2", new HashMap<String, String>() {{
+ put("variable.name", "Replacement2");
+ put("self_reference", "${config-type2/self_reference}"); // This essentially references itself.
+ put("${config-type/variable.name}_reference", "Replacement in the key");
+ }});
+ }
+ };
+
+ Assert.assertEquals("concrete",
+ KerberosDescriptor.replaceVariables("concrete", configurations));
+
+ Assert.assertEquals("Hello World",
+ KerberosDescriptor.replaceVariables("${global_variable}", configurations));
+
+ Assert.assertEquals("Replacement1",
+ KerberosDescriptor.replaceVariables("${config-type/variable.name}", configurations));
+
+ Assert.assertEquals("Replacement1|Replacement2",
+ KerberosDescriptor.replaceVariables("${config-type/variable.name}|${config-type2/variable.name}", configurations));
+
+ Assert.assertEquals("Replacement1|Replacement2|${config-type3/variable.name}",
+ KerberosDescriptor.replaceVariables("${config-type/variable.name}|${config-type2/variable.name}|${config-type3/variable.name}", configurations));
+
+ Assert.assertEquals("Replacement2|Replacement2",
+ KerberosDescriptor.replaceVariables("${config-type/variable.name1}|${config-type2/variable.name}", configurations));
+
+ Assert.assertEquals("Replacement1_reference",
+ KerberosDescriptor.replaceVariables("${config-type/variable.name}_reference", configurations));
+
+ Assert.assertEquals("dash",
+ KerberosDescriptor.replaceVariables("${variable-name}", configurations));
+
+ Assert.assertEquals("underscore",
+ KerberosDescriptor.replaceVariables("${variable_name}", configurations));
+
+ Assert.assertEquals("config_type_dot",
+ KerberosDescriptor.replaceVariables("${config_type/variable.name}", configurations));
+
+ Assert.assertEquals("config_type_dash",
+ KerberosDescriptor.replaceVariables("${config_type/variable-name}", configurations));
+
+ Assert.assertEquals("config_type_underscore",
+ KerberosDescriptor.replaceVariables("${config_type/variable_name}", configurations));
+
+ Assert.assertEquals("config.type_dot",
+ KerberosDescriptor.replaceVariables("${config.type/variable.name}", configurations));
+
+ Assert.assertEquals("config.type_dash",
+ KerberosDescriptor.replaceVariables("${config.type/variable-name}", configurations));
+
+ Assert.assertEquals("config.type_underscore",
+ KerberosDescriptor.replaceVariables("${config.type/variable_name}", configurations));
+
+ Assert.assertEquals("dot",
+ KerberosDescriptor.replaceVariables("${variable.name}", configurations));
+
+ // Replacement yields an empty string
+ Assert.assertEquals("",
+ KerberosDescriptor.replaceVariables("${config-type/variable.name2}", configurations));
+
+
+ // This might cause an infinite loop... we assume protection is in place...
+ try {
+ Assert.assertEquals("${config-type2/self_reference}",
+ KerberosDescriptor.replaceVariables("${config-type2/self_reference}", configurations));
+ Assert.fail(String.format("%s expected to be thrown", AmbariException.class.getName()));
+ } catch (AmbariException e) {
+ // This is expected...
+ }
+ }
+
+ @Test
+ public void testReplaceComplicatedVariables() throws AmbariException {
+ Map<String, Map<String, String>> configurations = new HashMap<String, Map<String, String>>() {
+ {
+ put("", new HashMap<String, String>() {{
+ put("host", "c6401.ambari.apache.org");
+ put("realm", "EXAMPLE.COM");
+ }});
+ }
+ };
+
+ Assert.assertEquals("hive.metastore.local=false,hive.metastore.uris=thrift://c6401.ambari.apache.org:9083,hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@EXAMPLE.COM",
+ KerberosDescriptor.replaceVariables("hive.metastore.local=false,hive.metastore.uris=thrift://${host}:9083,hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}", configurations));
+
+ Assert.assertEquals("Hello my realm is {EXAMPLE.COM}",
+ KerberosDescriptor.replaceVariables("Hello my realm is {${realm}}", configurations));
+
+ Assert.assertEquals("$c6401.ambari.apache.org",
+ KerberosDescriptor.replaceVariables("$${host}", configurations));
+ }
+
+ @Test
public void testGetReferencedIdentityDescriptor() throws IOException {
URL systemResourceURL = ClassLoader.getSystemResource("kerberos/test_get_referenced_identity_descriptor.json");
Assert.assertNotNull(systemResourceURL);
http://git-wip-us.apache.org/repos/asf/ambari/blob/64ed815a/ambari-web/app/controllers/main/admin/kerberos/step4_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/admin/kerberos/step4_controller.js b/ambari-web/app/controllers/main/admin/kerberos/step4_controller.js
index 0ea1ee2..a48baaf 100644
--- a/ambari-web/app/controllers/main/admin/kerberos/step4_controller.js
+++ b/ambari-web/app/controllers/main/admin/kerberos/step4_controller.js
@@ -196,10 +196,24 @@ App.KerberosWizardStep4Controller = App.WizardStep7Controller.extend(App.AddSecu
},
/**
- * Function to override kerberos descriptor property values
+ * Function to override kerberos descriptor's property values
*/
tweakConfigProperty: function(config) {
- // This is a placeholder in the event kerberos descriptor property values need to be changed
+ if (config.name === 'templeton.hive.properties') {
+ var defaultHiveMsPort = "9083";
+ var hiveMSHosts = App.HostComponent.find().filterProperty('componentName', 'HIVE_METASTORE');
+ if (hiveMSHosts.length > 1) {
+ var hiveMSHostNames = hiveMSHosts.mapProperty('hostName');
+ var port = config.value.match(/:[0-9]{2,4}/);
+ port = port ? port[0].slice(1) : defaultHiveMsPort;
+ for (var i = 0; i < hiveMSHostNames.length; i++) {
+ hiveMSHostNames[i] = "thrift://" + hiveMSHostNames[i] + ":" + port;
+ }
+ var configValue = config.value.replace(/thrift.+[0-9]{2,},/i, hiveMSHostNames.join('\\,') + ",");
+ config.set('value', configValue);
+ config.set('recommendedValue', configValue);
+ }
+ }
},
/**