You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2018/05/03 15:59:05 UTC
[ambari] branch trunk updated: [AMBARI-23745] - Replace Multiple
Matches Using Regex on Upgrades (#1160)
This is an automated email from the ASF dual-hosted git repository.
jonathanhurley pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 27d5a86 [AMBARI-23745] - Replace Multiple Matches Using Regex on Upgrades (#1160)
27d5a86 is described below
commit 27d5a866d03a87c200d8dc999c87d219e50bf6d8
Author: Jonathan Hurley <jo...@apache.org>
AuthorDate: Thu May 3 11:59:02 2018 -0400
[AMBARI-23745] - Replace Multiple Matches Using Regex on Upgrades (#1160)
---
.../serveraction/upgrades/ConfigureAction.java | 14 ++--
.../upgrade/ConfigUpgradeChangeDefinition.java | 68 +++++++++++++------
.../src/main/resources/upgrade-config.xsd | 1 +
.../ambari/server/state/UpgradeHelperTest.java | 78 +++++++++++++++++++++-
.../stacks/HDP/2.1.1/upgrades/config-upgrade.xml | 5 ++
.../stacks/HDP/2.1.1/upgrades/upgrade_test.xml | 1 +
6 files changed, 140 insertions(+), 27 deletions(-)
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
index 3eeca46..d3cc290 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
@@ -471,9 +471,15 @@ public class ConfigureAction extends AbstractUpgradeServerAction {
newValues.put(replacement.key, replaced);
- updateBufferWithMessage(outputBuffer,
- MessageFormat.format("Replaced {0}/{1} containing \"{2}\" with \"{3}\"", configType,
- replacement.key, replacement.find, replacement.replaceWith));
+ // customize the replacement message if the new value is empty
+ if (StringUtils.isEmpty(replacement.replaceWith)) {
+ updateBufferWithMessage(outputBuffer, MessageFormat.format(
+ "Removed \"{0}\" from {1}/{2}", replacement.find, configType, replacement.key));
+ } else {
+ updateBufferWithMessage(outputBuffer,
+ MessageFormat.format("Replaced {0}/{1} containing \"{2}\" with \"{3}\"", configType,
+ replacement.key, replacement.find, replacement.replaceWith));
+ }
}
} else {
updateBufferWithMessage(outputBuffer, MessageFormat.format(
@@ -526,7 +532,7 @@ public class ConfigureAction extends AbstractUpgradeServerAction {
newValues.put(insert.key, valueToInsertInto);
updateBufferWithMessage(outputBuffer, MessageFormat.format(
- "Updated {0}/{1} by inserting {2}", configType, insert.key, insert.value));
+ "Updated {0}/{1} by inserting \"{2}\"", configType, insert.key, insert.value));
} else {
updateBufferWithMessage(outputBuffer, MessageFormat.format(
"Skipping insertion for {0}/{1} because it does not exist or is empty.", configType,
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java
index 89b6567..ff9a123 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java
@@ -35,10 +35,11 @@ import javax.xml.bind.annotation.XmlType;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Config;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Objects;
+import com.google.common.base.MoreObjects;
/**
* The {@link ConfigUpgradeChangeDefinition} represents a configuration change. This change can be
@@ -204,7 +205,8 @@ public class ConfigUpgradeChangeDefinition {
List<Replace> list = new ArrayList<>();
for (Replace r : replacements) {
- if (null == r.key || null == r.find || null == r.replaceWith) {
+
+ if (StringUtils.isBlank(r.key) || StringUtils.isEmpty(r.find) || null == r.replaceWith) {
LOG.warn(String.format("Replacement %s is invalid", r));
continue;
}
@@ -215,48 +217,63 @@ public class ConfigUpgradeChangeDefinition {
}
/**
+ * Evaluates the {@link RegexReplace} instances defined for the upgrade and
+ * converts them into distinct {@link Replace} objects. In some cases, if the
+ * regex matches more than 1 string in the configuration, it will create
+ * multiple {@link Replace} objects, each with their own literal string to
+ * find/replace.
+ *
* @return the replacement tokens, never {@code null}
*/
public List<Replace> getRegexReplacements(Cluster cluster) {
-
if (null == regexReplacements) {
-
return Collections.emptyList();
}
List<Replace> list = new ArrayList<>();
for (RegexReplace regexReplaceObj : regexReplacements) {
- if (null == regexReplaceObj.key || null == regexReplaceObj.find || null == regexReplaceObj.replaceWith) {
+ if (StringUtils.isBlank(regexReplaceObj.key) || StringUtils.isEmpty(regexReplaceObj.find)
+ || null == regexReplaceObj.replaceWith) {
LOG.warn(String.format("Replacement %s is invalid", regexReplaceObj));
continue;
}
- try{
+ try {
Config config = cluster.getDesiredConfigByType(configType);
Map<String, String> properties = config.getProperties();
String content = properties.get(regexReplaceObj.key);
Pattern REGEX = Pattern.compile(regexReplaceObj.find, Pattern.MULTILINE);
-
Matcher patternMatchObj = REGEX.matcher(content);
- if (patternMatchObj.find() && patternMatchObj.groupCount()==1) {
- regexReplaceObj.find = patternMatchObj.group();
- Replace rep = regexReplaceObj.copyToReplaceObject();
- list.add(rep);
- }
-
- }catch(Exception e){
- String message = "getRegexReplacements : Error while fetching config properties : key - " + regexReplaceObj.key + " find - " + regexReplaceObj.find;
- LOG.error(message, e);
+ if (regexReplaceObj.matchAll) {
+ while (patternMatchObj.find()) {
+ regexReplaceObj.find = patternMatchObj.group();
+ if (StringUtils.isNotBlank(regexReplaceObj.find)) {
+ Replace rep = regexReplaceObj.copyToReplaceObject();
+ list.add(rep);
+ }
+ }
+ } else {
+ // find the first literal match and create a replacement for it
+ if (patternMatchObj.find() && patternMatchObj.groupCount() == 1) {
+ regexReplaceObj.find = patternMatchObj.group();
+ Replace rep = regexReplaceObj.copyToReplaceObject();
+ list.add(rep);
+ }
}
+ } catch (Exception e) {
+ LOG.error(String.format(
+ "There was an error while trying to execute a regex replacement for %s/%s. The regular expression was %s",
+ configType, regexReplaceObj.key, regexReplaceObj.find), e);
}
+ }
+
return list;
}
-
/**
* Gets the insertion directives.
*
@@ -319,7 +336,7 @@ public class ConfigUpgradeChangeDefinition {
@Override
public String toString() {
- return Objects.toStringHelper("Set").add("key", key)
+ return MoreObjects.toStringHelper("Set").add("key", key)
.add("value", value)
.add("ifKey", ifKey)
.add("ifType", ifType)
@@ -394,7 +411,7 @@ public class ConfigUpgradeChangeDefinition {
@Override
public String toString() {
- return Objects.toStringHelper(this).add("operation", operation)
+ return MoreObjects.toStringHelper(this).add("operation", operation)
.add("fromType", fromType)
.add("fromKey", fromKey)
.add("toKey", toKey)
@@ -437,7 +454,7 @@ public class ConfigUpgradeChangeDefinition {
@Override
public String toString() {
- return Objects.toStringHelper(this).add("key", key)
+ return MoreObjects.toStringHelper(this).add("key", key)
.add("find", find)
.add("replaceWith", replaceWith)
.add("ifKey", ifKey)
@@ -472,9 +489,16 @@ public class ConfigUpgradeChangeDefinition {
@XmlAttribute(name="replace-with")
public String replaceWith;
+ /**
+ * Find as many matching groups as possible and create replacements for each
+ * one. The default value is {@code false}.
+ */
+ @XmlAttribute(name = "match-all")
+ public boolean matchAll = false;
+
@Override
public String toString() {
- return Objects.toStringHelper(this).add("key", key)
+ return MoreObjects.toStringHelper(this).add("key", key)
.add("find", find)
.add("replaceWith",replaceWith)
.add("ifKey", ifKey)
@@ -544,7 +568,7 @@ public class ConfigUpgradeChangeDefinition {
*/
@Override
public String toString() {
- return Objects.toStringHelper(this).add("insertType", insertType)
+ return MoreObjects.toStringHelper(this).add("insertType", insertType)
.add("key", key)
.add("value",value)
.add("newlineBefore", newlineBefore)
diff --git a/ambari-server/src/main/resources/upgrade-config.xsd b/ambari-server/src/main/resources/upgrade-config.xsd
index cee7c85..c927df2 100644
--- a/ambari-server/src/main/resources/upgrade-config.xsd
+++ b/ambari-server/src/main/resources/upgrade-config.xsd
@@ -103,6 +103,7 @@
<xs:attribute name="key" use="required" type="xs:string"/>
<xs:attribute name="find" use="required" type="xs:string"/>
<xs:attribute name="replace-with" use="required" type="xs:string"/>
+ <xs:attribute name="match-all" use="optional" type="xs:boolean"/>
<xs:attribute name="if-key" use="optional" type="xs:string"/>
<xs:attribute name="if-type" use="optional" type="xs:string"/>
<xs:attribute name="if-value" use="optional" type="xs:string"/>
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
index 43a14ed..147bdd0 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
@@ -661,7 +661,7 @@ public class UpgradeHelperTest extends EasyMockSupport {
assertEquals(4, groups.get(0).items.size());
assertEquals(8, groups.get(1).items.size());
- assertEquals(5, groups.get(2).items.size());
+ assertEquals(6, groups.get(2).items.size());
assertEquals(7, groups.get(3).items.size());
assertEquals(8, groups.get(4).items.size());
}
@@ -1023,6 +1023,82 @@ public class UpgradeHelperTest extends EasyMockSupport {
assertEquals("fooValue", keyValuePairs.get(0).value);
}
+ /**
+ * Tests that the regex replacement is working for configurations.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testConfigureRegexTask() throws Exception {
+ Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1");
+ assertTrue(upgrades.containsKey("upgrade_test"));
+ UpgradePack upgrade = upgrades.get("upgrade_test");
+ ConfigUpgradePack cup = ambariMetaInfo.getConfigUpgradePack("HDP", "2.1.1");
+ assertNotNull(upgrade);
+
+ Cluster cluster = makeCluster();
+
+ UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING);
+
+ List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade,context);
+ assertEquals(7, groups.size());
+
+ // grab the regex task out of Hive
+ UpgradeGroupHolder hiveGroup = groups.get(4);
+ assertEquals("HIVE", hiveGroup.name);
+ ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(5).getTasks().get(0).getTasks().get(0);
+ assertEquals("hdp_2_1_1_regex_replace", configureTask.getId());
+
+ // now set the property in the if-check in the set element so that we have a match
+ Map<String, String> hiveConfigs = new HashMap<>();
+ StringBuilder builder = new StringBuilder();
+ builder.append("1-foo-2");
+ builder.append(System.lineSeparator());
+ builder.append("1-bar-2");
+ builder.append(System.lineSeparator());
+ builder.append("3-foo-4");
+ builder.append(System.lineSeparator());
+ builder.append("1-foobar-2");
+ builder.append(System.lineSeparator());
+ hiveConfigs.put("regex-replace-key-one", builder.toString());
+
+ ConfigurationRequest configurationRequest = new ConfigurationRequest();
+ configurationRequest.setClusterName(cluster.getClusterName());
+ configurationRequest.setType("hive-site");
+ configurationRequest.setVersionTag("version2");
+ configurationRequest.setProperties(hiveConfigs);
+
+ final ClusterRequest clusterRequest = new ClusterRequest(
+ cluster.getClusterId(), cluster.getClusterName(),
+ cluster.getDesiredStackVersion().getStackVersion(), null);
+
+ clusterRequest.setDesiredConfig(singletonList(configurationRequest));
+ m_managementController.updateClusters(new HashSet<ClusterRequest>() {
+ {
+ add(clusterRequest);
+ }
+ }, null);
+
+ // the configure task should now return different properties to set based on
+ // the if-condition checks
+ Map<String, String> configProperties = configureTask.getConfigurationChanges(cluster, cup);
+ assertFalse(configProperties.isEmpty());
+ assertEquals(configProperties.get(ConfigureTask.PARAMETER_CONFIG_TYPE), "hive-site");
+
+ String configurationJson = configProperties.get(ConfigureTask.PARAMETER_REPLACEMENTS);
+ assertNotNull(configurationJson);
+
+ List<ConfigUpgradeChangeDefinition.Replace> replacements = m_gson.fromJson(
+ configurationJson,
+ new TypeToken<List<ConfigUpgradeChangeDefinition.Replace>>() {}.getType());
+
+ assertEquals("1-foo-2\n", replacements.get(0).find);
+ assertEquals("REPLACED", replacements.get(0).replaceWith);
+ assertEquals("3-foo-4\n", replacements.get(1).find);
+ assertEquals("REPLACED", replacements.get(1).replaceWith);
+ assertEquals(2, replacements.size());
+ }
+
@Test
public void testConfigureTaskWithMultipleConfigurations() throws Exception {
Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1");
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml
index 307f4d4..777829e 100644
--- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml
+++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml
@@ -199,6 +199,11 @@
<type>hive-site</type>
<set key="fooKey" value="fooValue" if-type="hive-site" if-key="ifFooKey" if-value="ifFooValue"/>
</definition>
+
+ <definition xsi:type="configure" id="hdp_2_1_1_regex_replace">
+ <type>hive-site</type>
+ <regex-replace key="regex-replace-key-one" find="^\d-foo-\d[\r\n]+" replace-with="REPLACED" match-all="true"/>
+ </definition>
</changes>
</component>
</service>
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml
index 037e39a..da653f0 100644
--- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml
+++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml
@@ -227,6 +227,7 @@
<task xsi:type="configure" id="hdp_2_1_1_hive_server_conditions"/>
<task xsi:type="configure" id="hdp_2_1_1_hive_server_conditions_skip"/>
<task xsi:type="configure" id="hdp_2_1_1_no_conditions_met"/>
+ <task xsi:type="configure" id="hdp_2_1_1_regex_replace"/>
</pre-upgrade>
<pre-downgrade copy-upgrade="true" />
<upgrade />
--
To stop receiving notification emails like this one, please contact
jonathanhurley@apache.org.