You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2017/07/17 21:28:25 UTC
ambari git commit: AMBARI-21481. Upgrading IOP cluster with Spark2 to
Ambari 2.5.2 fails on start because config mapping spark2-javaopts-properties
is never selected (alejandro)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 13bcea0b0 -> 0ed09cd53
AMBARI-21481. Upgrading IOP cluster with Spark2 to Ambari 2.5.2 fails on start because config mapping spark2-javaopts-properties is never selected (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0ed09cd5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0ed09cd5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0ed09cd5
Branch: refs/heads/branch-2.5
Commit: 0ed09cd5342cfc4cac0d6061a7b7b9a3cef127c1
Parents: 13bcea0
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Fri Jul 14 16:15:07 2017 -0700
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Mon Jul 17 14:29:36 2017 -0700
----------------------------------------------------------------------
.../server/upgrade/UpgradeCatalog252.java | 99 ++++++++++++++++++++
.../configuration/spark-javaopts-properties.xml | 3 +
.../spark2-javaopts-properties.xml | 5 +-
.../4.2.5/services/SPARK2/metainfo.xml | 2 +-
.../configuration/spark-javaopts-properties.xml | 3 +
5 files changed, 110 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ed09cd5/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
index 3c8686c..ea1b034 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
@@ -18,12 +18,19 @@
package org.apache.ambari.server.upgrade;
import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.orm.DBAccessor.DBColumnInfo;
+import org.apache.ambari.server.orm.dao.ClusterDAO;
+import org.apache.ambari.server.orm.entities.ClusterConfigMappingEntity;
+import org.apache.ambari.server.orm.entities.ClusterEntity;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
@@ -34,6 +41,8 @@ import org.apache.commons.lang.StringUtils;
import com.google.common.collect.Sets;
import com.google.inject.Inject;
import com.google.inject.Injector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The {@link org.apache.ambari.server.upgrade.UpgradeCatalog252} upgrades Ambari from 2.5.1 to 2.5.2.
@@ -54,6 +63,13 @@ public class UpgradeCatalog252 extends AbstractUpgradeCatalog {
private static final String CLUSTER_ENV = "cluster-env";
+ private static final List<String> configTypesToEnsureSelected = Arrays.asList("spark2-javaopts-properties");
+
+ /**
+ * Logger.
+ */
+ private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog252.class);
+
/**
* Constructor.
*
@@ -102,6 +118,7 @@ public class UpgradeCatalog252 extends AbstractUpgradeCatalog {
@Override
protected void executeDMLUpdates() throws AmbariException, SQLException {
resetStackToolsAndFeatures();
+ ensureConfigTypesHaveAtLeastOneVersionSelected();
}
/**
@@ -197,4 +214,86 @@ public class UpgradeCatalog252 extends AbstractUpgradeCatalog {
updateConfigurationPropertiesForCluster(cluster, CLUSTER_ENV, newStackProperties, true, false);
}
}
+
+ /**
+ * When doing a cross-stack upgrade, we found that one config type (spark2-javaopts-properties)
+ * did not have any mappings that were selected, so it caused Ambari Server start to fail on the DB Consistency Checker.
+ * To fix this, iterate over all config types and ensure that at least one is selected.
+ * If none are selected, then pick the one with the greatest time stamp; this should be safe since we are only adding
+ * more data to use as opposed to removing.
+ */
+ private void ensureConfigTypesHaveAtLeastOneVersionSelected() {
+ ClusterDAO clusterDAO = injector.getInstance(ClusterDAO.class);
+ List<ClusterEntity> clusters = clusterDAO.findAll();
+
+ if (null == clusters) {
+ return;
+ }
+
+ for (ClusterEntity clusterEntity : clusters) {
+ LOG.info("Ensuring all config types have at least one selected config for cluster {}", clusterEntity.getClusterName());
+
+ boolean atLeastOneChanged = false;
+ Collection<ClusterConfigMappingEntity> configMappingEntities = clusterEntity.getConfigMappingEntities();
+
+ if (configMappingEntities != null) {
+ Set<String> configTypesNotSelected = new HashSet<>();
+ Set<String> configTypesWithAtLeastOneSelected = new HashSet<>();
+
+ for (ClusterConfigMappingEntity clusterConfigMappingEntity : configMappingEntities) {
+ String typeName = clusterConfigMappingEntity.getType();
+
+ if (clusterConfigMappingEntity.isSelected() == 1) {
+ configTypesWithAtLeastOneSelected.add(typeName);
+ } else {
+ configTypesNotSelected.add(typeName);
+ }
+ }
+
+ // Due to the ordering, eliminate any configs with at least one selected.
+ configTypesNotSelected.removeAll(configTypesWithAtLeastOneSelected);
+ if (!configTypesNotSelected.isEmpty()) {
+ LOG.info("The following config types have config mappings which don't have at least one as selected. {}", StringUtils.join(configTypesNotSelected, ", "));
+
+ LOG.info("Filtering only config types these config types: {}", StringUtils.join(configTypesToEnsureSelected, ", "));
+ // Get the intersection with a subset of configs that are allowed to be selected during the migration.
+ configTypesNotSelected.retainAll(configTypesToEnsureSelected);
+ }
+
+ if (!configTypesNotSelected.isEmpty()) {
+ LOG.info("The following config types have config mappings which don't have at least one as selected. {}", StringUtils.join(configTypesNotSelected, ", "));
+
+ for (String typeName : configTypesNotSelected) {
+ ClusterConfigMappingEntity clusterConfigMappingWithGreatestTimeStamp = null;
+
+ for (ClusterConfigMappingEntity clusterConfigMappingEntity : configMappingEntities) {
+ if (typeName.equals(clusterConfigMappingEntity.getType())) {
+
+ if (null == clusterConfigMappingWithGreatestTimeStamp) {
+ clusterConfigMappingWithGreatestTimeStamp = clusterConfigMappingEntity;
+ } else {
+ if (clusterConfigMappingEntity.getCreateTimestamp() >= clusterConfigMappingWithGreatestTimeStamp.getCreateTimestamp()) {
+ clusterConfigMappingWithGreatestTimeStamp = clusterConfigMappingEntity;
+ }
+ }
+ }
+ }
+
+ if (null != clusterConfigMappingWithGreatestTimeStamp) {
+ LOG.info("Saving. Config type {} has a mapping with tag {} and greatest timestamp {} that is not selected, so will mark it selected.",
+ typeName, clusterConfigMappingWithGreatestTimeStamp.getTag(), clusterConfigMappingWithGreatestTimeStamp.getCreateTimestamp());
+ atLeastOneChanged = true;
+ clusterConfigMappingWithGreatestTimeStamp.setSelected(1);
+ }
+ }
+ } else {
+ LOG.info("All config types have at least one mapping that is selected. Nothing to do.");
+ }
+ }
+
+ if (atLeastOneChanged) {
+ clusterDAO.mergeConfigMappings(configMappingEntities);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ed09cd5/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/SPARK/configuration/spark-javaopts-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/SPARK/configuration/spark-javaopts-properties.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/SPARK/configuration/spark-javaopts-properties.xml
index a197e34..c8fe152 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/SPARK/configuration/spark-javaopts-properties.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/SPARK/configuration/spark-javaopts-properties.xml
@@ -23,6 +23,9 @@
<name>content</name>
<description>Spark-javaopts-properties</description>
<value> </value>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
<on-ambari-upgrade add="true"/>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ed09cd5/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/configuration/spark2-javaopts-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/configuration/spark2-javaopts-properties.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/configuration/spark2-javaopts-properties.xml
index f8d50fc..e8c3f5a 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/configuration/spark2-javaopts-properties.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/configuration/spark2-javaopts-properties.xml
@@ -23,7 +23,10 @@
<name>content</name>
<description>Spark2-javaopts-properties</description>
<value> </value>
- <on-ambari-upgrade add="true"/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ed09cd5/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/metainfo.xml
index bf75f47..1692890 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SPARK2/metainfo.xml
@@ -79,7 +79,7 @@
<configuration-dependencies>
<config-type>spark2-defaults</config-type>
- <config-type>spark2-javaopts-properties</config-type>
+ <config-type>spark2-javaopts-properties</config-type>
<config-type>spark2-thrift-sparkconf</config-type>
</configuration-dependencies>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ed09cd5/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/SPARK/configuration/spark-javaopts-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/SPARK/configuration/spark-javaopts-properties.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/SPARK/configuration/spark-javaopts-properties.xml
index 77a7282..f7c8c7e 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/SPARK/configuration/spark-javaopts-properties.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/SPARK/configuration/spark-javaopts-properties.xml
@@ -23,6 +23,9 @@
<name>content</name>
<description>Spark-javaopts-properties</description>
<value> </value>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
<on-ambari-upgrade add="true"/>
</property>
</configuration>