You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vb...@apache.org on 2016/09/15 14:21:20 UTC
ambari git commit: AMBARI-18048. ambari-server upgrade should report
all the config properties it added/modified.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 53b34516a -> 3bf3bc593
AMBARI-18048. ambari-server upgrade should report all the config properties it added/modified.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3bf3bc59
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3bf3bc59
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3bf3bc59
Branch: refs/heads/branch-2.5
Commit: 3bf3bc593b5ea4c294547c41c03cf2bd14fbfde8
Parents: 53b3451
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Thu Sep 15 17:20:53 2016 +0300
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Thu Sep 15 17:20:53 2016 +0300
----------------------------------------------------------------------
.../server/configuration/Configuration.java | 147 +++++++++++++++++++
.../server/upgrade/AbstractUpgradeCatalog.java | 66 +++++++--
.../server/upgrade/SchemaUpgradeHelper.java | 41 +++---
.../ambari/server/upgrade/UpgradeCatalog.java | 7 +
.../ambari/server/upgrade/UpgradeTest.java | 2 +-
5 files changed, 235 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/3bf3bc59/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index b70c5f4..c5a8d65 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -20,8 +20,10 @@ package org.apache.ambari.server.configuration;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
+import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
+import java.io.Writer;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@@ -65,6 +67,7 @@ import org.apache.ambari.server.security.encryption.CredentialProvider;
import org.apache.ambari.server.state.services.MetricsRetrievalService;
import org.apache.ambari.server.state.services.RetryUpgradeActionService;
import org.apache.ambari.server.state.stack.OsFamily;
+import org.apache.ambari.server.upgrade.AbstractUpgradeCatalog;
import org.apache.ambari.server.utils.AmbariPath;
import org.apache.ambari.server.utils.DateUtils;
import org.apache.ambari.server.utils.HostUtils;
@@ -84,10 +87,14 @@ import org.apache.commons.lang.math.NumberUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -2326,6 +2333,10 @@ public class Configuration {
public static final ConfigurationProperty<Integer> METRIC_RETRIEVAL_SERVICE_REQUEST_TTL = new ConfigurationProperty<>(
"metrics.retrieval-service.request.ttl", 5);
+ // Ambari server log4j file name
+ public static final String AMBARI_LOG_FILE = "log4j.properties";
+
+
/**
* The number of tasks that can be queried from the database at once In the
* case of more tasks, multiple queries are issued
@@ -2340,6 +2351,8 @@ public class Configuration {
Configuration.class);
private Properties properties;
+ private Properties log4jProperties = new Properties();
+ private String ambariUpgradeConfigUpdatesFilePath;
private JsonObject hostChangesJson;
private Map<String, String> configsMap;
private Map<String, String> agentConfigsMap;
@@ -2352,6 +2365,140 @@ public class Configuration {
private Map<String, String> databaseConnectorNames = new HashMap<>();
private Map<String, String> databasePreviousConnectorNames = new HashMap<>();
+
+ /**
+ * Find, read, and parse the log4j.properties file.
+ * @return the properties that were found or empty if no file was found
+ */
+ public Properties getLog4jProperties() {
+ if (!log4jProperties.isEmpty()) {
+ return log4jProperties;
+ }
+
+ //Get log4j.properties file stream from classpath
+ InputStream inputStream = Configuration.class.getClassLoader().getResourceAsStream(AMBARI_LOG_FILE);
+
+ if (inputStream == null) {
+ throw new RuntimeException(AMBARI_LOG_FILE + " not found in classpath");
+ }
+
+ // load the properties
+ try {
+ log4jProperties.load(inputStream);
+ inputStream.close();
+ } catch (FileNotFoundException fnf) {
+ LOG.info("No configuration file " + AMBARI_LOG_FILE + " found in classpath.", fnf);
+ } catch (IOException ie) {
+ throw new IllegalArgumentException("Can't read configuration file " +
+ AMBARI_LOG_FILE, ie);
+ }
+
+ return log4jProperties;
+ }
+
+
+ public void wrtiteToAmbariUpgradeConfigUpdatesFile(Multimap<AbstractUpgradeCatalog.ConfigUpdateType, Entry<String, String>> propertiesToLog,
+ String configType, String serviceName, String wrtiteToAmbariUpgradeConfigUpdatesFile) {
+ try {
+ if (ambariUpgradeConfigUpdatesFilePath == null) {
+ Properties log4jProperties = getLog4jProperties();
+ if (log4jProperties != null) {
+ String logPath = log4jProperties.getProperty("ambari.log.dir");
+ String rootPath = log4jProperties.getProperty("ambari.root.dir");
+ logPath = StringUtils.replace(logPath, "${ambari.root.dir}", rootPath);
+ logPath = StringUtils.replace(logPath, "//", "/");
+ if (StringUtils.isNotEmpty(logPath)) {
+ ambariUpgradeConfigUpdatesFilePath = logPath + File.separator + wrtiteToAmbariUpgradeConfigUpdatesFile;
+ }
+ } else {
+ LOG.warn("Log4j properties are not available");
+ }
+ }
+ } catch(Exception e) {
+ LOG.warn("Failed to create log file name or get path for it:", e);
+ }
+
+ if (StringUtils.isNotEmpty(ambariUpgradeConfigUpdatesFilePath)) {
+ Gson gson = new GsonBuilder().setPrettyPrinting().create();
+ Writer fileWriter = null;
+ try {
+ JsonObject rootJson = readFileToJSON(ambariUpgradeConfigUpdatesFilePath);
+ buildServiceJson(propertiesToLog, configType, serviceName, rootJson);
+
+ fileWriter = new FileWriter(ambariUpgradeConfigUpdatesFilePath);
+ gson.toJson(rootJson, fileWriter);
+ } catch (IllegalArgumentException e) {
+ JsonObject rootJson = new JsonObject();
+ buildServiceJson(propertiesToLog, configType, serviceName, rootJson);
+
+ try {
+ fileWriter = new FileWriter(ambariUpgradeConfigUpdatesFilePath);
+ gson.toJson(rootJson, fileWriter);
+ } catch (IOException e1) {
+ LOG.error("Unable to write data into " + ambariUpgradeConfigUpdatesFilePath, e);
+ }
+ } catch (IOException e) {
+ LOG.error("Unable to write data into " + ambariUpgradeConfigUpdatesFilePath, e);
+ } finally {
+ try {
+ fileWriter.close();
+ } catch (IOException e) {
+ LOG.error("Unable to close file " + ambariUpgradeConfigUpdatesFilePath, e);
+ }
+ }
+ }
+ }
+
+ private void buildServiceJson(Multimap<AbstractUpgradeCatalog.ConfigUpdateType, Entry<String, String>> propertiesToLog,
+ String configType, String serviceName, JsonObject rootJson) {
+ JsonElement serviceJson = null;
+ serviceJson = rootJson.get(serviceName);
+ JsonObject serviceJsonObject = null;
+ if (serviceJson != null) {
+ serviceJsonObject = serviceJson.getAsJsonObject();
+ } else {
+ serviceJsonObject = new JsonObject();
+ }
+ buildConfigJson(propertiesToLog, serviceJsonObject, configType);
+ if (serviceName == null) {
+ serviceName = "General";
+ }
+
+ rootJson.add(serviceName, serviceJsonObject);
+ }
+
+ private void buildConfigJson(Multimap<AbstractUpgradeCatalog.ConfigUpdateType, Entry<String, String>> propertiesToLog,
+ JsonObject serviceJson, String configType) {
+ JsonElement configJson = null;
+ configJson = serviceJson.get(configType);
+ JsonObject configJsonObject = null;
+ if (configJson != null) {
+ configJsonObject = configJson.getAsJsonObject();
+ } else {
+ configJsonObject = new JsonObject();
+ }
+ buildConfigUpdateTypes(propertiesToLog, configJsonObject);
+ serviceJson.add(configType, configJsonObject);
+ }
+
+ private void buildConfigUpdateTypes(Multimap<AbstractUpgradeCatalog.ConfigUpdateType, Entry<String, String>> propertiesToLog,
+ JsonObject configJson) {
+ for (AbstractUpgradeCatalog.ConfigUpdateType configUpdateType : propertiesToLog.keySet()) {
+ JsonElement currentConfigUpdateType = configJson.get(configUpdateType.getDescription());
+ JsonObject currentConfigUpdateTypeJsonObject = null;
+ if (currentConfigUpdateType != null) {
+ currentConfigUpdateTypeJsonObject = currentConfigUpdateType.getAsJsonObject();
+ } else {
+ currentConfigUpdateTypeJsonObject = new JsonObject();
+ }
+ for (Entry<String, String> property : propertiesToLog.get(configUpdateType)) {
+ currentConfigUpdateTypeJsonObject.add(property.getKey(), new JsonPrimitive(property.getValue()));
+ }
+ configJson.add(configUpdateType.getDescription(), currentConfigUpdateTypeJsonObject);
+ }
+ }
+
+
/**
* The Kerberos authentication-specific properties container (for convenience)
*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/3bf3bc59/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
index 8fa720f..8187bc2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
@@ -17,6 +17,11 @@
*/
package org.apache.ambari.server.upgrade;
+import javax.persistence.EntityManager;
+import javax.xml.bind.JAXBException;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
@@ -24,6 +29,7 @@ import java.io.StringReader;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.AbstractMap;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@@ -39,11 +45,6 @@ import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import javax.persistence.EntityManager;
-import javax.xml.bind.JAXBException;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.configuration.Configuration;
@@ -84,7 +85,9 @@ import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.xml.sax.InputSource;
+import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.inject.Inject;
@@ -140,6 +143,8 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
private static final Map<String, UpgradeCatalog> upgradeCatalogMap =
new HashMap<String, UpgradeCatalog>();
+ protected String ambariUpgradeConfigUpdatesFileName;
+
@Inject
public AbstractUpgradeCatalog(Injector injector) {
this.injector = injector;
@@ -553,11 +558,22 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
oldConfigProperties = oldConfig.getProperties();
}
+ Multimap<ConfigUpdateType, Entry<String, String>> propertiesToLog = ArrayListMultimap.create();
+ String serviceName = cluster.getServiceByConfigType(configType);
+
Map<String, String> mergedProperties =
- mergeProperties(oldConfigProperties, properties, updateIfExists);
+ mergeProperties(oldConfigProperties, properties, updateIfExists, propertiesToLog);
if (removePropertiesList != null) {
- mergedProperties = removeProperties(mergedProperties, removePropertiesList);
+ mergedProperties = removeProperties(mergedProperties, removePropertiesList, propertiesToLog);
+ }
+
+ if (propertiesToLog.size() > 0) {
+ try {
+ configuration.wrtiteToAmbariUpgradeConfigUpdatesFile(propertiesToLog, configType, serviceName, ambariUpgradeConfigUpdatesFileName);
+ } catch(Exception e) {
+ LOG.error("Write to config updates file failed:", e);
+ }
}
if (!Maps.difference(oldConfigProperties, mergedProperties).areEqual()) {
@@ -642,28 +658,53 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
private Map<String, String> mergeProperties(Map<String, String> originalProperties,
Map<String, String> newProperties,
- boolean updateIfExists) {
+ boolean updateIfExists, Multimap<AbstractUpgradeCatalog.ConfigUpdateType, Entry<String, String>> propertiesToLog) {
Map<String, String> properties = new HashMap<String, String>(originalProperties);
for (Map.Entry<String, String> entry : newProperties.entrySet()) {
- if (!properties.containsKey(entry.getKey()) || updateIfExists) {
+ if (!properties.containsKey(entry.getKey())) {
properties.put(entry.getKey(), entry.getValue());
+ propertiesToLog.put(ConfigUpdateType.ADDED, entry);
+ }
+ if (updateIfExists) {
+ properties.put(entry.getKey(), entry.getValue());
+ propertiesToLog.put(ConfigUpdateType.UPDATED, entry);
}
}
return properties;
}
- private Map<String, String> removeProperties(Map<String, String> originalProperties, Set<String> removeList){
+ private Map<String, String> removeProperties(Map<String, String> originalProperties,
+ Set<String> removeList, Multimap<AbstractUpgradeCatalog.ConfigUpdateType, Entry<String, String>> propertiesToLog){
Map<String, String> properties = new HashMap<String, String>();
properties.putAll(originalProperties);
for (String removeProperty: removeList){
if (originalProperties.containsKey(removeProperty)){
properties.remove(removeProperty);
+ propertiesToLog.put(ConfigUpdateType.REMOVED, new AbstractMap.SimpleEntry<String, String>(removeProperty, ""));
}
}
return properties;
}
+ public enum ConfigUpdateType {
+ ADDED("Added"),
+ UPDATED("Updated"),
+ REMOVED("Removed");
+
+
+ private final String description;
+
+
+ private ConfigUpdateType(String description) {
+ this.description = description;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+ }
+
/**
* Iterates through a collection of AbstractKerberosDescriptorContainers to find and update
* identity descriptor references.
@@ -895,6 +936,11 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
}
@Override
+ public void setConfigUpdatesFileName(String ambariUpgradeConfigUpdatesFileName) {
+ this.ambariUpgradeConfigUpdatesFileName = ambariUpgradeConfigUpdatesFileName;
+ }
+
+ @Override
public void upgradeData() throws AmbariException, SQLException {
executeDMLUpdates();
updateTezHistoryUrlBase();
http://git-wip-us.apache.org/repos/asf/ambari/blob/3bf3bc59/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
index 1f4470b..2256acf 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
@@ -17,32 +17,35 @@
*/
package org.apache.ambari.server.upgrade;
-import com.google.inject.Guice;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.multibindings.Multibinder;
-import com.google.inject.persist.PersistService;
-import org.apache.ambari.server.AmbariException;
-import org.apache.ambari.server.audit.AuditLoggerModule;
-import org.apache.ambari.server.configuration.Configuration;
-import org.apache.ambari.server.controller.ControllerModule;
-import org.apache.ambari.server.orm.DBAccessor;
-import org.apache.ambari.server.utils.EventBusSynchronizer;
-import org.apache.ambari.server.utils.VersionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.Set;
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.audit.AuditLoggerModule;
+import org.apache.ambari.server.configuration.Configuration;
+import org.apache.ambari.server.controller.ControllerModule;
+import org.apache.ambari.server.orm.DBAccessor;
+import org.apache.ambari.server.utils.EventBusSynchronizer;
+import org.apache.ambari.server.utils.VersionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.inject.Guice;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.multibindings.Multibinder;
+import com.google.inject.persist.PersistService;
+
public class SchemaUpgradeHelper {
private static final Logger LOG = LoggerFactory.getLogger
(SchemaUpgradeHelper.class);
@@ -226,12 +229,13 @@ public class SchemaUpgradeHelper {
}
}
- public void executeDMLUpdates(List<UpgradeCatalog> upgradeCatalogs) throws AmbariException {
+ public void executeDMLUpdates(List<UpgradeCatalog> upgradeCatalogs, String ambariUpgradeConfigUpdatesFileName) throws AmbariException {
LOG.info("Executing DML changes.");
if (upgradeCatalogs != null && !upgradeCatalogs.isEmpty()) {
for (UpgradeCatalog upgradeCatalog : upgradeCatalogs) {
try {
+ upgradeCatalog.setConfigUpdatesFileName(ambariUpgradeConfigUpdatesFileName);
upgradeCatalog.upgradeData();
} catch (Exception e) {
LOG.error("Upgrade failed. ", e);
@@ -345,13 +349,16 @@ public class SchemaUpgradeHelper {
List<UpgradeCatalog> upgradeCatalogs =
schemaUpgradeHelper.getUpgradePath(sourceVersion, targetVersion);
+ String date = new SimpleDateFormat("MM-dd-yyyy_HH:mm:ss").format(new Date());
+ String ambariUpgradeConfigUpdatesFileName = "ambari_upgrade_config_changes_" + date + ".json";
+
schemaUpgradeHelper.executeUpgrade(upgradeCatalogs);
schemaUpgradeHelper.startPersistenceService();
schemaUpgradeHelper.executePreDMLUpdates(upgradeCatalogs);
- schemaUpgradeHelper.executeDMLUpdates(upgradeCatalogs);
+ schemaUpgradeHelper.executeDMLUpdates(upgradeCatalogs, ambariUpgradeConfigUpdatesFileName);
schemaUpgradeHelper.executeOnPostUpgrade(upgradeCatalogs);
http://git-wip-us.apache.org/repos/asf/ambari/blob/3bf3bc59/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java
index 41c2215..21273fd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java
@@ -47,6 +47,13 @@ public interface UpgradeCatalog {
void upgradeData() throws AmbariException, SQLException;
/**
+ * Set the file name, to store all config changes during upgrade
+ * @param ambariUpgradeConfigUpdatesFileName
+ */
+ void setConfigUpdatesFileName(String ambariUpgradeConfigUpdatesFileName);
+
+
+ /**
* Defines if Upgrade Catalog should be executed last
* @return
*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/3bf3bc59/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java
index ff556b1..dd146fa 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java
@@ -221,7 +221,7 @@ public class UpgradeTest {
schemaUpgradeHelper.executePreDMLUpdates(upgradeCatalogs);
- schemaUpgradeHelper.executeDMLUpdates(upgradeCatalogs);
+ schemaUpgradeHelper.executeDMLUpdates(upgradeCatalogs, "test");
schemaUpgradeHelper.executeOnPostUpgrade(upgradeCatalogs);