You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/11/28 21:20:20 UTC
[01/24] ambari git commit: AMBARI-22453. ambari-server setup should
surface GPL software agreement (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/branch-2.6 754b1f609 -> c43673f69
AMBARI-22453. ambari-server setup should surface GPL software agreement (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/81439af0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/81439af0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/81439af0
Branch: refs/heads/branch-2.6
Commit: 81439af03e4464fcf5bb7fb8b2bbc5ec858cbf64
Parents: 56079f2
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Nov 16 11:22:58 2017 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Nov 16 11:22:58 2017 +0200
----------------------------------------------------------------------
.../ambari/server/agent/ExecutionCommand.java | 1 +
.../server/configuration/Configuration.java | 13 ++++++++++
.../AmbariCustomCommandExecutionHelper.java | 2 ++
.../internal/ClientConfigResourceProvider.java | 2 ++
ambari-server/src/main/python/ambari-server.py | 2 ++
.../python/ambari_server/serverConfiguration.py | 27 ++++++++++++++++----
.../main/python/ambari_server/serverSetup.py | 23 ++++++++++++++---
7 files changed, 61 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/81439af0/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
index 91df506..9d5e29e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
@@ -455,6 +455,7 @@ public class ExecutionCommand extends AgentCommand {
String IGNORE_PACKAGE_DEPENDENCIES = "ignore_package_dependencies";
String JDK_LOCATION = "jdk_location";
String JAVA_HOME = "java_home";
+ String GPL_LICENSE_ACCEPTED = "gpl_license_accepted";
String JAVA_VERSION = "java_version";
String JDK_NAME = "jdk_name";
String JCE_NAME = "jce_name";
http://git-wip-us.apache.org/repos/asf/ambari/blob/81439af0/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index 0bd6af6..085f420 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -726,6 +726,15 @@ public class Configuration {
"server.version.file", null);
/**
+ * The full path to the file which contains the Ambari Server version.
+ */
+ @Markdown(
+ description = "Whether user accepted GPL license.",
+ examples = { "true", "false" })
+ public static final ConfigurationProperty<String> GPL_LICENSE_ACCEPTED = new ConfigurationProperty<>(
+ "gpl.license.accepted", null);
+
+ /**
* The location of the JDK on the Ambari Agent hosts.
*/
@Markdown(
@@ -5319,6 +5328,10 @@ public class Configuration {
return NumberUtils.toInt(getProperty(VERSION_DEFINITION_READ_TIMEOUT));
}
+ public Boolean getGplLicenseAccepted(){
+ return getProperty(GPL_LICENSE_ACCEPTED).toLowerCase().equals("true");
+ }
+
public String getAgentStackRetryOnInstallCount(){
return getProperty(AGENT_STACK_RETRY_COUNT);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/81439af0/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index 04719a4..f75e315 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -29,6 +29,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GPL_LICENSE_ACCEPTED;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION;
@@ -1305,6 +1306,7 @@ public class AmbariCustomCommandExecutionHelper {
hostLevelParams.put(HOST_SYS_PREPPED, configs.areHostsSysPrepped());
hostLevelParams.put(AGENT_STACK_RETRY_ON_UNAVAILABILITY, configs.isAgentStackRetryOnInstallEnabled());
hostLevelParams.put(AGENT_STACK_RETRY_COUNT, configs.getAgentStackRetryOnInstallCount());
+ hostLevelParams.put(GPL_LICENSE_ACCEPTED, configs.getGplLicenseAccepted().toString());
Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs();
Map<PropertyInfo, String> notManagedHdfsPathMap = configHelper.getPropertiesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
http://git-wip-us.apache.org/repos/asf/ambari/blob/81439af0/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
index 0b048ad..0786d52 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
@@ -37,6 +37,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAM
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_GROUPS;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_LIST;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GPL_LICENSE_ACCEPTED;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
@@ -370,6 +371,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
hostLevelParams.putAll(managementController.getRcaParameters());
hostLevelParams.put(AGENT_STACK_RETRY_ON_UNAVAILABILITY, configs.isAgentStackRetryOnInstallEnabled());
hostLevelParams.put(AGENT_STACK_RETRY_COUNT, configs.getAgentStackRetryOnInstallCount());
+ hostLevelParams.put(GPL_LICENSE_ACCEPTED, configs.getGplLicenseAccepted().toString());
// Write down os specific info for the service
ServiceOsSpecific anyOs = null;
http://git-wip-us.apache.org/repos/asf/ambari/blob/81439af0/ambari-server/src/main/python/ambari-server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py
index 60c6005..6d242a3 100755
--- a/ambari-server/src/main/python/ambari-server.py
+++ b/ambari-server/src/main/python/ambari-server.py
@@ -409,6 +409,7 @@ def init_parser_options(parser):
parser.add_option('--skip-database-check', action="store_true", default=False, help="Skip database consistency check", dest="skip_database_check")
parser.add_option('--skip-view-extraction', action="store_true", default=False, help="Skip extraction of system views", dest="skip_view_extraction")
parser.add_option('--auto-fix-database', action="store_true", default=False, help="Automatically fix database consistency issues", dest="fix_database_consistency")
+ parser.add_option('--accept-gpl', action="store_true", default=False, help="Automatically accepts GPL license", dest="accept_gpl")
add_parser_options('--mpack',
default=None,
help="Specify the path for management pack to be installed/upgraded",
@@ -512,6 +513,7 @@ def init_parser_options(parser):
parser.add_option('--skip-database-check', action="store_true", default=False, help="Skip database consistency check", dest="skip_database_check")
parser.add_option('--skip-view-extraction', action="store_true", default=False, help="Skip extraction of system views", dest="skip_view_extraction")
parser.add_option('--auto-fix-database', action="store_true", default=False, help="Automatically fix database consistency issues", dest="fix_database_consistency")
+ parser.add_option('--accept-gpl', action="store_true", default=False, help="Automatically accepts GPL license", dest="accept_gpl")
parser.add_option('--force-version', action="store_true", default=False, help="Force version to current", dest="force_repo_version")
parser.add_option('--version', dest="stack_versions", default=None, action="append", type="string",
help="Specify stack version that needs to be enabled. All other stacks versions will be disabled")
http://git-wip-us.apache.org/repos/asf/ambari/blob/81439af0/ambari-server/src/main/python/ambari_server/serverConfiguration.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
index c7ac134..262e944 100644
--- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py
+++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
@@ -40,6 +40,7 @@ from ambari_server.properties import Properties
from ambari_server.userInput import get_validated_string_input
from ambari_server.utils import compare_versions, locate_file, on_powerpc
from ambari_server.ambariPath import AmbariPath
+from ambari_server.userInput import get_YN_input
OS_VERSION = OSCheck().get_os_major_version()
@@ -189,6 +190,15 @@ SETUP_OR_UPGRADE_MSG = "- If this is a new setup, then run the \"ambari-server s
"- If this is an upgrade of an existing setup, run the \"ambari-server upgrade\" command.\n" \
"Refer to the Ambari documentation for more information on setup and upgrade."
+GPL_LICENSE_PROMPT_TEXT = """To download GPL licensed products like lzo you must accept the license terms below:
+LICENSE_LINE_1
+LICENSE_LINE_2
+LICENSE_LINE_3
+LICENSE_LINE_4
+LICENSE_LINE_5
+LICENSE_LINE_6
+Do you accept the GPL License Agreement [y/n] (y)?"""
+
DEFAULT_DB_NAME = "ambari"
SECURITY_KEYS_DIR = "security.server.keys_dir"
@@ -203,6 +213,9 @@ BOOTSTRAP_SETUP_AGENT_SCRIPT = 'bootstrap.setup_agent.script'
STACKADVISOR_SCRIPT = 'stackadvisor.script'
PID_DIR_PROPERTY = 'pid.dir'
SERVER_TMP_DIR_PROPERTY = "server.tmp.dir"
+
+GPL_LICENSE_ACCEPTED_PROPERTY = 'gpl.license.accepted'
+
REQUIRED_PROPERTIES = [OS_FAMILY_PROPERTY, OS_TYPE_PROPERTY, COMMON_SERVICES_PATH_PROPERTY, SERVER_VERSION_FILE_PATH,
WEBAPP_DIR_PROPERTY, STACK_LOCATION_KEY, SECURITY_KEYS_DIR, JDBC_DATABASE_NAME_PROPERTY,
NR_USER_PROPERTY, JAVA_HOME_PROPERTY, JDBC_PASSWORD_PROPERTY, SHARED_RESOURCES_DIR,
@@ -357,7 +370,7 @@ class ServerConfigDefaults(object):
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
-
+
self.JAVA_SHARE_PATH = "/usr/share/java"
self.SHARE_PATH = "/usr/share"
self.OUT_DIR = parse_log4j_file(get_conf_dir() + "/log4j.properties")['ambari.log.dir'].replace("//", "/")
@@ -382,10 +395,10 @@ class ServerConfigDefaults(object):
self.PID_DIR = properties.get_property(PID_DIR_PROPERTY)
self.BOOTSTRAP_DIR = properties.get_property(BOOTSTRAP_DIR_PROPERTY)
self.RECOMMENDATIONS_DIR = properties.get_property(RECOMMENDATIONS_DIR_PROPERTY)
-
+
# this directories should be pre-created by user and be writable.
self.check_if_directories_writable([self.OUT_DIR, self.PID_DIR])
-
+
self.DEFAULT_LIBS_DIR = ""
self.DEFAULT_VLIBS_DIR = ""
@@ -425,7 +438,7 @@ class ServerConfigDefaults(object):
self.MESSAGE_ERROR_RESET_NOT_ROOT = ""
self.MESSAGE_ERROR_UPGRADE_NOT_ROOT = ""
self.MESSAGE_CHECK_FIREWALL = ""
-
+
def check_if_directories_writable(self, directories):
for directory in directories:
if not os.path.isdir(directory):
@@ -434,7 +447,7 @@ class ServerConfigDefaults(object):
except Exception as ex:
# permission denied here is expected when ambari runs as non-root
print_info_msg("Could not create {0}. Reason: {1}".format(directory, ex))
-
+
if not os.path.isdir(directory) or not os.access(directory, os.W_OK):
raise FatalException(-1, "Unable to access {0} directory. Confirm the directory is created and is writable by Ambari Server user account '{1}'".format(directory, getpass.getuser()))
@@ -1136,6 +1149,10 @@ def update_ambari_env():
return 0
+def prompt_gpl_agreement():
+ result = get_YN_input(GPL_LICENSE_PROMPT_TEXT, True)
+ return str(result).lower()
+
def update_ambari_properties():
prev_conf_file = search_file(configDefaults.AMBARI_PROPERTIES_BACKUP_FILE, get_conf_dir())
conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir())
http://git-wip-us.apache.org/repos/asf/ambari/blob/81439af0/ambari-server/src/main/python/ambari_server/serverSetup.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverSetup.py b/ambari-server/src/main/python/ambari_server/serverSetup.py
index c1643f2..8a14066 100644
--- a/ambari-server/src/main/python/ambari_server/serverSetup.py
+++ b/ambari-server/src/main/python/ambari_server/serverSetup.py
@@ -38,10 +38,10 @@ from ambari_commons.str_utils import compress_backslashes
from ambari_server.dbConfiguration import DBMSConfigFactory, TAR_GZ_ARCHIVE_TYPE, default_connectors_map, check_jdbc_drivers
from ambari_server.serverConfiguration import configDefaults, JDKRelease, \
get_ambari_properties, get_is_secure, get_is_persisted, get_java_exe_path, get_JAVA_HOME, get_missing_properties, \
- get_resources_location, get_value_from_properties, read_ambari_user, update_properties, validate_jdk, write_property, \
+ get_resources_location, get_value_from_properties, read_ambari_user, update_properties, validate_jdk, prompt_gpl_agreement, write_property, \
JAVA_HOME, JAVA_HOME_PROPERTY, JCE_NAME_PROPERTY, JDBC_RCA_URL_PROPERTY, JDBC_URL_PROPERTY, \
JDK_NAME_PROPERTY, JDK_RELEASES, NR_USER_PROPERTY, OS_FAMILY, OS_FAMILY_PROPERTY, OS_TYPE, OS_TYPE_PROPERTY, OS_VERSION, \
- VIEWS_DIR_PROPERTY, JDBC_DATABASE_PROPERTY, JDK_DOWNLOAD_SUPPORTED_PROPERTY, JCE_DOWNLOAD_SUPPORTED_PROPERTY, SETUP_DONE_PROPERTIES
+ VIEWS_DIR_PROPERTY, JDBC_DATABASE_PROPERTY, JDK_DOWNLOAD_SUPPORTED_PROPERTY, JCE_DOWNLOAD_SUPPORTED_PROPERTY, SETUP_DONE_PROPERTIES, GPL_LICENSE_ACCEPTED_PROPERTY
from ambari_server.serverUtils import is_server_runing
from ambari_server.setupSecurity import adjust_directory_permissions
from ambari_server.userInput import get_YN_input, get_validated_string_input
@@ -79,8 +79,6 @@ UNTAR_JDK_ARCHIVE = "tar --no-same-owner -xvf {0}"
JDK_PROMPT = "[{0}] {1}\n"
JDK_VALID_CHOICES = "^[{0}{1:d}]$"
-
-
def get_supported_jdbc_drivers():
factory = DBMSConfigFactory()
return factory.get_supported_jdbc_drivers()
@@ -1083,6 +1081,20 @@ def check_setup_already_done():
return not bool(get_missing_properties(properties, property_set=SETUP_DONE_PROPERTIES))
+def write_gpl_license_accepted(options):
+ properties = get_ambari_properties()
+ if properties == -1:
+ err = "Error getting ambari properties"
+ raise FatalException(-1, err)
+
+ if get_silent():
+ result = str(options.accept_gpl).lower()
+ else:
+ result = prompt_gpl_agreement()
+
+ properties.process_pair(GPL_LICENSE_ACCEPTED_PROPERTY, result)
+ update_properties(properties)
+
#
# Setup the Ambari Server.
#
@@ -1130,6 +1142,9 @@ def setup(options):
err = 'Downloading or installing JDK failed: {0}. Exiting.'.format(e)
raise FatalException(e.code, err)
+ print 'Prompting GPL software agreement...'
+ write_gpl_license_accepted(options)
+
print 'Completing setup...'
retcode = configure_os_settings()
if not retcode == 0:
[05/24] ambari git commit: AMBARI-22454. ambari-server upgrade to
2.6.1 should surface the GPL agreement (aonishuk)
Posted by jo...@apache.org.
AMBARI-22454. ambari-server upgrade to 2.6.1 should surface the GPL agreement (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3df5ae74
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3df5ae74
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3df5ae74
Branch: refs/heads/branch-2.6
Commit: 3df5ae74a5917923d254a47ba116816e899dfbce
Parents: 4b01b65
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Fri Nov 17 13:44:58 2017 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Fri Nov 17 13:44:58 2017 +0200
----------------------------------------------------------------------
.../server/upgrade/AbstractUpgradeCatalog.java | 8 ++
.../server/upgrade/SchemaUpgradeHelper.java | 27 +++++
.../ambari/server/upgrade/UpgradeCatalog.java | 7 ++
.../server/upgrade/UpgradeCatalog261.java | 118 +++++++++++++++++++
.../python/ambari_server/serverConfiguration.py | 22 +++-
.../main/python/ambari_server/serverSetup.py | 20 +---
.../main/python/ambari_server/serverUpgrade.py | 18 ++-
.../src/test/python/TestAmbariServer.py | 2 +-
8 files changed, 200 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
index dac871e..5ecb638 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
@@ -140,6 +140,7 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
new HashMap<>();
protected String ambariUpgradeConfigUpdatesFileName;
+ private Map<String,String> upgradeJsonOutput = new HashMap<>();
@Inject
public AbstractUpgradeCatalog(Injector injector) {
@@ -258,6 +259,13 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
}
/**
+ * {@inheritDoc}
+ */
+ public Map<String,String> getUpgradeJsonOutput() {
+ return upgradeJsonOutput;
+ }
+
+ /**
* Update metainfo to new version.
*/
@Transactional
http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
index a2dea40..963fd26 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
@@ -29,7 +29,11 @@ import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.Set;
+import java.util.Map;
+import java.util.HashMap;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.audit.AuditLoggerModule;
import org.apache.ambari.server.configuration.Configuration;
@@ -56,6 +60,7 @@ public class SchemaUpgradeHelper {
private DBAccessor dbAccessor;
private Configuration configuration;
private static final String[] rcaTableNames = {"workflow", "job", "task", "taskAttempt", "hdfsEvent", "mapreduceEvent", "clusterEvent"};
+ static final Gson gson = new GsonBuilder().create();
@Inject
public SchemaUpgradeHelper(Set<UpgradeCatalog> allUpgradeCatalogs,
@@ -184,6 +189,7 @@ public class SchemaUpgradeHelper {
catalogBinder.addBinding().to(UpgradeCatalog251.class);
catalogBinder.addBinding().to(UpgradeCatalog252.class);
catalogBinder.addBinding().to(UpgradeCatalog260.class);
+ catalogBinder.addBinding().to(UpgradeCatalog261.class);
catalogBinder.addBinding().to(UpdateAlertScriptPaths.class);
catalogBinder.addBinding().to(FinalUpgradeCatalog.class);
@@ -254,6 +260,26 @@ public class SchemaUpgradeHelper {
}
}
+ public void outputUpgradeJsonOutput(List<UpgradeCatalog> upgradeCatalogs)
+ throws AmbariException {
+ LOG.info("Combining upgrade json output.");
+ Map<String,String> combinedUpgradeJsonOutput = new HashMap<>();
+
+ if (upgradeCatalogs != null && !upgradeCatalogs.isEmpty()) {
+ for (UpgradeCatalog upgradeCatalog : upgradeCatalogs) {
+ try {
+ combinedUpgradeJsonOutput.putAll(upgradeCatalog.getUpgradeJsonOutput());
+
+ } catch (Exception e) {
+ LOG.error("Upgrade failed. ", e);
+ throw new AmbariException(e.getMessage(), e);
+ }
+ }
+ }
+ String content = gson.toJson(combinedUpgradeJsonOutput);
+ System.out.println(content);
+ }
+
public void resetUIState() throws AmbariException {
LOG.info("Resetting UI state.");
try {
@@ -424,6 +450,7 @@ public class SchemaUpgradeHelper {
schemaUpgradeHelper.executeDMLUpdates(upgradeCatalogs, ambariUpgradeConfigUpdatesFileName);
schemaUpgradeHelper.executeOnPostUpgrade(upgradeCatalogs);
+ schemaUpgradeHelper.outputUpgradeJsonOutput(upgradeCatalogs);
schemaUpgradeHelper.resetUIState();
http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java
index 21273fd..a005951 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java
@@ -18,6 +18,8 @@
package org.apache.ambari.server.upgrade;
import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.ambari.server.AmbariException;
@@ -88,4 +90,9 @@ public interface UpgradeCatalog {
* Update schema version in the database to the Target one
*/
void updateDatabaseSchemaVersion();
+
+ /*
+ Get upgrade json output, which is sent to python executing process.
+ */
+ Map<String,String> getUpgradeJsonOutput();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog261.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog261.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog261.java
new file mode 100644
index 0000000..7f463aa
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog261.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.upgrade;
+
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.SQLException;
+import java.util.Map;
+
+/**
+ * The {@link UpgradeCatalog261} upgrades Ambari from 2.6.0 to 2.6.1.
+ */
+public class UpgradeCatalog261 extends AbstractUpgradeCatalog {
+ private static final String CORE_SITE = "core-site";
+ private static final String COMPRESSION_CODECS_PROPERTY = "io.compression.codecs";
+ private static final String COMPRESSION_CODECS_LZO = "com.hadoop.compression.lzo";
+ private static final String LZO_ENABLED_JSON_KEY = "lzo_enabled";
+
+ private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog261.class);
+
+ /**
+ * Constructor.
+ *
+ * @param injector
+ */
+ @Inject
+ public UpgradeCatalog261(Injector injector) {
+ super(injector);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String getSourceVersion() {
+ return "2.6.0";
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String getTargetVersion() {
+ return "2.6.1";
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void executeDDLUpdates() throws AmbariException, SQLException {
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void executePreDMLUpdates() throws AmbariException, SQLException {
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void executeDMLUpdates() throws AmbariException, SQLException {
+ // TODO: make this map with clusterids as keys
+ this.getUpgradeJsonOutput().put(LZO_ENABLED_JSON_KEY, isLzoEnabled().toString());
+ }
+
+ protected Boolean isLzoEnabled() throws AmbariException {
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+ Clusters clusters = ambariManagementController.getClusters();
+ if (clusters != null) {
+ Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters);
+ if (clusterMap != null && !clusterMap.isEmpty()) {
+ for (final Cluster cluster : clusterMap.values()) {
+
+ Config coreSite = cluster.getDesiredConfigByType(CORE_SITE);
+ if (coreSite != null) {
+ Map<String, String> coreSiteProperties = coreSite.getProperties();
+
+ if (coreSiteProperties.containsKey(COMPRESSION_CODECS_PROPERTY)) {
+ String compressionCodecs = coreSiteProperties.get(COMPRESSION_CODECS_PROPERTY);
+
+ if(compressionCodecs.contains(COMPRESSION_CODECS_LZO)) {
+ return true;
+ }
+ }
+ }
+ }
+ }
+ }
+ return false;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/python/ambari_server/serverConfiguration.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
index 262e944..6e19b48 100644
--- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py
+++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
@@ -1149,9 +1149,25 @@ def update_ambari_env():
return 0
-def prompt_gpl_agreement():
- result = get_YN_input(GPL_LICENSE_PROMPT_TEXT, True)
- return str(result).lower()
+def write_gpl_license_accepted(is_silent=False):
+ properties = get_ambari_properties()
+ if properties == -1:
+ err = "Error getting ambari properties"
+ raise FatalException(-1, err)
+
+
+ if GPL_LICENSE_ACCEPTED_PROPERTY in properties.keys() and properties.get_property(GPL_LICENSE_ACCEPTED_PROPERTY).lower() == "true":
+ return True
+
+ if is_silent:
+ result = options.accept_gpl
+ else:
+ result = get_YN_input(GPL_LICENSE_PROMPT_TEXT, True)
+
+ properties.process_pair(GPL_LICENSE_ACCEPTED_PROPERTY, str(result).lower())
+ update_properties(properties)
+
+ return result
def update_ambari_properties():
prev_conf_file = search_file(configDefaults.AMBARI_PROPERTIES_BACKUP_FILE, get_conf_dir())
http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/python/ambari_server/serverSetup.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverSetup.py b/ambari-server/src/main/python/ambari_server/serverSetup.py
index 8a14066..4ffe0bc 100644
--- a/ambari-server/src/main/python/ambari_server/serverSetup.py
+++ b/ambari-server/src/main/python/ambari_server/serverSetup.py
@@ -38,7 +38,7 @@ from ambari_commons.str_utils import compress_backslashes
from ambari_server.dbConfiguration import DBMSConfigFactory, TAR_GZ_ARCHIVE_TYPE, default_connectors_map, check_jdbc_drivers
from ambari_server.serverConfiguration import configDefaults, JDKRelease, \
get_ambari_properties, get_is_secure, get_is_persisted, get_java_exe_path, get_JAVA_HOME, get_missing_properties, \
- get_resources_location, get_value_from_properties, read_ambari_user, update_properties, validate_jdk, prompt_gpl_agreement, write_property, \
+ get_resources_location, get_value_from_properties, read_ambari_user, update_properties, validate_jdk, write_gpl_license_accepted, write_property, \
JAVA_HOME, JAVA_HOME_PROPERTY, JCE_NAME_PROPERTY, JDBC_RCA_URL_PROPERTY, JDBC_URL_PROPERTY, \
JDK_NAME_PROPERTY, JDK_RELEASES, NR_USER_PROPERTY, OS_FAMILY, OS_FAMILY_PROPERTY, OS_TYPE, OS_TYPE_PROPERTY, OS_VERSION, \
VIEWS_DIR_PROPERTY, JDBC_DATABASE_PROPERTY, JDK_DOWNLOAD_SUPPORTED_PROPERTY, JCE_DOWNLOAD_SUPPORTED_PROPERTY, SETUP_DONE_PROPERTIES, GPL_LICENSE_ACCEPTED_PROPERTY
@@ -1081,20 +1081,6 @@ def check_setup_already_done():
return not bool(get_missing_properties(properties, property_set=SETUP_DONE_PROPERTIES))
-def write_gpl_license_accepted(options):
- properties = get_ambari_properties()
- if properties == -1:
- err = "Error getting ambari properties"
- raise FatalException(-1, err)
-
- if get_silent():
- result = str(options.accept_gpl).lower()
- else:
- result = prompt_gpl_agreement()
-
- properties.process_pair(GPL_LICENSE_ACCEPTED_PROPERTY, result)
- update_properties(properties)
-
#
# Setup the Ambari Server.
#
@@ -1142,8 +1128,8 @@ def setup(options):
err = 'Downloading or installing JDK failed: {0}. Exiting.'.format(e)
raise FatalException(e.code, err)
- print 'Prompting GPL software agreement...'
- write_gpl_license_accepted(options)
+ print 'Checking GPL software agreement...'
+ write_gpl_license_accepted(is_silent=get_silent())
print 'Completing setup...'
retcode = configure_os_settings()
http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/python/ambari_server/serverUpgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverUpgrade.py b/ambari-server/src/main/python/ambari_server/serverUpgrade.py
index 421adfc..a864609 100644
--- a/ambari-server/src/main/python/ambari_server/serverUpgrade.py
+++ b/ambari-server/src/main/python/ambari_server/serverUpgrade.py
@@ -28,6 +28,7 @@ import re
import glob
import optparse
import logging
+import ambari_simplejson as json
from ambari_commons.exceptions import FatalException
from ambari_commons.logging_utils import print_info_msg, print_warning_msg, print_error_msg, get_verbose
@@ -41,7 +42,7 @@ from ambari_server.serverConfiguration import configDefaults, get_resources_loca
update_database_name_property, get_admin_views_dir, get_views_dir, get_views_jars, \
AMBARI_PROPERTIES_FILE, IS_LDAP_CONFIGURED, LDAP_PRIMARY_URL_PROPERTY, RESOURCES_DIR_PROPERTY, \
SETUP_OR_UPGRADE_MSG, update_krb_jaas_login_properties, AMBARI_KRB_JAAS_LOGIN_FILE, get_db_type, update_ambari_env, \
- AMBARI_ENV_FILE, JDBC_DATABASE_PROPERTY, get_default_views_dir
+ AMBARI_ENV_FILE, JDBC_DATABASE_PROPERTY, get_default_views_dir, write_gpl_license_accepted
from ambari_server.setupSecurity import adjust_directory_permissions, \
generate_env, ensure_can_start_under_current_user
from ambari_server.utils import compare_versions
@@ -73,6 +74,11 @@ SCHEMA_UPGRADE_DEBUG = False
SUSPEND_START_MODE = False
+INSALLED_LZO_WITHOUT_GPL_TEXT = "By saying no, Ambari will not automatically install LZO on any new host in the cluster." + \
+"It is up to you to ensure LZO is installed and configured appropriately." + \
+"Without LZO being installed and configured data compressed with LZO will not be readable. " + \
+"Are you sure you want to proceed? [y/n] (n)?"
+
def load_stack_values(version, filename):
import xml.etree.ElementTree as ET
values = {}
@@ -146,6 +152,10 @@ def run_schema_upgrade(args):
environ = generate_env(args, ambari_user, current_user)
(retcode, stdout, stderr) = run_os_command(command, env=environ)
+ upgrade_response = json.loads(stdout)
+
+ check_gpl_license_approved(upgrade_response)
+
print_info_msg("Return code from schema upgrade command, retcode = {0}".format(str(retcode)), True)
if stdout:
print_info_msg("Console output from schema upgrade command:", True)
@@ -161,6 +171,12 @@ def run_schema_upgrade(args):
print_info_msg('Schema upgrade completed', True)
return retcode
+def check_gpl_license_approved(upgrade_response):
+ if 'lzo_enabled' not in upgrade_response or upgrade_response['lzo_enabled'].lower() != "true":
+ return
+
+ while not write_gpl_license_accepted() and not get_YN_input(INSALLED_LZO_WITHOUT_GPL_TEXT, False):
+ pass
#
# Upgrades the Ambari Server.
http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/test/python/TestAmbariServer.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py
index 840b9a6..a53d274 100644
--- a/ambari-server/src/test/python/TestAmbariServer.py
+++ b/ambari-server/src/test/python/TestAmbariServer.py
@@ -4938,7 +4938,7 @@ class TestAmbariServer(TestCase):
ensure_can_start_under_current_user_mock, get_jdbc_mock,
ensure_jdbc_driver_is_installed_mock):
java_exe_path_mock.return_value = "/usr/lib/java/bin/java"
- run_os_command_mock.return_value = (0, None, None)
+ run_os_command_mock.return_value = (0, '{"lzo_enabled":"false"}', None)
get_conf_dir_mock.return_value = '/etc/conf'
command = '/usr/lib/java/bin/java -cp /etc/conf' + os.pathsep + 'test' + os.pathsep + 'path12' + \
os.pathsep +'/path/to/jdbc.jar ' \
[16/24] ambari git commit: AMBARI-22484. Stack advisor should
disallow lzo enable without accepting license agreement. Additional fixes.
(mpapirkovskyy)
Posted by jo...@apache.org.
AMBARI-22484. Stack advisor should disallow lzo enable without accepting license agreement. Additional fixes. (mpapirkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d301d345
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d301d345
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d301d345
Branch: refs/heads/branch-2.6
Commit: d301d34517d5f48b824b95e9aab01176ec4c4a10
Parents: c0fbb86
Author: Myroslav Papirkovskyi <mp...@hortonworks.com>
Authored: Wed Nov 22 20:37:22 2017 +0200
Committer: Myroslav Papirkovskyi <mp...@hortonworks.com>
Committed: Wed Nov 22 20:40:06 2017 +0200
----------------------------------------------------------------------
.../stackadvisor/StackAdvisorRequest.java | 8 +++++
.../stacks/HDP/2.0.6/services/stack_advisor.py | 36 +++++++++++++++-----
.../src/main/resources/stacks/stack_advisor.py | 9 +++++
.../stacks/2.0.6/common/test_stack_advisor.py | 14 ++++++--
4 files changed, 55 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d301d345/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
index bc1e079..d750a6c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
@@ -118,6 +118,9 @@ public class StackAdvisorRequest {
this.configGroups = configGroups;
}
+ /**
+ * @return true if GPL license is accepted, false otherwise
+ */
public Boolean getGplLicenseAccepted() {
return gplLicenseAccepted;
}
@@ -194,6 +197,11 @@ public class StackAdvisorRequest {
return this;
}
+ /**
+ * Set GPL license acceptance parameter to request.
+ * @param gplLicenseAccepted is GPL license accepted.
+ * @return stack advisor request builder.
+ */
public StackAdvisorRequestBuilder withGPLLicenseAccepted(
Boolean gplLicenseAccepted) {
this.instance.gplLicenseAccepted = gplLicenseAccepted;
http://git-wip-us.apache.org/repos/asf/ambari/blob/d301d345/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 6d80f1a..3f8a772 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -517,20 +517,38 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
self.recommendHadoopProxyUsers(configurations, services, hosts)
def getLZOSupportValidationItems(self, properties, services):
+ '''
+ Checks GPL license is accepted when GPL software is used.
+ :param properties: dict of properties' name and value pairs
+ :param services: list of services
+ :return: NOT_APPLICABLE messages in case GPL license is not accepted
+ '''
services_list = self.get_services_list(services)
+ validations = []
if "HDFS" in services_list:
lzo_allowed = services["gpl-license-accepted"]
- property_name = "io.compression.codec.lzo.class"
- if property_name in properties:
- property_value = properties.get(property_name)
- if not lzo_allowed and "com.hadoop.compression.lzo.LzoCodec" in property_value:
- return [{"config-name": property_name, "item": self.getErrorItem(
- "Your Ambari Server has not been configured to download LZO and install it. "
- "LZO is GPL software and requires you to accept a license prior to use. "
- "Please refer to this documentation to configure Ambari before proceeding.")}]
- return []
+ self.validatePropertyToLZOCodec("io.compression.codecs", properties, lzo_allowed, validations)
+ self.validatePropertyToLZOCodec("io.compression.codec.lzo.class", properties, lzo_allowed, validations)
+ return validations
+
+ def validatePropertyToLZOCodec(self, property_name, properties, lzo_allowed, validations):
+ '''
+ Checks specified property contains LZO codec class and requires GPL license acceptance.
+ :param property_name: property name
+ :param properties: dict of properties' name and value pairs
+ :param lzo_allowed: is gpl license accepted
+ :param validations: list with validation failures
+ '''
+ lzo_codec_class = "com.hadoop.compression.lzo.LzoCodec"
+ if property_name in properties:
+ property_value = properties.get(property_name)
+ if not lzo_allowed and lzo_codec_class in property_value:
+ validations.append({"config-name": property_name, "item": self.getNotApplicableItem(
+ "Your Ambari Server has not been configured to download LZO and install it. "
+ "LZO is GPL software and requires you to accept a license prior to use. "
+ "Please refer to the documentation to configure Ambari before proceeding.")})
def recommendHbaseConfigurations(self, configurations, clusterData, services, hosts):
# recommendations for HBase env config
http://git-wip-us.apache.org/repos/asf/ambari/blob/d301d345/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 8ccbaba..ab3fd4f 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -888,6 +888,15 @@ class DefaultStackAdvisor(StackAdvisor):
"""
return {"level": "ERROR", "message": message}
+ def getNotApplicableItem(self, message):
+ '''
+ Creates report about validation error that can not be ignored.
+ UI should not allow the proceeding of work.
+ :param message: error description.
+ :return: report about error.
+ '''
+ return {"level": "NOT_APPLICABLE", "message": message}
+
def getComponentHostNames(self, servicesDict, serviceName, componentName):
for service in servicesDict["services"]:
if service["StackServices"]["service_name"] == serviceName:
http://git-wip-us.apache.org/repos/asf/ambari/blob/d301d345/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index e011847..e5adb19 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -2362,7 +2362,8 @@ class TestHDP206StackAdvisor(TestCase):
services = {"gpl-license-accepted": True, "services": [{"StackServices": {"service_name": "HDFS"}}]}
# 1) ok: gpl is allowed
- properties = {'io.compression.codec.lzo.class': 'com.hadoop.compression.lzo.LzoCodec'}
+ properties = {'io.compression.codec.lzo.class': 'com.hadoop.compression.lzo.LzoCodec',
+ 'io.compression.codecs': 'AnotherCodec, com.hadoop.compression.lzo.LzoCodec'}
res_expected = []
res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, {}, configurations, services, '')
@@ -2373,10 +2374,17 @@ class TestHDP206StackAdvisor(TestCase):
res_expected = [{'config-type': 'core-site',
'message': 'Your Ambari Server has not been configured to download LZO and install it. '
'LZO is GPL software and requires you to accept a license prior to use. '
- 'Please refer to this documentation to configure Ambari before proceeding.',
+ 'Please refer to the documentation to configure Ambari before proceeding.',
+ 'type': 'configuration',
+ 'config-name': 'io.compression.codecs',
+ 'level': 'NOT_APPLICABLE'},
+ {'config-type': 'core-site',
+ 'message': 'Your Ambari Server has not been configured to download LZO and install it. '
+ 'LZO is GPL software and requires you to accept a license prior to use. '
+ 'Please refer to the documentation to configure Ambari before proceeding.',
'type': 'configuration',
'config-name': 'io.compression.codec.lzo.class',
- 'level': 'ERROR'}]
+ 'level': 'NOT_APPLICABLE'}]
res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, {}, configurations, services, '')
self.assertEquals(res, res_expected)
[14/24] ambari git commit: AMBARI-22489. Port AMBARI-22467 to
BigInsight (aonishuk)
Posted by jo...@apache.org.
AMBARI-22489. Port AMBARI-22467 to BigInsight (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/aea1a3c4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/aea1a3c4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/aea1a3c4
Branch: refs/heads/branch-2.6
Commit: aea1a3c48fdf58be5fad04d7b983c65d28a248f9
Parents: c7457a8
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Wed Nov 22 12:32:04 2017 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Wed Nov 22 12:32:04 2017 +0200
----------------------------------------------------------------------
.../BigInsights/4.0/services/HDFS/metainfo.xml | 17 ------------
.../BigInsights/4.2/services/HDFS/metainfo.xml | 20 --------------
.../4.2/services/HDFS/package/scripts/hdfs.py | 5 ++--
.../4.2/services/HIVE/package/scripts/hive.py | 3 +++
.../4.2/services/OOZIE/package/scripts/oozie.py | 12 +++++----
.../services/OOZIE/package/scripts/params.py | 28 ++------------------
.../4.2/services/YARN/package/scripts/yarn.py | 3 +++
7 files changed, 17 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml
index 918cdb3..82c82d0 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml
@@ -152,10 +152,6 @@
<package>
<name>hadoop</name>
</package>
- <package>
- <name>hadoop-lzo</name>
- <condition>should_install_lzo</condition>
- </package>
</packages>
</osSpecific>
@@ -165,15 +161,6 @@
<package>
<name>snappy</name>
</package>
- <package>
- <name>lzo</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadoop-lzo-native</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop-libhdfs</name>
</package>
<package>
@@ -192,10 +179,6 @@
<name>libsnappy-dev</name>
</package>
<package>
- <name>liblzo2-2</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop-hdfs</name>
</package>
<package>
http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml
index 72ce2be..4253eb1 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml
@@ -182,11 +182,6 @@
<package>
<name>hadoop_4_2_*</name>
</package>
- <package>
- <name>hadoop-lzo</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
</packages>
</osSpecific>
<osSpecific>
@@ -196,16 +191,6 @@
<name>snappy</name>
</package>
<package>
- <name>lzo</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadoop-lzo-native</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop_4_2_*-libhdfs</name>
</package>
<package>
@@ -229,11 +214,6 @@
<name>libsnappy-dev</name>
</package>
<package>
- <name>liblzo2-2</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop-hdfs</name>
</package>
<package>
http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py
index 2b158bd..9334de0 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py
@@ -20,10 +20,10 @@ Ambari Agent
"""
from resource_management import *
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
import sys
import os
-
def hdfs(name=None):
import params
@@ -115,8 +115,7 @@ def hdfs(name=None):
content=Template("slaves.j2")
)
- if params.lzo_enabled and len(params.lzo_packages) > 0:
- Package(params.lzo_packages)
+ install_lzo_if_needed()
def install_snappy():
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py
index 0446e1c..f451a65 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py
@@ -34,6 +34,7 @@ from resource_management.core.shell import as_user
from resource_management.libraries.functions.is_empty import is_empty
from resource_management.libraries.resources.xml_config import XmlConfig
from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
from resource_management.core.exceptions import Fail
from resource_management.core.shell import as_sudo
from resource_management.core.shell import quote_bash_args
@@ -48,6 +49,8 @@ from ambari_commons import OSConst
def hive(name=None):
import params
+ install_lzo_if_needed()
+
XmlConfig("hive-site.xml",
conf_dir = params.hive_conf_dir,
configurations = params.config['configurations']['hive-site'],
http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py
index 64a83c0..0459432 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py
@@ -26,6 +26,7 @@ from resource_management.core.resources.system import Execute
from resource_management.core.source import DownloadSource
from resource_management.core.source import InlineTemplate
from resource_management.core.source import Template
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
from resource_management.libraries.functions import format
from resource_management.libraries.functions import compare_versions
from resource_management.libraries.resources.xml_config import XmlConfig
@@ -99,6 +100,12 @@ def oozie(is_server=False):
pass
oozie_ownership()
+
+ if params.lzo_enabled:
+ install_lzo_if_needed()
+ Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
+ not_if = no_op_test,
+ )
if is_server:
oozie_server_specific()
@@ -227,11 +234,6 @@ def oozie_server_specific():
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension.jar'),
not_if = no_op_test,
)
- #if params.lzo_enabled and len(params.lzo_packages_for_current_host) > 0:
- # Package(params.lzo_packages_for_current_host)
- # Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
- # not_if = no_op_test,
- # )
prepare_war_cmd_file = format("{oozie_home}/.prepare_war_cmd")
prepare_war_cmd = format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war {oozie_secure}")
http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py
index d1a4a66..ee9b88a 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py
@@ -27,6 +27,7 @@ from resource_management.libraries.functions import get_port_from_url
from resource_management.libraries.functions import format
from resource_management.libraries.functions.version import format_stack_version
from resource_management.libraries.functions.version import compare_versions
+from resource_management.libraries.functions.lzo_utils import should_install_lzo
from resource_management.libraries.resources import HdfsResource
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import stack_select
@@ -234,29 +235,4 @@ HdfsResource = functools.partial(
default_fs = default_fs
)
-#LZO support
-
-#-----LZO is not suppported in IOP distribution since it is GPL license--------
-
-'''
-io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
-lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
-
-# stack_is_iop40_or_further
-underscored_version = stack_version_unformatted.replace('.', '_')
-dashed_version = stack_version_unformatted.replace('.', '-')
-lzo_packages_to_family = {
- "any": ["hadoop-lzo", ],
- "redhat": ["lzo", "hadoop-lzo-native"],
- "suse": ["lzo", "hadoop-lzo-native"],
- "ubuntu": ["liblzo2-2", ]
-}
-
-
-lzo_packages_to_family["redhat"] += [format("hadooplzo_{underscorred_version}_*")]
-lzo_packages_to_family["suse"] += [format("hadooplzo_{underscorred_version}_*")]
-lzo_packages_to_family["ubuntu"] += [format("hadooplzo_{dashed_version}_*")]
-
-lzo_packages_for_current_host = lzo_packages_to_family['any'] + lzo_packages_to_family[System.get_instance().os_family]
-all_lzo_packages = set(itertools.chain(*lzo_packages_to_family.values()))
-'''
+lzo_enabled = should_install_lzo()
http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py
index a284183..9006d2e 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py
@@ -20,6 +20,7 @@ Ambari Agent
"""
from resource_management import *
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
import sys
import os
@@ -50,6 +51,8 @@ def create_local_dir(dir_name):
def yarn(name = None):
import params
+
+ install_lzo_if_needed()
if name == "historyserver":
if params.yarn_log_aggregation_enabled:
[07/24] ambari git commit: AMBARI-22454. ambari-server upgrade to
2.6.1 should surface the GPL agreement (aonishuk)
Posted by jo...@apache.org.
AMBARI-22454. ambari-server upgrade to 2.6.1 should surface the GPL agreement (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f35fd4e6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f35fd4e6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f35fd4e6
Branch: refs/heads/branch-2.6
Commit: f35fd4e672d071a22c172855f74bda7e05712fd4
Parents: 2f02bc6
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Fri Nov 17 16:03:20 2017 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Fri Nov 17 16:04:18 2017 +0200
----------------------------------------------------------------------
.../src/main/python/ambari_server/serverConfiguration.py | 7 ++-----
ambari-server/src/main/python/ambari_server/serverSetup.py | 5 +++--
2 files changed, 5 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f35fd4e6/ambari-server/src/main/python/ambari_server/serverConfiguration.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
index 6e19b48..8697e4c 100644
--- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py
+++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
@@ -1149,7 +1149,7 @@ def update_ambari_env():
return 0
-def write_gpl_license_accepted(is_silent=False):
+def write_gpl_license_accepted():
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
@@ -1159,10 +1159,7 @@ def write_gpl_license_accepted(is_silent=False):
if GPL_LICENSE_ACCEPTED_PROPERTY in properties.keys() and properties.get_property(GPL_LICENSE_ACCEPTED_PROPERTY).lower() == "true":
return True
- if is_silent:
- result = options.accept_gpl
- else:
- result = get_YN_input(GPL_LICENSE_PROMPT_TEXT, True)
+ result = get_YN_input(GPL_LICENSE_PROMPT_TEXT, True)
properties.process_pair(GPL_LICENSE_ACCEPTED_PROPERTY, str(result).lower())
update_properties(properties)
http://git-wip-us.apache.org/repos/asf/ambari/blob/f35fd4e6/ambari-server/src/main/python/ambari_server/serverSetup.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverSetup.py b/ambari-server/src/main/python/ambari_server/serverSetup.py
index 4ffe0bc..50992d1 100644
--- a/ambari-server/src/main/python/ambari_server/serverSetup.py
+++ b/ambari-server/src/main/python/ambari_server/serverSetup.py
@@ -1128,8 +1128,9 @@ def setup(options):
err = 'Downloading or installing JDK failed: {0}. Exiting.'.format(e)
raise FatalException(e.code, err)
- print 'Checking GPL software agreement...'
- write_gpl_license_accepted(is_silent=get_silent())
+ if not get_silent() or options.accept_gpl:
+ print 'Checking GPL software agreement...'
+ write_gpl_license_accepted()
print 'Completing setup...'
retcode = configure_os_settings()
[18/24] ambari git commit: AMBARI-22496. Admin View - Manage Versions
page should show HDP-GPL repo if necessary (vsubramanian)
Posted by jo...@apache.org.
AMBARI-22496. Admin View - Manage Versions page should show HDP-GPL repo if necessary (vsubramanian)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/059fadf4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/059fadf4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/059fadf4
Branch: refs/heads/branch-2.6
Commit: 059fadf460f2fc59ecb60403141c8b36a4c89668
Parents: ba957a8
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
Authored: Wed Nov 22 16:59:13 2017 -0800
Committer: Vivek Ratnavel Subramanian <vi...@gmail.com>
Committed: Wed Nov 22 16:59:13 2017 -0800
----------------------------------------------------------------------
.../stackVersions/StackVersionsEditCtrl.js | 27 ++++++++++++++++++-
.../ui/admin-web/app/scripts/services/Stack.js | 14 ++++++++++
.../views/stackVersions/stackVersionPage.html | 28 +++++++++++---------
3 files changed, 55 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/059fadf4/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsEditCtrl.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsEditCtrl.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsEditCtrl.js
index 09e3c28..b7fa741 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsEditCtrl.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsEditCtrl.js
@@ -33,6 +33,15 @@ angular.module('ambariAdminConsole')
display_name: ''
};
$scope.defaulfOSRepos = {}; // a copy of initial loaded repo info for "changed" check later
+ $scope.isGPLAccepted = false;
+
+ $scope.isGPLRepo = function (repository) {
+ return repository.Repositories.tags.indexOf('GPL') >= 0;
+ };
+
+ $scope.showRepo = function (repository) {
+ return $scope.isGPLAccepted || !$scope.isGPLRepo(repository);
+ };
$scope.loadStackVersionInfo = function () {
return Stack.getRepo($routeParams.versionId, $routeParams.stackName).then(function (response) {
@@ -67,6 +76,9 @@ angular.module('ambariAdminConsole')
// load supported os type base on stack version
$scope.afterStackVersionRead();
+ // Load GPL license accepted value
+ $scope.fetchGPLLicenseAccepted();
+
// if user reach here from UI click, repo status should be cached
// otherwise re-fetch repo status from cluster end point.
$scope.repoStatus = Cluster.repoStatusCache[$scope.id];
@@ -85,6 +97,15 @@ angular.module('ambariAdminConsole')
};
/**
+ * Load GPL License Accepted value
+ */
+ $scope.fetchGPLLicenseAccepted = function () {
+ Stack.getGPLLicenseAccepted().then(function (data) {
+ $scope.isGPLAccepted = data === 'true';
+ })
+ };
+
+ /**
* Load supported OS list
*/
$scope.afterStackVersionRead = function () {
@@ -170,7 +191,11 @@ angular.module('ambariAdminConsole')
$scope.updateRepoVersions = function () {
var skip = $scope.skipValidation || $scope.useRedhatSatellite;
- return Stack.validateBaseUrls(skip, $scope.osList, $scope.upgradeStack).then(function (invalidUrls) {
+ // Filter out repositories that are not shown in the UI
+ var osList = Object.assign([], $scope.osList).map(function(os) {
+ return Object.assign({}, os, {repositories: os.repositories.filter(function(repo) { return $scope.showRepo(repo); })});
+ });
+ return Stack.validateBaseUrls(skip, osList, $scope.upgradeStack).then(function (invalidUrls) {
if (invalidUrls.length === 0) {
Stack.updateRepo($scope.upgradeStack.stack_name, $scope.upgradeStack.stack_version, $scope.id, $scope.updateObj).then(function () {
Alert.success($t('versions.alerts.versionEdited', {
http://git-wip-us.apache.org/repos/asf/ambari/blob/059fadf4/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Stack.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Stack.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Stack.js
index d5f4401..46f4f42 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Stack.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Stack.js
@@ -87,6 +87,20 @@ angular.module('ambariAdminConsole')
return deferred.promise;
},
+ getGPLLicenseAccepted: function() {
+ var deferred = $q.defer();
+
+ $http.get(Settings.baseUrl + '/services/AMBARI/components/AMBARI_SERVER?fields=RootServiceComponents/properties/gpl.license.accepted&minimal_response=true', {mock: 'true'})
+ .then(function(data) {
+ deferred.resolve(data.data.RootServiceComponents.properties['gpl.license.accepted']);
+ })
+ .catch(function(data) {
+ deferred.reject(data);
+ });
+
+ return deferred.promise;
+ },
+
allPublicStackVersions: function() {
var url = '/version_definitions?fields=VersionDefinition/stack_default,VersionDefinition/type,' +
'operating_systems/repositories/Repositories/*,VersionDefinition/stack_services,' +
http://git-wip-us.apache.org/repos/asf/ambari/blob/059fadf4/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/stackVersionPage.html
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/stackVersionPage.html b/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/stackVersionPage.html
index aeaaa71..447d755 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/stackVersionPage.html
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/stackVersionPage.html
@@ -182,20 +182,22 @@
<div class="col-sm-9">
<div class="form-group repo-name-url {{repository.Repositories.repo_name}}"
ng-class="{'has-error': repository.hasError }" ng-repeat="repository in os.repositories">
- <span class="repo-name-label control-label col-sm-3">{{repository.Repositories.repo_id}}</span>
- <div class="col-sm-7 repo-url">
- <input type="text" class="form-control"
- placeholder="{{(repository.Repositories.repo_name.indexOf('UTILS') < 0 )?('versions.repository.placeholder' | translate) : ''}}"
- ng-model="repository.Repositories.base_url"
- ng-change="onRepoUrlChange(repository)" ng-disabled="useRedhatSatellite">
+ <div ng-if="showRepo(repository)">
+ <span class="repo-name-label control-label col-sm-3">{{repository.Repositories.repo_id}}</span>
+ <div class="col-sm-7 repo-url">
+ <input type="text" class="form-control"
+ placeholder="{{(repository.Repositories.repo_name.indexOf('UTILS') < 0 )?('versions.repository.placeholder' | translate) : ''}}"
+ ng-model="repository.Repositories.base_url"
+ ng-change="onRepoUrlChange(repository)" ng-disabled="useRedhatSatellite">
+ </div>
+ <i class="fa fa-undo orange-icon cursor-pointer"
+ ng-if="selectedOption.index == 1 && repository.Repositories.base_url != repository.Repositories.initial_base_url
+ || selectedOption.index == 2 && repository.Repositories.base_url != ''
+ || editController && repository.Repositories.base_url != repository.Repositories.initial_base_url"
+ ng-click="undoChange(repository)"
+ tooltip-html-unsafe="{{'common.undo' | translate}}"
+ aria-hidden="true"></i>
</div>
- <i class="fa fa-undo orange-icon cursor-pointer"
- ng-if="selectedOption.index == 1 && repository.Repositories.base_url != repository.Repositories.initial_base_url
- || selectedOption.index == 2 && repository.Repositories.base_url != ''
- || editController && repository.Repositories.base_url != repository.Repositories.initial_base_url"
- ng-click="undoChange(repository)"
- tooltip-html-unsafe="{{'common.undo' | translate}}"
- aria-hidden="true"></i>
</div>
</div>
<div class="col-sm-1 remove-icon" ng-click="removeOS()" ng-class="{'disabled' : useRedhatSatellite}"><i
[15/24] ambari git commit: AMBARI-22500. Modify AMBARI-22387 to Check
for LZO + No Opt-in (dlysnichenko)
Posted by jo...@apache.org.
AMBARI-22500. Modify AMBARI-22387 to Check for LZO + No Opt-in (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c0fbb86a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c0fbb86a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c0fbb86a
Branch: refs/heads/branch-2.6
Commit: c0fbb86a4c34190be01ff287586959b5320ea98d
Parents: aea1a3c
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Wed Nov 22 17:44:07 2017 +0200
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Wed Nov 22 17:44:07 2017 +0200
----------------------------------------------------------------------
.../java/org/apache/ambari/server/checks/LZOCheck.java | 6 +++++-
.../org/apache/ambari/server/checks/LZOCheckTest.java | 13 ++++++++++---
2 files changed, 15 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c0fbb86a/ambari-server/src/main/java/org/apache/ambari/server/checks/LZOCheck.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/LZOCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/LZOCheck.java
index 9c0286b..1eaacea 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/checks/LZOCheck.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/LZOCheck.java
@@ -52,6 +52,9 @@ public class LZOCheck extends AbstractCheckDescriptor {
*/
@Override
public void perform(PrerequisiteCheck prerequisiteCheck, PrereqCheckRequest request) throws AmbariException {
+ if (config.getGplLicenseAccepted()){
+ return;
+ }
List<String> errorMessages = new ArrayList<>();
PrereqCheckStatus checkStatus = PrereqCheckStatus.WARNING;
@@ -66,7 +69,8 @@ public class LZOCheck extends AbstractCheckDescriptor {
}
if (!errorMessages.isEmpty()) {
- prerequisiteCheck.setFailReason(StringUtils.join(errorMessages, "You have LZO codec enabled in the core-site config of your cluster. LZO is no longer installed automatically. " +
+ prerequisiteCheck.setFailReason(StringUtils.join(errorMessages, "You have LZO codec enabled in the core-site config of your cluster. " +
+ "You have to accept GPL license during ambari-server setup to have LZO installed automatically. " +
"If any hosts require LZO, it should be installed before starting the upgrade. " +
"Consult Ambari documentation for instructions on how to do this."));
prerequisiteCheck.getFailedOn().add("LZO");
http://git-wip-us.apache.org/repos/asf/ambari/blob/c0fbb86a/ambari-server/src/test/java/org/apache/ambari/server/checks/LZOCheckTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/LZOCheckTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/LZOCheckTest.java
index e50e936..13f52a5 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/checks/LZOCheckTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/LZOCheckTest.java
@@ -60,6 +60,9 @@ public class LZOCheckTest {
@Mock
private RepositoryVersionEntity m_repositoryVersion;
+ @Mock
+ private Configuration configuration;
+
final Map<String, Service> m_services = new HashMap<>();
@Before
@@ -70,8 +73,7 @@ public class LZOCheckTest {
return clusters;
}
};
- Configuration config = Mockito.mock(Configuration.class);
- lZOCheck.config = config;
+ lZOCheck.config = configuration;
m_services.clear();
@@ -99,7 +101,6 @@ public class LZOCheckTest {
public void testPerform() throws Exception {
final Cluster cluster = Mockito.mock(Cluster.class);
final Map<String, Service> services = new HashMap<>();
- final Service service = Mockito.mock(Service.class);
Mockito.when(cluster.getServices()).thenReturn(services);
Mockito.when(cluster.getClusterId()).thenReturn(1L);
@@ -115,6 +116,7 @@ public class LZOCheckTest {
Mockito.when(cluster.getConfig(Mockito.anyString(), Mockito.anyString())).thenReturn(config);
final Map<String, String> properties = new HashMap<>();
Mockito.when(config.getProperties()).thenReturn(properties);
+ Mockito.when(configuration.getGplLicenseAccepted()).thenReturn(false);
PrerequisiteCheck check = new PrerequisiteCheck(null, null);
lZOCheck.perform(check, new PrereqCheckRequest("cluster"));
@@ -141,5 +143,10 @@ public class LZOCheckTest {
check = new PrerequisiteCheck(null, null);
lZOCheck.perform(check, new PrereqCheckRequest("cluster"));
Assert.assertEquals(PrereqCheckStatus.WARNING, check.getStatus());
+
+ Mockito.when(configuration.getGplLicenseAccepted()).thenReturn(true);
+ check = new PrerequisiteCheck(null, null);
+ lZOCheck.perform(check, new PrereqCheckRequest("cluster"));
+ Assert.assertEquals(PrereqCheckStatus.PASS, check.getStatus());
}
}
[08/24] ambari git commit: AMBARI-22467. YARN, MapReduce2, Hive,
and Oozie Should Conditionally Install LZO (aonishuk)
Posted by jo...@apache.org.
AMBARI-22467. YARN, MapReduce2, Hive, and Oozie Should Conditionally Install LZO (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/096fedbd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/096fedbd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/096fedbd
Branch: refs/heads/branch-2.6
Commit: 096fedbd38ef241446ccee5020a62b62fc3f94bf
Parents: f35fd4e
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Mon Nov 20 12:37:22 2017 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Mon Nov 20 12:37:22 2017 +0200
----------------------------------------------------------------------
.../libraries/functions/__init__.py | 2 +-
.../libraries/functions/get_lzo_packages.py | 50 -----------
.../libraries/functions/lzo_utils.py | 93 ++++++++++++++++++++
.../libraries/functions/package_conditions.py | 8 +-
.../common-services/HDFS/2.1.0.2.0/metainfo.xml | 30 -------
.../HDFS/2.1.0.2.0/package/scripts/hdfs.py | 6 +-
.../2.1.0.2.0/package/scripts/install_params.py | 10 ---
.../2.1.0.2.0/package/scripts/params_linux.py | 7 --
.../HIVE/0.12.0.2.0/package/scripts/hive.py | 4 +
.../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 48 +++++-----
.../4.0.0.2.0/package/scripts/params_linux.py | 8 +-
.../YARN/2.1.0.2.0/package/scripts/yarn.py | 3 +
.../stacks/HDP/2.2/services/HDFS/metainfo.xml | 35 --------
.../stacks/HDP/2.3/services/HDFS/metainfo.xml | 30 -------
.../stacks/2.2/configs/oozie-upgrade.json | 3 +-
15 files changed, 131 insertions(+), 206 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py b/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
index f144b2d..b907844 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
@@ -39,7 +39,7 @@ from resource_management.libraries.functions.version import *
from resource_management.libraries.functions.format_jvm_option import *
from resource_management.libraries.functions.constants import *
from resource_management.libraries.functions.get_stack_version import *
-from resource_management.libraries.functions.get_lzo_packages import *
+from resource_management.libraries.functions.lzo_utils import *
from resource_management.libraries.functions.setup_ranger_plugin import *
from resource_management.libraries.functions.curl_krb_request import *
from resource_management.libraries.functions.get_bare_principal import *
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py
deleted file mode 100644
index cfbb7d8..0000000
--- a/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-__all__ = ["get_lzo_packages"]
-
-from ambari_commons.os_check import OSCheck
-from resource_management.libraries.functions.stack_features import check_stack_feature
-from resource_management.libraries.functions import StackFeature
-from resource_management.libraries.script.script import Script
-
-# TODO: Make list of lzo packages stack driven
-def get_lzo_packages(stack_version_unformatted):
- lzo_packages = []
- script_instance = Script.get_instance()
- if OSCheck.is_suse_family() and int(OSCheck.get_os_major_version()) >= 12:
- lzo_packages += ["liblzo2-2", "hadoop-lzo-native"]
- elif OSCheck.is_redhat_family() or OSCheck.is_suse_family():
- lzo_packages += ["lzo", "hadoop-lzo-native"]
- elif OSCheck.is_ubuntu_family():
- lzo_packages += ["liblzo2-2"]
-
- if stack_version_unformatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_unformatted):
- if OSCheck.is_ubuntu_family():
- lzo_packages += [script_instance.format_package_name("hadooplzo-${stack_version}") ,
- script_instance.format_package_name("hadooplzo-${stack_version}-native")]
- else:
- lzo_packages += [script_instance.format_package_name("hadooplzo_${stack_version}"),
- script_instance.format_package_name("hadooplzo_${stack_version}-native")]
- else:
- lzo_packages += ["hadoop-lzo"]
-
- return lzo_packages
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py b/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py
new file mode 100644
index 0000000..d6d987f
--- /dev/null
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+__all__ = ["should_install_lzo", "get_lzo_packages", "install_lzo_if_needed"]
+
+from ambari_commons.os_check import OSCheck
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions import StackFeature, stack_features
+from resource_management.libraries.script.script import Script
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions.expect import expect
+from resource_management.core.resources.packaging import Package
+
+INSTALLING_LZO_WITHOUT_GPL = "Cannot install LZO. The GPL license must be explicitly enabled using 'ambari-server setup' on the Ambari host, then restart the server and try again."
+
+def get_lzo_packages():
+ lzo_packages = []
+ script_instance = Script.get_instance()
+ if OSCheck.is_suse_family() and int(OSCheck.get_os_major_version()) >= 12:
+ lzo_packages += ["liblzo2-2", "hadoop-lzo-native"]
+ elif OSCheck.is_redhat_family() or OSCheck.is_suse_family():
+ lzo_packages += ["lzo", "hadoop-lzo-native"]
+ elif OSCheck.is_ubuntu_family():
+ lzo_packages += ["liblzo2-2"]
+
+
+ stack_version_unformatted = stack_features.get_stack_feature_version(Script.get_config()) # only used to check stack_feature, NOT as package version!
+ if stack_version_unformatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_unformatted):
+ if OSCheck.is_ubuntu_family():
+ lzo_packages += [script_instance.format_package_name("hadooplzo-${stack_version}") ,
+ script_instance.format_package_name("hadooplzo-${stack_version}-native")]
+ else:
+ lzo_packages += [script_instance.format_package_name("hadooplzo_${stack_version}"),
+ script_instance.format_package_name("hadooplzo_${stack_version}-native")]
+ else:
+ lzo_packages += ["hadoop-lzo"]
+
+ return lzo_packages
+
+def should_install_lzo():
+ """
+ Return true if lzo is enabled via core-site.xml and GPL license (required for lzo) is accepted.
+ """
+ config = Script.get_config()
+ io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
+ lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
+
+ if not lzo_enabled:
+ return False
+
+ is_gpl_license_accepted = default("/hostLevelParams/gpl_license_accepted", False)
+ if not is_gpl_license_accepted:
+ Logger.warning(INSTALLING_LZO_WITHOUT_GPL)
+ return False
+
+ return True
+
+def install_lzo_if_needed():
+ """
+ Install lzo package if {#should_install_lzo} is true
+ """
+ if not should_install_lzo():
+ return
+
+ lzo_packages = get_lzo_packages()
+
+ config = Script.get_config()
+ agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability']
+ agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int)
+
+ Package(lzo_packages,
+ retry_on_repo_unavailability=agent_stack_retry_on_unavailability,
+ retry_count=agent_stack_retry_count
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
index 31e78b9..ded63cf 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
@@ -19,7 +19,7 @@ limitations under the License.
Ambari Agent
"""
-__all__ = ["is_lzo_enabled", "should_install_phoenix", "should_install_ams_collector", "should_install_ams_grafana",
+__all__ = ["should_install_phoenix", "should_install_ams_collector", "should_install_ams_grafana",
"should_install_mysql", "should_install_ranger_tagsync"]
import os
@@ -44,12 +44,6 @@ def _has_local_components(config, components, indicator_function = any):
def _has_applicable_local_component(config, components):
return _has_local_components(config, components, any)
-def should_install_lzo():
- config = Script.get_config()
- io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
- lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
- return lzo_enabled
-
def should_install_phoenix():
phoenix_hosts = default('/clusterHostInfo/phoenix_query_server_hosts', [])
phoenix_enabled = default('/configurations/hbase-env/phoenix_sql_enabled', False)
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
index 2df1b9f..2ab5ef8 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
@@ -203,11 +203,6 @@
<package>
<name>hadoop</name>
</package>
- <package>
- <name>hadoop-lzo</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
</packages>
</osSpecific>
@@ -224,16 +219,6 @@
<name>snappy-devel</name>
</package>
<package>
- <name>lzo</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadoop-lzo-native</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop-libhdfs</name>
</package>
</packages>
@@ -252,16 +237,6 @@
<name>snappy-devel</name>
</package>
<package>
- <name>liblzo2-2</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadoop-lzo-native</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop-libhdfs</name>
</package>
</packages>
@@ -280,11 +255,6 @@
<name>libsnappy-dev</name>
</package>
<package>
- <name>liblzo2-2</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop-hdfs</name>
</package>
<package>
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
index 6f702d3..23aa08c 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
@@ -23,6 +23,7 @@ from resource_management import *
import os
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
from ambari_commons import OSConst
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
def hdfs(name=None):
@@ -133,10 +134,7 @@ def hdfs(name=None):
content=Template("slaves.j2")
)
- if params.lzo_enabled and len(params.lzo_packages) > 0:
- Package(params.lzo_packages,
- retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
- retry_count=params.agent_stack_retry_count)
+ install_lzo_if_needed()
def install_snappy():
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
index fe488c3..dc3279f 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
@@ -23,17 +23,7 @@ if OSCheck.is_windows_family():
exclude_packages = []
else:
from resource_management.libraries.functions.default import default
- from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
from resource_management.libraries.script.script import Script
_config = Script.get_config()
stack_version_unformatted = str(_config['hostLevelParams']['stack_version'])
-
- # The logic for LZO also exists in OOZIE's params.py
- io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
- lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
- lzo_packages = get_lzo_packages(stack_version_unformatted)
-
- exclude_packages = []
- if not lzo_enabled:
- exclude_packages += lzo_packages
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
index 5ca012b..5e1f4ac 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
@@ -40,7 +40,6 @@ from resource_management.libraries.functions.get_not_managed_resources import ge
from resource_management.libraries.script.script import Script
from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.functions.format_jvm_option import format_jvm_option
-from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs
from resource_management.libraries.functions import is_empty
from resource_management.libraries.functions.get_architecture import get_architecture
@@ -383,12 +382,6 @@ HdfsResource = functools.partial(
immutable_paths = get_not_managed_resources(),
dfs_type = dfs_type
)
-
-
-# The logic for LZO also exists in OOZIE's params.py
-io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
-lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
-lzo_packages = get_lzo_packages(stack_version_unformatted)
name_node_params = default("/commandParams/namenode", None)
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
index abbe59e..fbff099 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
@@ -41,6 +41,7 @@ from resource_management.core.logger import Logger
from resource_management.core import utils
from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook
from resource_management.libraries.functions.security_commons import update_credential_provider_path
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
from ambari_commons.constants import SERVICE
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
@@ -105,6 +106,9 @@ def hive(name=None):
@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
def hive(name=None):
import params
+
+ install_lzo_if_needed()
+
hive_client_conf_path = format("{stack_root}/current/{component_directory}/conf")
# Permissions 644 for conf dir (client) files, and 600 for conf.server
mode_identified = 0644 if params.hive_config_dir == hive_client_conf_path else 0600
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
index 7d22603..2316f79 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
@@ -35,6 +35,7 @@ from resource_management.libraries.functions.stack_features import check_stack_f
from resource_management.libraries.functions.oozie_prepare_war import prepare_war
from resource_management.libraries.functions.copy_tarball import get_current_version
from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
from resource_management.libraries.script.script import Script
from resource_management.libraries.functions.security_commons import update_credential_provider_path
from resource_management.core.resources.packaging import Package
@@ -185,16 +186,21 @@ def oozie(is_server=False, upgrade_type=None):
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
)
- pass
oozie_ownership()
-
- if is_server:
+
+ if params.lzo_enabled:
+ install_lzo_if_needed()
+ Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
+ not_if = no_op_test,
+ )
+
+ if is_server:
oozie_server_specific(upgrade_type)
-
+
def oozie_ownership():
import params
-
+
File ( format("{conf_dir}/hadoop-config.xml"),
owner = params.oozie_user,
group = params.user_group
@@ -237,14 +243,14 @@ def get_oozie_ext_zip_source_paths(upgrade_type, params):
def oozie_server_specific(upgrade_type):
import params
-
+
no_op_test = as_user(format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.oozie_user)
-
+
File(params.pid_file,
action="delete",
not_if=no_op_test
)
-
+
oozie_server_directories = [format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir, params.oozie_webapps_conf_dir, params.oozie_server_dir]
Directory( oozie_server_directories,
owner = params.oozie_user,
@@ -253,25 +259,25 @@ def oozie_server_specific(upgrade_type):
create_parents = True,
cd_access="a",
)
-
+
Directory(params.oozie_libext_dir,
create_parents = True,
)
-
+
hashcode_file = format("{oozie_home}/.hashcode")
skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share")
untar_sharelib = ('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home)
Execute( untar_sharelib, # time-expensive
- not_if = format("{no_op_test} || {skip_recreate_sharelib}"),
+ not_if = format("{no_op_test} || {skip_recreate_sharelib}"),
sudo = True,
)
configure_cmds = []
# Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path
source_ext_zip_paths = get_oozie_ext_zip_source_paths(upgrade_type, params)
-
+
# Copy the first oozie ext-2.2.zip file that is found.
# This uses a list to handle the cases when migrating from some versions of BigInsights to HDP.
if source_ext_zip_paths is not None:
@@ -285,8 +291,8 @@ def oozie_server_specific(upgrade_type):
sudo=True,
)
break
-
-
+
+
Directory(params.oozie_webapps_conf_dir,
owner = params.oozie_user,
group = params.user_group,
@@ -305,14 +311,6 @@ def oozie_server_specific(upgrade_type):
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'),
not_if = no_op_test)
- if params.lzo_enabled and len(params.all_lzo_packages) > 0:
- Package(params.all_lzo_packages,
- retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
- retry_count=params.agent_stack_retry_count)
- Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
- not_if = no_op_test,
- )
-
prepare_war(params)
File(hashcode_file,
@@ -363,7 +361,7 @@ def oozie_server_specific(upgrade_type):
Directory(params.oozie_server_dir,
owner = params.oozie_user,
group = params.user_group,
- recursive_ownership = True,
+ recursive_ownership = True,
)
if params.security_enabled:
File(os.path.join(params.conf_dir, 'zkmigrator_jaas.conf'),
@@ -408,7 +406,7 @@ def copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type=None, upgrade_direction=N
effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version(params.version)
if not check_stack_feature(StackFeature.ATLAS_HOOK_SUPPORT, effective_version):
return
-
+
# Important that oozie_server_hostnames is sorted by name so that this only runs on a single Oozie server.
if not (len(params.oozie_server_hostnames) > 0 and params.hostname == params.oozie_server_hostnames[0]):
Logger.debug("Will not attempt to copy Atlas Hive hook to DFS since this is not the first Oozie Server "
@@ -463,7 +461,7 @@ def copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type=None, upgrade_direction=N
recursive_chmod=True,
replace_existing_files=True
)
-
+
Logger.info("Copying Atlas Hive hook properties file to Oozie Sharelib in DFS.")
atlas_hook_filepath_source = os.path.join(params.hive_conf_dir, params.atlas_hook_filename)
atlas_hook_file_path_dest_in_dfs = os.path.join(hive_sharelib_dir, params.atlas_hook_filename)
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
index cd1f7ad..8f38843 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
@@ -31,7 +31,7 @@ from resource_management.libraries.functions import get_port_from_url
from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster
from resource_management.libraries.script.script import Script
-from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
+from resource_management.libraries.functions.lzo_utils import should_install_lzo
from resource_management.libraries.functions.expect import expect
from resource_management.libraries.functions.get_architecture import get_architecture
from resource_management.libraries.functions.stack_features import get_stack_feature_version
@@ -387,8 +387,4 @@ HdfsResource = functools.partial(
is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled']
-# The logic for LZO also exists in HDFS' params.py
-io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
-lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
-
-all_lzo_packages = get_lzo_packages(stack_version_unformatted)
+lzo_enabled = should_install_lzo()
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
index f6e8391..197382c 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
@@ -27,6 +27,7 @@ from resource_management.libraries.script.script import Script
from resource_management.core.resources.service import ServiceConfig
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.is_empty import is_empty
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
from resource_management.core.resources.system import Directory
from resource_management.core.resources.system import File
from resource_management.libraries.resources.xml_config import XmlConfig
@@ -110,6 +111,8 @@ def yarn(name=None, config_dir=None):
"""
import params
+ install_lzo_if_needed()
+
if config_dir is None:
config_dir = params.hadoop_conf_dir
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml
index 07a9ef8..8aa69a9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml
@@ -40,19 +40,6 @@
<name>snappy-devel</name>
</package>
<package>
- <name>lzo</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadooplzo_${stack_version}</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadooplzo_${stack_version}-native</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop_${stack_version}-libhdfs</name>
</package>
</packages>
@@ -74,19 +61,6 @@
<name>snappy-devel</name>
</package>
<package>
- <name>liblzo2-2</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadooplzo_${stack_version}</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadooplzo_${stack_version}-native</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop_${stack_version}-libhdfs</name>
</package>
</packages>
@@ -120,15 +94,6 @@
<name>libsnappy-dev</name>
</package>
<package>
- <name>hadooplzo-${stack_version}</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>liblzo2-2</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>libhdfs0-${stack_version}</name>
</package>
</packages>
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml
index 63e85e3..e02bf6b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml
@@ -72,19 +72,6 @@
<name>snappy-devel</name>
</package>
<package>
- <name>lzo</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadooplzo_${stack_version}</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadooplzo_${stack_version}-native</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop_${stack_version}-libhdfs</name>
</package>
</packages>
@@ -106,19 +93,6 @@
<name>snappy-devel</name>
</package>
<package>
- <name>liblzo2-2</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadooplzo_${stack_version}</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadooplzo_${stack_version}-native</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop_${stack_version}-libhdfs</name>
</package>
</packages>
@@ -152,10 +126,6 @@
<name>libsnappy-dev</name>
</package>
<package>
- <name>hadooplzo-${stack_version}</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>libhdfs0-${stack_version}</name>
</package>
</packages>
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json b/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
index 86ca03a..29cbddc 100644
--- a/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
+++ b/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
@@ -55,7 +55,8 @@
"mysql_jdbc_url": "http://c6401.ambari.apache.org:8080/resources//mysql-connector-java.jar",
"custom_mysql_jdbc_name" : "mysql-connector-java.jar",
"custom_oracle_jdbc_name" : "oracle-jdbc-driver.jar",
- "custom_postgres_jdbc_name" : "test-postgres-jdbc.jar"
+ "custom_postgres_jdbc_name" : "test-postgres-jdbc.jar",
+ "gpl_license_accepted": "true"
},
"commandType": "EXECUTION_COMMAND",
"roleParams": {
[11/24] ambari git commit: AMBARI-22480. Validate blueprint does not
allow lzo enable without setup with license agreement. JavaDocs update.
(mpapirkovskyy)
Posted by jo...@apache.org.
AMBARI-22480. Validate blueprint does not allow lzo enable without setup with license agreement. JavaDocs update. (mpapirkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/519f5270
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/519f5270
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/519f5270
Branch: refs/heads/branch-2.6
Commit: 519f52706020bfa276d44f21f2120fb1154c4985
Parents: 9d55c0f
Author: Myroslav Papirkovskyi <mp...@hortonworks.com>
Authored: Tue Nov 21 17:16:51 2017 +0200
Committer: Myroslav Papirkovskyi <mp...@hortonworks.com>
Committed: Tue Nov 21 19:24:46 2017 +0200
----------------------------------------------------------------------
.../main/java/org/apache/ambari/server/topology/BlueprintImpl.java | 1 +
.../java/org/apache/ambari/server/topology/BlueprintValidator.java | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/519f5270/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java
index 06fd9f2..9b67c32 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java
@@ -350,6 +350,7 @@ public class BlueprintImpl implements Blueprint {
* Validate blueprint configuration.
*
* @throws InvalidTopologyException if the blueprint configuration is invalid
+ * @throws GPLLicenseNotAcceptedException ambari was configured to use gpl software, but gpl license is not accepted
*/
@Override
public void validateRequiredProperties() throws InvalidTopologyException, GPLLicenseNotAcceptedException {
http://git-wip-us.apache.org/repos/asf/ambari/blob/519f5270/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java
index c201a7f..38ef01f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java
@@ -36,6 +36,7 @@ public interface BlueprintValidator {
* This doesn't include password properties.
*
* @throws InvalidTopologyException if required properties are not set in blueprint
+ * @throws GPLLicenseNotAcceptedException ambari was configured to use gpl software, but gpl license is not accepted
*/
void validateRequiredProperties() throws InvalidTopologyException, GPLLicenseNotAcceptedException;
}
[06/24] ambari git commit: Merge branch 'branch-2.6' into
branch-feature-AMBARI-22457
Posted by jo...@apache.org.
Merge branch 'branch-2.6' into branch-feature-AMBARI-22457
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2f02bc69
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2f02bc69
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2f02bc69
Branch: refs/heads/branch-2.6
Commit: 2f02bc69376aa03cd56ca8789cd1d02e902604bc
Parents: 3df5ae7 c09c69b
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Fri Nov 17 08:30:31 2017 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Fri Nov 17 08:30:31 2017 -0500
----------------------------------------------------------------------
.../libraries/functions/copy_tarball.py | 102 +++++-
.../libraries/functions/tar_archive.py | 6 +-
.../internal/UpgradeResourceProvider.java | 36 ++
.../upgrades/CreateAndConfigureAction.java | 164 +++++++++
.../ambari/server/state/stack/UpgradePack.java | 3 +
.../state/stack/upgrade/ClusterGrouping.java | 2 +
.../stack/upgrade/CreateAndConfigureTask.java | 57 +++
.../ambari/server/state/stack/upgrade/Task.java | 6 +-
.../HIVE/0.12.0.2.0/configuration/hive-site.xml | 2 +-
.../4.0/properties/stack_packages.json | 12 -
.../stacks/HDP/2.0.6/role_command_order.json | 1 +
.../services/HIVE/configuration/hive-site.xml | 2 +-
.../services/HIVE/configuration/hive-site.xml | 2 +-
.../stacks/HDP/2.5/upgrades/config-upgrade.xml | 23 ++
.../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 23 +-
.../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 45 ++-
.../configuration/spark2-log4j-properties.xml | 56 +++
.../src/main/resources/upgrade-pack.xsd | 9 +-
.../upgrades/CreateAndConfigureActionTest.java | 357 +++++++++++++++++++
.../stacks/2.1/FALCON/test_falcon_server.py | 6 +-
ambari-web/app/views/common/controls_view.js | 8 +-
.../test/views/common/controls_view_test.js | 2 +-
22 files changed, 866 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
[17/24] ambari git commit: AMBARI-22454. ambari-server upgrade to
2.6.1 should surface the GPL agreement. Fix checkstyle. (Attila Doroszlai via
mpapyrkovskyy)
Posted by jo...@apache.org.
AMBARI-22454. ambari-server upgrade to 2.6.1 should surface the GPL agreement. Fix checkstyle. (Attila Doroszlai via mpapyrkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ba957a86
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ba957a86
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ba957a86
Branch: refs/heads/branch-2.6
Commit: ba957a86990f533387f4a788452a6aae9fb758cc
Parents: d301d34
Author: Myroslav Papirkovskyi <mp...@hortonworks.com>
Authored: Wed Nov 22 21:57:15 2017 +0200
Committer: Myroslav Papirkovskyi <mp...@hortonworks.com>
Committed: Wed Nov 22 22:23:53 2017 +0200
----------------------------------------------------------------------
.../AmbariCustomCommandExecutionHelper.java | 14 +-------------
.../internal/ClientConfigResourceProvider.java | 2 +-
.../ambari/server/upgrade/SchemaUpgradeHelper.java | 8 ++++----
.../apache/ambari/server/upgrade/UpgradeCatalog.java | 1 -
.../ambari/server/upgrade/UpgradeCatalog261.java | 11 ++++++-----
5 files changed, 12 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba957a86/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index f75e315..06c3b04 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -26,10 +26,10 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMPONENT
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CUSTOM_COMMAND;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GPL_LICENSE_ACCEPTED;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GPL_LICENSE_ACCEPTED;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION;
@@ -58,8 +58,6 @@ import java.util.Random;
import java.util.Set;
import java.util.TreeMap;
-import org.apache.ambari.annotations.Experimental;
-import org.apache.ambari.annotations.ExperimentalFeature;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.Role;
import org.apache.ambari.server.RoleCommand;
@@ -67,7 +65,6 @@ import org.apache.ambari.server.actionmanager.HostRoleCommand;
import org.apache.ambari.server.actionmanager.HostRoleStatus;
import org.apache.ambari.server.actionmanager.Stage;
import org.apache.ambari.server.agent.AgentCommand.AgentCommandType;
-import org.apache.ambari.server.agent.CommandRepository;
import org.apache.ambari.server.agent.ExecutionCommand;
import org.apache.ambari.server.agent.ExecutionCommand.KeyNames;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
@@ -78,9 +75,6 @@ import org.apache.ambari.server.controller.spi.Resource;
import org.apache.ambari.server.controller.spi.SystemException;
import org.apache.ambari.server.metadata.ActionMetadata;
import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
-import org.apache.ambari.server.orm.entities.OperatingSystemEntity;
-import org.apache.ambari.server.orm.entities.RepositoryEntity;
-import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.CommandScriptDefinition;
@@ -95,7 +89,6 @@ import org.apache.ambari.server.state.HostState;
import org.apache.ambari.server.state.MaintenanceState;
import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.state.PropertyInfo.PropertyType;
-import org.apache.ambari.server.state.RepositoryInfo;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.ServiceComponent;
import org.apache.ambari.server.state.ServiceComponentHost;
@@ -112,12 +105,7 @@ import org.apache.commons.lang.math.NumberUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Function;
-import com.google.common.collect.ListMultimap;
import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
import com.google.inject.Inject;
import com.google.inject.Singleton;
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba957a86/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
index 0786d52..a7c712b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
@@ -21,6 +21,7 @@ package org.apache.ambari.server.controller.internal;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_COUNT;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_ON_UNAVAILABILITY;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GPL_LICENSE_ACCEPTED;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME;
@@ -37,7 +38,6 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAM
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_GROUPS;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_LIST;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GPL_LICENSE_ACCEPTED;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba957a86/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
index 963fd26..2ab8dc8 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
@@ -26,14 +26,12 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import java.util.Set;
-import java.util.Map;
-import java.util.HashMap;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.audit.AuditLoggerModule;
import org.apache.ambari.server.configuration.Configuration;
@@ -45,6 +43,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.jdbc.support.JdbcUtils;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
import com.google.inject.Guice;
import com.google.inject.Inject;
import com.google.inject.Injector;
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba957a86/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java
index a005951..37a3b5e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java
@@ -18,7 +18,6 @@
package org.apache.ambari.server.upgrade;
import java.sql.SQLException;
-import java.util.HashMap;
import java.util.Map;
import org.apache.ambari.server.AmbariException;
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba957a86/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog261.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog261.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog261.java
index 7f463aa..ba95833 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog261.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog261.java
@@ -17,8 +17,9 @@
*/
package org.apache.ambari.server.upgrade;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
+import java.sql.SQLException;
+import java.util.Map;
+
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.state.Cluster;
@@ -27,8 +28,8 @@ import org.apache.ambari.server.state.Config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.sql.SQLException;
-import java.util.Map;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
/**
* The {@link UpgradeCatalog261} upgrades Ambari from 2.6.0 to 2.6.1.
@@ -115,4 +116,4 @@ public class UpgradeCatalog261 extends AbstractUpgradeCatalog {
}
return false;
}
-}
\ No newline at end of file
+}
[10/24] ambari git commit: AMBARI-22484. Stack advisor should
disallow lzo enable without accepting license agreement. (Myroslav
Papirkovskyy via swagle)
Posted by jo...@apache.org.
AMBARI-22484. Stack advisor should disallow lzo enable without accepting license agreement. (Myroslav Papirkovskyy via swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9d55c0f5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9d55c0f5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9d55c0f5
Branch: refs/heads/branch-2.6
Commit: 9d55c0f5034c2765e4428d9b3ad56c455be54bfc
Parents: 1fce565
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Mon Nov 20 10:58:47 2017 -0800
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Mon Nov 20 10:58:47 2017 -0800
----------------------------------------------------------------------
.../stackadvisor/StackAdvisorRequest.java | 11 ++++++++
.../commands/StackAdvisorCommand.java | 2 ++
.../ambari/server/controller/AmbariServer.java | 3 ++-
.../internal/StackAdvisorResourceProvider.java | 9 +++++--
.../GPLLicenseNotAcceptedException.java | 28 ++++++++++++++++++++
.../stacks/HDP/2.0.6/services/stack_advisor.py | 23 +++++++++++++++-
.../ValidationResourceProviderTest.java | 4 ++-
.../stacks/2.0.6/common/test_stack_advisor.py | 25 +++++++++++++++++
8 files changed, 100 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
index 64180e5..bc1e079 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
@@ -48,6 +48,7 @@ public class StackAdvisorRequest {
private List<ChangedConfigInfo> changedConfigurations = new LinkedList<ChangedConfigInfo>();
private Set<RecommendationResponse.ConfigGroup> configGroups;
private Map<String, String> userContext = new HashMap<String, String>();
+ private Boolean gplLicenseAccepted;
public String getStackName() {
return stackName;
@@ -117,6 +118,10 @@ public class StackAdvisorRequest {
this.configGroups = configGroups;
}
+ public Boolean getGplLicenseAccepted() {
+ return gplLicenseAccepted;
+ }
+
private StackAdvisorRequest(String stackName, String stackVersion) {
this.stackName = stackName;
this.stackVersion = stackVersion;
@@ -189,6 +194,12 @@ public class StackAdvisorRequest {
return this;
}
+ public StackAdvisorRequestBuilder withGPLLicenseAccepted(
+ Boolean gplLicenseAccepted) {
+ this.instance.gplLicenseAccepted = gplLicenseAccepted;
+ return this;
+ }
+
public StackAdvisorRequest build() {
return this.instance;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
index 5440462..48924f8 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
@@ -94,6 +94,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
private static final String CONFIGURATIONS_PROPERTY = "configurations";
private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed-configurations";
private static final String USER_CONTEXT_PROPERTY = "user-context";
+ private static final String GPL_LICENSE_ACCEPTED = "gpl-license-accepted";
private static final String AMBARI_SERVER_CONFIGURATIONS_PROPERTY = "ambari-server-properties";
private File recommendationsDir;
@@ -205,6 +206,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
JsonNode userContext = mapper.valueToTree(request.getUserContext());
root.put(USER_CONTEXT_PROPERTY, userContext);
+ root.put(GPL_LICENSE_ACCEPTED, request.getGplLicenseAccepted());
}
private void populateConfigGroups(ObjectNode root,
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
index b2a68d6..154d4e7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
@@ -904,7 +904,8 @@ public class AmbariServer {
PersistKeyValueService.init(injector.getInstance(PersistKeyValueImpl.class));
KeyService.init(injector.getInstance(PersistKeyValueImpl.class));
BootStrapResource.init(injector.getInstance(BootStrapImpl.class));
- StackAdvisorResourceProvider.init(injector.getInstance(StackAdvisorHelper.class));
+ StackAdvisorResourceProvider.init(injector.getInstance(StackAdvisorHelper.class),
+ injector.getInstance(Configuration.class));
StageUtils.setGson(injector.getInstance(Gson.class));
StageUtils.setTopologyManager(injector.getInstance(TopologyManager.class));
StageUtils.setConfiguration(injector.getInstance(Configuration.class));
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
index dd32c6b..c2993db 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
@@ -36,6 +36,7 @@ import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestBuilder;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestType;
import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
+import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.spi.Request;
import org.apache.ambari.server.controller.spi.Resource.Type;
@@ -79,12 +80,14 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
private static final String CONFIG_GROUPS_HOSTS_PROPERTY = "hosts";
protected static StackAdvisorHelper saHelper;
+ protected static Configuration configuration;
protected static final String USER_CONTEXT_OPERATION_PROPERTY = "user_context/operation";
protected static final String USER_CONTEXT_OPERATION_DETAILS_PROPERTY = "user_context/operation_details";
@Inject
- public static void init(StackAdvisorHelper instance) {
+ public static void init(StackAdvisorHelper instance, Configuration serverConfig) {
saHelper = instance;
+ configuration = serverConfig;
}
protected StackAdvisorResourceProvider(Set<String> propertyIds, Map<Type, String> keyPropertyIds,
@@ -130,6 +133,7 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
hgHostsMap);
Map<String, Map<String, Map<String, String>>> configurations = calculateConfigurations(request);
Map<String, String> userContext = readUserContext(request);
+ Boolean gplLicenseAccepted = configuration.getGplLicenseAccepted();
List<ChangedConfigInfo> changedConfigurations =
requestType == StackAdvisorRequestType.CONFIGURATION_DEPENDENCIES ?
@@ -144,7 +148,8 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
withConfigurations(configurations).
withConfigGroups(configGroups).
withChangedConfigurations(changedConfigurations).
- withUserContext(userContext).build();
+ withUserContext(userContext).
+ withGPLLicenseAccepted(gplLicenseAccepted).build();
} catch (Exception e) {
LOG.warn("Error occurred during preparation of stack advisor request", e);
Response response = Response.status(Status.BAD_REQUEST)
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java
new file mode 100644
index 0000000..b444d01
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+/**
+ * Indicates an not permitted LZO usage.
+ */
+public class GPLLicenseNotAcceptedException extends Exception {
+ public GPLLicenseNotAcceptedException(String s) {
+ super(s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index c36a41f..6d80f1a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -516,6 +516,22 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
# recommendations for "hadoop.proxyuser.*.hosts", "hadoop.proxyuser.*.groups" properties in core-site
self.recommendHadoopProxyUsers(configurations, services, hosts)
+ def getLZOSupportValidationItems(self, properties, services):
+ services_list = self.get_services_list(services)
+
+ if "HDFS" in services_list:
+ lzo_allowed = services["gpl-license-accepted"]
+ property_name = "io.compression.codec.lzo.class"
+ if property_name in properties:
+ property_value = properties.get(property_name)
+ if not lzo_allowed and "com.hadoop.compression.lzo.LzoCodec" in property_value:
+ return [{"config-name": property_name, "item": self.getErrorItem(
+ "Your Ambari Server has not been configured to download LZO and install it. "
+ "LZO is GPL software and requires you to accept a license prior to use. "
+ "Please refer to this documentation to configure Ambari before proceeding.")}]
+
+ return []
+
def recommendHbaseConfigurations(self, configurations, clusterData, services, hosts):
# recommendations for HBase env config
@@ -1173,7 +1189,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
def getServiceConfigurationValidators(self):
return {
"HDFS": { "hdfs-site": self.validateHDFSConfigurations,
- "hadoop-env": self.validateHDFSConfigurationsEnv},
+ "hadoop-env": self.validateHDFSConfigurationsEnv,
+ "core-site": self.validateHDFSConfigurationsCoreSite},
"MAPREDUCE2": {"mapred-site": self.validateMapReduce2Configurations},
"YARN": {"yarn-site": self.validateYARNConfigurations,
"yarn-env": self.validateYARNEnvConfigurations},
@@ -1783,6 +1800,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
{"config-name": 'namenode_opt_maxnewsize', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'namenode_opt_maxnewsize')}]
return self.toConfigurationValidationProblems(validationItems, "hadoop-env")
+ def validateHDFSConfigurationsCoreSite(self, properties, recommendedDefaults, configurations, services, hosts):
+ return self.toConfigurationValidationProblems(self.getLZOSupportValidationItems(properties, services),
+ "core-site")
+
def validatorOneDataDirPerPartition(self, properties, propertyName, services, hosts, clusterEnv):
if not propertyName in properties:
return self.getErrorItem("Value should be set")
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java
index cdbf2d7..e245e7f 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java
@@ -21,6 +21,7 @@ package org.apache.ambari.server.controller.internal;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorHelper;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
+import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.spi.Request;
import org.apache.ambari.server.controller.spi.RequestStatus;
@@ -49,13 +50,14 @@ public class ValidationResourceProviderTest {
doReturn(stackAdvisorRequest).when(provider).prepareStackAdvisorRequest(request);
StackAdvisorHelper saHelper = mock(StackAdvisorHelper.class);
+ Configuration configuration = mock(Configuration.class);
ValidationResponse response = mock(ValidationResponse.class);
Version version = mock(Version.class);
doReturn(3).when(response).getId();
doReturn(version).when(response).getVersion();
doReturn(response).when(saHelper).validate(any(StackAdvisorRequest.class));
- ValidationResourceProvider.init(saHelper);
+ ValidationResourceProvider.init(saHelper, configuration);
RequestStatus status = provider.createResources(request);
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 9e00704..e011847 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -2356,6 +2356,31 @@ class TestHDP206StackAdvisor(TestCase):
res = self.stackAdvisor.validateHDFSConfigurationsEnv(properties, recommendedDefaults, configurations, '', '')
self.assertEquals(res, res_expected)
+ def test_validateHDFSConfigurationsCoreSite(self):
+
+ configurations = {}
+ services = {"gpl-license-accepted": True, "services": [{"StackServices": {"service_name": "HDFS"}}]}
+
+ # 1) ok: gpl is allowed
+ properties = {'io.compression.codec.lzo.class': 'com.hadoop.compression.lzo.LzoCodec'}
+ res_expected = []
+
+ res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, {}, configurations, services, '')
+ self.assertEquals(res, res_expected)
+
+ # 2) fail: gpl is not allowed
+ services["gpl-license-accepted"] = False
+ res_expected = [{'config-type': 'core-site',
+ 'message': 'Your Ambari Server has not been configured to download LZO and install it. '
+ 'LZO is GPL software and requires you to accept a license prior to use. '
+ 'Please refer to this documentation to configure Ambari before proceeding.',
+ 'type': 'configuration',
+ 'config-name': 'io.compression.codec.lzo.class',
+ 'level': 'ERROR'}]
+
+ res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, {}, configurations, services, '')
+ self.assertEquals(res, res_expected)
+
def test_validateOneDataDirPerPartition(self):
recommendedDefaults = {
'dfs.datanode.du.reserved': '1024'
[24/24] ambari git commit: Merge branch 'branch-feature-AMBARI-22457'
into branch-2.6
Posted by jo...@apache.org.
Merge branch 'branch-feature-AMBARI-22457' into branch-2.6
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c43673f6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c43673f6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c43673f6
Branch: refs/heads/branch-2.6
Commit: c43673f69a150f07cc25821ecaede025f7d003e2
Parents: 754b1f6 ed50bc3
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Tue Nov 28 15:45:00 2017 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Tue Nov 28 15:45:00 2017 -0500
----------------------------------------------------------------------
.../stackVersions/StackVersionsEditCtrl.js | 27 ++++-
.../ui/admin-web/app/scripts/services/Stack.js | 14 +++
.../views/stackVersions/stackVersionPage.html | 28 +++--
.../libraries/functions/__init__.py | 2 +-
.../libraries/functions/copy_tarball.py | 99 ++++++++++++++-
.../libraries/functions/get_lzo_packages.py | 50 --------
.../libraries/functions/lzo_utils.py | 93 +++++++++++++++
.../libraries/functions/package_conditions.py | 8 +-
.../ambari/server/agent/ExecutionCommand.java | 1 +
.../stackadvisor/StackAdvisorRequest.java | 19 +++
.../commands/StackAdvisorCommand.java | 2 +
.../apache/ambari/server/checks/LZOCheck.java | 6 +-
.../server/configuration/Configuration.java | 12 ++
.../AmbariCustomCommandExecutionHelper.java | 14 +--
.../AmbariManagementControllerImpl.java | 8 +-
.../ambari/server/controller/AmbariServer.java | 3 +-
.../server/controller/RepositoryResponse.java | 23 +++-
.../internal/BlueprintResourceProvider.java | 36 +++---
.../internal/ClientConfigResourceProvider.java | 2 +
.../internal/RepositoryResourceProvider.java | 11 +-
.../internal/StackAdvisorResourceProvider.java | 9 +-
.../VersionDefinitionResourceProvider.java | 8 ++
.../server/orm/entities/RepositoryEntity.java | 20 ++++
.../apache/ambari/server/stack/RepoUtil.java | 1 +
.../ambari/server/state/RepositoryInfo.java | 24 +++-
.../ambari/server/state/stack/RepoTag.java | 34 ++++++
.../server/state/stack/RepositoryXml.java | 17 ++-
.../stack/upgrade/RepositoryVersionHelper.java | 19 +++
.../ambari/server/topology/Blueprint.java | 40 +++----
.../ambari/server/topology/BlueprintImpl.java | 18 +--
.../server/topology/BlueprintValidator.java | 5 +-
.../server/topology/BlueprintValidatorImpl.java | 23 +++-
.../GPLLicenseNotAcceptedException.java | 28 +++++
.../server/upgrade/AbstractUpgradeCatalog.java | 8 ++
.../server/upgrade/SchemaUpgradeHelper.java | 27 +++++
.../ambari/server/upgrade/UpgradeCatalog.java | 6 +
.../server/upgrade/UpgradeCatalog261.java | 119 +++++++++++++++++++
ambari-server/src/main/python/ambari-server.py | 2 +
.../python/ambari_server/serverConfiguration.py | 40 ++++++-
.../main/python/ambari_server/serverSetup.py | 10 +-
.../main/python/ambari_server/serverUpgrade.py | 18 ++-
.../common-services/HDFS/2.1.0.2.0/metainfo.xml | 30 -----
.../HDFS/2.1.0.2.0/package/scripts/hdfs.py | 6 +-
.../2.1.0.2.0/package/scripts/install_params.py | 10 --
.../2.1.0.2.0/package/scripts/params_linux.py | 7 --
.../HIVE/0.12.0.2.0/package/scripts/hive.py | 4 +
.../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 47 ++++----
.../4.0.0.2.0/package/scripts/params_linux.py | 8 +-
.../TEZ/0.4.0.2.1/package/scripts/tez.py | 5 +-
.../configuration-mapred/mapred-site.xml | 2 +-
.../YARN/2.1.0.2.0/package/scripts/yarn.py | 3 +
.../BigInsights/4.0/services/HDFS/metainfo.xml | 17 ---
.../BigInsights/4.2/services/HDFS/metainfo.xml | 20 ----
.../4.2/services/HDFS/package/scripts/hdfs.py | 5 +-
.../4.2/services/HIVE/package/scripts/hive.py | 3 +
.../4.2/services/OOZIE/package/scripts/oozie.py | 12 +-
.../services/OOZIE/package/scripts/params.py | 28 +----
.../4.2/services/YARN/package/scripts/yarn.py | 3 +
.../stacks/HDP/2.0.6/services/stack_advisor.py | 41 ++++++-
.../stacks/HDP/2.2/services/HDFS/metainfo.xml | 35 ------
.../2.2/services/TEZ/configuration/tez-site.xml | 4 +-
.../YARN/configuration-mapred/mapred-site.xml | 2 +-
.../stacks/HDP/2.3/services/HDFS/metainfo.xml | 30 -----
.../stacks/HDP/2.6/upgrades/config-upgrade.xml | 6 +-
.../src/main/resources/stacks/stack_advisor.py | 9 ++
.../src/main/resources/version_definition.xsd | 23 +++-
.../ambari/server/checks/LZOCheckTest.java | 13 +-
.../AmbariManagementControllerImplTest.java | 3 +-
.../ClientConfigResourceProviderTest.java | 2 +
.../RepositoryResourceProviderTest.java | 15 ++-
.../ValidationResourceProviderTest.java | 4 +-
.../state/repository/VersionDefinitionTest.java | 35 +++++-
.../upgrade/RepositoryVersionHelperTest.java | 2 +-
.../server/topology/BlueprintImplTest.java | 87 ++++++++++++--
.../topology/BlueprintValidatorImplTest.java | 18 +--
.../src/test/python/TestAmbariServer.py | 7 +-
.../stacks/2.0.6/common/test_stack_advisor.py | 33 +++++
.../stacks/2.2/configs/oozie-upgrade.json | 3 +-
.../resources/version_definition_with_tags.xml | 86 ++++++++++++++
ambari-web/app/controllers/installer.js | 53 +++++----
.../app/controllers/wizard/step8_controller.js | 12 +-
.../app/mappers/repository_version_mapper.js | 3 +-
ambari-web/app/mappers/stack_mapper.js | 3 +-
ambari-web/app/models/repository.js | 17 +++
.../app/models/stack_version/repository.js | 3 +-
ambari-web/app/templates/wizard/step1.hbs | 2 +
ambari-web/test/controllers/installer_test.js | 14 ++-
contrib/version-builder/version_builder.py | 13 +-
88 files changed, 1259 insertions(+), 463 deletions(-)
----------------------------------------------------------------------
[22/24] ambari git commit: AMBARI-22480. Validate blueprint does not
allow lzo enable without setup with license agreement. Added
io.compression.codecs property validation. (mpapirkovskyy)
Posted by jo...@apache.org.
AMBARI-22480. Validate blueprint does not allow lzo enable without setup with license agreement. Added io.compression.codecs property validation. (mpapirkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3a0ae86b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3a0ae86b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3a0ae86b
Branch: refs/heads/branch-2.6
Commit: 3a0ae86bbdbffb46cccb8bbe74c0d9093c18b561
Parents: 606d876
Author: Myroslav Papirkovskyi <mp...@hortonworks.com>
Authored: Tue Nov 28 14:51:58 2017 +0200
Committer: Myroslav Papirkovskyi <mp...@hortonworks.com>
Committed: Tue Nov 28 19:33:21 2017 +0200
----------------------------------------------------------------------
.../server/topology/BlueprintValidatorImpl.java | 4 +++-
.../server/topology/BlueprintImplTest.java | 20 +++++++++++++++++++-
2 files changed, 22 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/3a0ae86b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
index 7b7f7d7..2f0d97c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
@@ -45,6 +45,7 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
private final Stack stack;
public static final String LZO_CODEC_CLASS_PROPERTY_NAME = "io.compression.codec.lzo.class";
+ public static final String CODEC_CLASSES_PROPERTY_NAME = "io.compression.codecs";
public static final String LZO_CODEC_CLASS = "com.hadoop.compression.lzo.LzoCodec";
@Inject
@@ -110,7 +111,8 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
String propertyName = propertyEntry.getKey();
String propertyValue = propertyEntry.getValue();
if (propertyValue != null) {
- if (!gplEnabled && configType.equals("core-site") && propertyName.equals(LZO_CODEC_CLASS_PROPERTY_NAME)
+ if (!gplEnabled && configType.equals("core-site")
+ && (propertyName.equals(LZO_CODEC_CLASS_PROPERTY_NAME) || propertyName.equals(CODEC_CLASSES_PROPERTY_NAME))
&& propertyValue.contains(LZO_CODEC_CLASS)) {
throw new GPLLicenseNotAcceptedException("Your Ambari server has not been configured to download LZO GPL software. " +
"Please refer to documentation to configure Ambari before proceeding.");
http://git-wip-us.apache.org/repos/asf/ambari/blob/3a0ae86b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
index 1a9fd1f..5778404 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
@@ -176,7 +176,24 @@ public class BlueprintImplTest {
}
@Test(expected = GPLLicenseNotAcceptedException.class)
- public void testValidateConfigurations__gplIsNotAllowed() throws InvalidTopologyException,
+ public void testValidateConfigurations__gplIsNotAllowedCodecsProperty() throws InvalidTopologyException,
+ GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException {
+ Map<String, Map<String, String>> lzoProperties = new HashMap<>();
+ lzoProperties.put("core-site", new HashMap<String, String>(){{
+ put(BlueprintValidatorImpl.CODEC_CLASSES_PROPERTY_NAME, "OtherCodec, " + BlueprintValidatorImpl.LZO_CODEC_CLASS);
+ }});
+ Configuration lzoUsageConfiguration = new Configuration(lzoProperties, EMPTY_ATTRIBUTES, EMPTY_CONFIGURATION);
+
+ org.apache.ambari.server.configuration.Configuration serverConfig = setupConfigurationWithGPLLicense(false);
+ replay(stack, group1, group2, serverConfig);
+
+ Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, lzoUsageConfiguration, null);
+ blueprint.validateRequiredProperties();
+ verify(stack, group1, group2, serverConfig);
+ }
+
+ @Test(expected = GPLLicenseNotAcceptedException.class)
+ public void testValidateConfigurations__gplIsNotAllowedLZOProperty() throws InvalidTopologyException,
GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException {
Map<String, Map<String, String>> lzoProperties = new HashMap<>();
lzoProperties.put("core-site", new HashMap<String, String>(){{
@@ -198,6 +215,7 @@ public class BlueprintImplTest {
Map<String, Map<String, String>> lzoProperties = new HashMap<>();
lzoProperties.put("core-site", new HashMap<String, String>(){{
put(BlueprintValidatorImpl.LZO_CODEC_CLASS_PROPERTY_NAME, BlueprintValidatorImpl.LZO_CODEC_CLASS);
+ put(BlueprintValidatorImpl.CODEC_CLASSES_PROPERTY_NAME, "OtherCodec, " + BlueprintValidatorImpl.LZO_CODEC_CLASS);
}});
Configuration lzoUsageConfiguration = new Configuration(lzoProperties, EMPTY_ATTRIBUTES, EMPTY_CONFIGURATION);
[04/24] ambari git commit: AMBARI-22453. ambari-server setup should
surface GPL software agreement (aonishuk)
Posted by jo...@apache.org.
AMBARI-22453. ambari-server setup should surface GPL software agreement (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4b01b65d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4b01b65d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4b01b65d
Branch: refs/heads/branch-2.6
Commit: 4b01b65d9bdf7a5a170890dde13492a0871d444a
Parents: 6cc05d9
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Fri Nov 17 13:14:09 2017 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Fri Nov 17 13:14:09 2017 +0200
----------------------------------------------------------------------
ambari-server/src/test/python/TestAmbariServer.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4b01b65d/ambari-server/src/test/python/TestAmbariServer.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py
index eb04771..840b9a6 100644
--- a/ambari-server/src/test/python/TestAmbariServer.py
+++ b/ambari-server/src/test/python/TestAmbariServer.py
@@ -3684,7 +3684,8 @@ class TestAmbariServer(TestCase):
@patch("ambari_server.serverSetup.service_setup")
@patch("ambari_server.serverSetup.read_ambari_user")
@patch("ambari_server.serverSetup.expand_jce_zip_file")
- def test_setup_linux(self, expand_jce_zip_file_mock, read_ambari_user_mock,
+ @patch("ambari_server.serverSetup.write_gpl_license_accepted")
+ def test_setup_linux(self, write_gpl_license_accepted_mock, expand_jce_zip_file_mock, read_ambari_user_mock,
service_setup_mock, adjust_dirs_mock, extract_views_mock, proceedJDBCProperties_mock, is_root_mock,
disable_security_enhancements_mock, check_jdbc_drivers_mock, check_ambari_user_mock,
download_jdk_mock, configure_os_settings_mock, get_ambari_properties_mock,
@@ -3797,6 +3798,7 @@ class TestAmbariServer(TestCase):
check_jdbc_drivers_mock.return_value = 0
download_jdk_mock.return_value = 0
configure_os_settings_mock.return_value = 0
+ write_gpl_license_accepted_mock.return_value = 0
result = setup(args)
@@ -6825,7 +6827,6 @@ class TestAmbariServer(TestCase):
read_ambari_user_method, read_master_key_method,
get_is_persisted_method, get_is_secure_method, exists_mock,
save_passwd_for_alias_method):
-
is_root_method.return_value = True
p = Properties()
[23/24] ambari git commit: Merge branch 'branch-2.6' into
branch-feature-AMBARI-22457
Posted by jo...@apache.org.
Merge branch 'branch-2.6' into branch-feature-AMBARI-22457
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ed50bc38
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ed50bc38
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ed50bc38
Branch: refs/heads/branch-2.6
Commit: ed50bc3830e63e01961cbc03ef6d7c611c243c1c
Parents: 3a0ae86 754b1f6
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Tue Nov 28 15:39:47 2017 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Tue Nov 28 15:39:47 2017 -0500
----------------------------------------------------------------------
.../libraries/functions/component_version.py | 9 +-
.../ambari/logfeeder/filter/FilterKeyValue.java | 4 +-
.../configsets/hadoop_logs/conf/managed-schema | 3 +
.../model/response/ServiceLogData.java | 2 +
.../ambari/logsearch/solr/SolrConstants.java | 1 +
.../solr/model/SolrServiceLogData.java | 17 ++++
.../actionmanager/ExecutionCommandWrapper.java | 1 -
.../internal/UpgradeResourceProvider.java | 38 --------
.../dispatchers/AmbariSNMPDispatcher.java | 11 ++-
.../2.1.0.2.0/package/scripts/params_linux.py | 6 ++
.../custom_actions/scripts/ru_execute_tasks.py | 5 +-
.../4.0/properties/stack_packages.json | 6 ++
.../HDFS/package/scripts/namenode_upgrade.py | 8 +-
.../4.0/stack-advisor/stack_advisor_25.py | 2 +-
.../HDP/2.0.6/properties/stack_packages.json | 2 +-
.../services/YARN/configuration/yarn-site.xml | 4 +-
.../stacks/HDP/2.5/services/stack_advisor.py | 2 +-
.../stacks/HDP/2.6/services/OOZIE/metainfo.xml | 34 +++++++
.../stacks/HDP/2.6/services/stack_advisor.py | 3 +
.../stacks/HDP/2.6/upgrades/config-upgrade.xml | 14 ++-
.../HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 11 ++-
.../stacks/HDP/2.6/upgrades/upgrade-2.6.xml | 4 +
.../dispatchers/AmbariSNMPDispatcherTest.java | 97 +++++++++++---------
.../AlertNoticeDispatchServiceTest.java | 63 ++++++++++---
.../custom_actions/test_ru_execute_tasks.py | 6 +-
.../stacks/2.5/common/test_stack_advisor.py | 2 +-
.../stacks/2.6/common/test_stack_advisor.py | 11 +++
.../scripts/containers/ComponentDetailView.jsx | 2 +-
.../storm/src/main/resources/ui/package.json | 20 ++--
29 files changed, 257 insertions(+), 131 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/ed50bc38/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
----------------------------------------------------------------------
[03/24] ambari git commit: AMBARI-22461. VDF defined HDP-GPL repo
should be tagged appropriately (ncole)
Posted by jo...@apache.org.
AMBARI-22461. VDF defined HDP-GPL repo should be tagged appropriately (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6cc05d95
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6cc05d95
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6cc05d95
Branch: refs/heads/branch-2.6
Commit: 6cc05d95c16b0ead495159aa58be3b221bd67530
Parents: 46237fe
Author: Nate Cole <nc...@hortonworks.com>
Authored: Thu Nov 16 08:42:20 2017 -0500
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Thu Nov 16 11:03:04 2017 -0500
----------------------------------------------------------------------
.../AmbariManagementControllerImpl.java | 8 +-
.../server/controller/RepositoryResponse.java | 23 +++++-
.../internal/RepositoryResourceProvider.java | 11 ++-
.../VersionDefinitionResourceProvider.java | 8 ++
.../server/orm/entities/RepositoryEntity.java | 20 +++++
.../apache/ambari/server/stack/RepoUtil.java | 1 +
.../ambari/server/state/RepositoryInfo.java | 24 +++++-
.../ambari/server/state/stack/RepoTag.java | 34 ++++++++
.../server/state/stack/RepositoryXml.java | 17 +++-
.../stack/upgrade/RepositoryVersionHelper.java | 19 +++++
.../src/main/resources/version_definition.xsd | 23 +++++-
.../RepositoryResourceProviderTest.java | 15 ++--
.../state/repository/VersionDefinitionTest.java | 35 +++++++-
.../upgrade/RepositoryVersionHelperTest.java | 2 +-
.../resources/version_definition_with_tags.xml | 86 ++++++++++++++++++++
contrib/version-builder/version_builder.py | 13 ++-
16 files changed, 314 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index a6315f8..ffa8da3 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -88,7 +88,6 @@ import org.apache.ambari.server.actionmanager.HostRoleCommand;
import org.apache.ambari.server.actionmanager.RequestFactory;
import org.apache.ambari.server.actionmanager.Stage;
import org.apache.ambari.server.actionmanager.StageFactory;
-import org.apache.ambari.server.agent.CommandRepository;
import org.apache.ambari.server.agent.ExecutionCommand;
import org.apache.ambari.server.agent.ExecutionCommand.KeyNames;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
@@ -193,6 +192,7 @@ import org.apache.ambari.server.state.quicklinksprofile.QuickLinksProfile;
import org.apache.ambari.server.state.repository.VersionDefinitionXml;
import org.apache.ambari.server.state.scheduler.RequestExecutionFactory;
import org.apache.ambari.server.state.stack.OsFamily;
+import org.apache.ambari.server.state.stack.RepoTag;
import org.apache.ambari.server.state.stack.RepositoryXml;
import org.apache.ambari.server.state.stack.WidgetLayout;
import org.apache.ambari.server.state.stack.WidgetLayoutInfo;
@@ -4469,7 +4469,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
final RepositoryResponse response = new RepositoryResponse(repository.getBaseUrl(), osType,
repository.getRepositoryId(),
repository.getName(), repository.getDistribution(), repository.getComponents(), "", "", "",
- repository.getApplicableServices());
+ repository.getApplicableServices(),
+ repository.getTags());
if (null != versionDefinitionId) {
response.setVersionDefinitionId(versionDefinitionId);
} else {
@@ -4477,6 +4478,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
}
response.setStackName(repositoryVersion.getStackName());
response.setStackVersion(repositoryVersion.getStackVersion());
+
responses.add(response);
}
break;
@@ -4498,7 +4500,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
for (RepositoryXml.Repo repo : os.getRepos()) {
RepositoryResponse resp = new RepositoryResponse(repo.getBaseUrl(), os.getFamily(),
repo.getRepoId(), repo.getRepoName(), repo.getDistribution(), repo.getComponents(), repo.getMirrorsList(),
- repo.getBaseUrl(), repo.getLatestUri(), Collections.EMPTY_LIST);
+ repo.getBaseUrl(), repo.getLatestUri(), Collections.<String>emptyList(), Collections.<RepoTag>emptySet());
resp.setVersionDefinitionId(versionDefinitionId);
resp.setStackName(stackId.getStackName());
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryResponse.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryResponse.java
index 5ec46f0..46a82bb 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryResponse.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryResponse.java
@@ -19,9 +19,11 @@
package org.apache.ambari.server.controller;
import java.util.List;
+import java.util.Set;
import org.apache.ambari.annotations.Experimental;
import org.apache.ambari.annotations.ExperimentalFeature;
+import org.apache.ambari.server.state.stack.RepoTag;
public class RepositoryResponse {
@@ -45,9 +47,12 @@ public class RepositoryResponse {
comment = "Remove logic for handling custom service repos after enabling multi-mpack cluster deployment")
private List<String> applicableServices;
+ private Set<RepoTag> tags;
+
public RepositoryResponse(String baseUrl, String osType, String repoId,
String repoName, String distribution, String components,
- String mirrorsList, String defaultBaseUrl, String latestBaseUrl, List<String> applicableServices) {
+ String mirrorsList, String defaultBaseUrl, String latestBaseUrl, List<String> applicableServices,
+ Set<RepoTag> repoTags) {
setBaseUrl(baseUrl);
setOsType(osType);
setRepoId(repoId);
@@ -58,6 +63,7 @@ public class RepositoryResponse {
setDefaultBaseUrl(defaultBaseUrl);
setLatestBaseUrl(latestBaseUrl);
setApplicableServices(applicableServices);
+ setTags(repoTags);
}
public String getStackName() {
@@ -203,4 +209,19 @@ public class RepositoryResponse {
public void setApplicableServices(List<String> applicableServices) {
this.applicableServices = applicableServices;
}
+
+ /**
+ * @return the repo tags
+ */
+ public Set<RepoTag> getTags() {
+ return tags;
+ }
+
+ /**
+ * @param repoTags the repo tags
+ */
+ public void setTags(Set<RepoTag> repoTags) {
+ tags = repoTags;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java
index 3c10e43..cec43dd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java
@@ -64,6 +64,7 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid
public static final String REPOSITORY_REPOSITORY_VERSION_ID_PROPERTY_ID = PropertyHelper.getPropertyId("Repositories", "repository_version_id");
public static final String REPOSITORY_VERSION_DEFINITION_ID_PROPERTY_ID = PropertyHelper.getPropertyId("Repositories", "version_definition_id");
public static final String REPOSITORY_UNIQUE_PROPERTY_ID = PropertyHelper.getPropertyId("Repositories", "unique");
+ public static final String REPOSITORY_TAGS_PROPERTY_ID = PropertyHelper.getPropertyId("Repositories", "tags");
@Experimental(feature = ExperimentalFeature.CUSTOM_SERVICE_REPOS,
comment = "Remove logic for handling custom service repos after enabling multi-mpack cluster deployment")
public static final String REPOSITORY_APPLICABLE_SERVICES_PROPERTY_ID = PropertyHelper.getPropertyId("Repositories", "applicable_services");
@@ -98,6 +99,7 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid
add(REPOSITORY_CLUSTER_STACK_VERSION_PROPERTY_ID);
add(REPOSITORY_UNIQUE_PROPERTY_ID);
add(REPOSITORY_APPLICABLE_SERVICES_PROPERTY_ID);
+ add(REPOSITORY_TAGS_PROPERTY_ID);
}
};
@@ -123,7 +125,7 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid
throws SystemException, UnsupportedPropertyException,
NoSuchResourceException, NoSuchParentResourceException {
- final Set<RepositoryRequest> requests = new HashSet<RepositoryRequest>();
+ final Set<RepositoryRequest> requests = new HashSet<>();
Iterator<Map<String,Object>> iterator = request.getProperties().iterator();
if (iterator.hasNext()) {
@@ -148,7 +150,7 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid
throws SystemException, UnsupportedPropertyException,
NoSuchResourceException, NoSuchParentResourceException {
- final Set<RepositoryRequest> requests = new HashSet<RepositoryRequest>();
+ final Set<RepositoryRequest> requests = new HashSet<>();
if (predicate == null) {
requests.add(getRequest(Collections.<String, Object>emptyMap()));
@@ -166,7 +168,7 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid
}
});
- Set<Resource> resources = new HashSet<Resource>();
+ Set<Resource> resources = new HashSet<>();
for (RepositoryResponse response : responses) {
Resource resource = new ResourceImpl(Resource.Type.Repository);
@@ -184,6 +186,7 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid
setResourceProperty(resource, REPOSITORY_LATEST_BASE_URL_PROPERTY_ID, response.getLatestBaseUrl(), requestedIds);
setResourceProperty(resource, REPOSITORY_UNIQUE_PROPERTY_ID, response.isUnique(), requestedIds);
setResourceProperty(resource, REPOSITORY_APPLICABLE_SERVICES_PROPERTY_ID, response.getApplicableServices(), requestedIds);
+ setResourceProperty(resource, REPOSITORY_TAGS_PROPERTY_ID, response.getTags(), requestedIds);
if (null != response.getClusterVersionId()) {
setResourceProperty(resource, REPOSITORY_CLUSTER_STACK_VERSION_PROPERTY_ID, response.getClusterVersionId(), requestedIds);
}
@@ -207,7 +210,7 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid
public RequestStatus createResources(Request request) throws SystemException, UnsupportedPropertyException, ResourceAlreadyExistsException, NoSuchParentResourceException {
final String validateOnlyProperty = request.getRequestInfoProperties().get(RepositoryResourceDefinition.VALIDATE_ONLY_DIRECTIVE);
if (BooleanUtils.toBoolean(validateOnlyProperty)) {
- final Set<RepositoryRequest> requests = new HashSet<RepositoryRequest>();
+ final Set<RepositoryRequest> requests = new HashSet<>();
final Iterator<Map<String,Object>> iterator = request.getProperties().iterator();
if (iterator.hasNext()) {
for (Map<String, Object> propertyMap : request.getProperties()) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java
index 3ca81d7..6ab863b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java
@@ -63,6 +63,7 @@ import org.apache.ambari.server.state.RepositoryType;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.StackInfo;
import org.apache.ambari.server.state.repository.VersionDefinitionXml;
+import org.apache.ambari.server.state.stack.RepoTag;
import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.IOUtils;
@@ -780,6 +781,13 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc
repoElement.put(PropertyHelper.getPropertyName(RepositoryResourceProvider.REPOSITORY_STACK_VERSION_PROPERTY_ID),
entity.getStackVersion());
+ ArrayNode tagsNode = factory.arrayNode();
+ for (RepoTag repoTag : repo.getTags()) {
+ tagsNode.add(repoTag.toString());
+ }
+ repoElement.put(PropertyHelper.getPropertyName(
+ RepositoryResourceProvider.REPOSITORY_TAGS_PROPERTY_ID), tagsNode);
+
@Experimental(feature = ExperimentalFeature.CUSTOM_SERVICE_REPOS,
comment = "Remove logic for handling custom service repos after enabling multi-mpack cluster deployment")
ArrayNode applicableServicesNode = factory.arrayNode();
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java
index 034f3bc..5f23ca7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java
@@ -17,10 +17,13 @@
*/
package org.apache.ambari.server.orm.entities;
+import java.util.Collections;
import java.util.List;
+import java.util.Set;
import org.apache.ambari.annotations.Experimental;
import org.apache.ambari.annotations.ExperimentalFeature;
+import org.apache.ambari.server.state.stack.RepoTag;
/**
* Emulates entity to provide a quick way to change it to real entity in future.
@@ -38,6 +41,8 @@ public class RepositoryEntity {
comment = "Remove logic for handling custom service repos after enabling multi-mpack cluster deployment")
private List<String> applicableServices;
+ private Set<RepoTag> tags;
+
public String getName() {
return name;
}
@@ -106,6 +111,20 @@ public class RepositoryEntity {
this.applicableServices = applicableServices;
}
+ /**
+ * @return the repo tags
+ */
+ public Set<RepoTag> getTags() {
+ return tags == null ? Collections.<RepoTag>emptySet() : tags;
+ }
+
+ /**
+ * @param repoTags the tags to set
+ */
+ public void setTags(Set<RepoTag> repoTags) {
+ tags = repoTags;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
@@ -132,4 +151,5 @@ public class RepositoryEntity {
result = 31 * result + (applicableServices != null ? applicableServices.hashCode() : 0);
return result;
}
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java
index 81b4da9..166abb5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java
@@ -200,6 +200,7 @@ public class RepoUtil {
re.setDistribution(repoInfo.getDistribution());
re.setComponents(repoInfo.getComponents());
re.setApplicableServices(repoInfo.getApplicableServices());
+ re.setTags(repoInfo.getTags());
return re;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
index bd53c11..9272d56 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
@@ -18,16 +18,19 @@
package org.apache.ambari.server.state;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
-import com.google.common.base.Objects;
+import java.util.Set;
import org.apache.ambari.annotations.Experimental;
import org.apache.ambari.annotations.ExperimentalFeature;
import org.apache.ambari.server.controller.RepositoryResponse;
+import org.apache.ambari.server.state.stack.RepoTag;
import org.apache.commons.lang.StringUtils;
import com.google.common.base.Function;
+import com.google.common.base.Objects;
import com.google.common.base.Strings;
public class RepositoryInfo {
@@ -46,6 +49,7 @@ public class RepositoryInfo {
@Experimental(feature = ExperimentalFeature.CUSTOM_SERVICE_REPOS,
comment = "Remove logic for handling custom service repos after enabling multi-mpack cluster deployment")
private List<String> applicableServices = new LinkedList<>();
+ private Set<RepoTag> tags = new HashSet<>();
/**
* @return the baseUrl
@@ -229,7 +233,8 @@ public class RepositoryInfo {
public RepositoryResponse convertToResponse()
{
return new RepositoryResponse(getBaseUrl(), getOsType(), getRepoId(), getRepoName(), getDistribution(),
- getComponents(), getMirrorsList(), getDefaultBaseUrl(), getLatestBaseUrl(), getApplicableServices());
+ getComponents(), getMirrorsList(), getDefaultBaseUrl(), getLatestBaseUrl(), getApplicableServices(),
+ getTags());
}
/**
@@ -297,4 +302,19 @@ public class RepositoryInfo {
public void setAmbariManagedRepositories(boolean ambariManagedRepositories) {
this.ambariManagedRepositories = ambariManagedRepositories;
}
+
+ /**
+ * @return the tags for this repository
+ */
+ public Set<RepoTag> getTags() {
+ return tags;
+ }
+
+ /**
+ * @param repoTags the tags for this repository
+ */
+ public void setTags(Set<RepoTag> repoTags) {
+ tags = repoTags;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepoTag.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepoTag.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepoTag.java
new file mode 100644
index 0000000..08a2635
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepoTag.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.state.stack;
+
+import javax.xml.bind.annotation.XmlEnum;
+
+/**
+ * A Repo tag is a way to allow Ambari to place logic surrounding repository details.
+ * This is used instead of random strings to tightly control how tags are used.
+ */
+@XmlEnum
+public enum RepoTag {
+
+ /**
+ * The repository may contain GPL-Licensed software
+ */
+ GPL
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java
index 79ace89..e468566 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java
@@ -27,6 +27,7 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlElementWrapper;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
@@ -43,7 +44,7 @@ public class RepositoryXml implements Validable{
@XmlElement(name="latest")
private String latestUri;
@XmlElement(name="os")
- private List<Os> oses = new ArrayList<Os>();
+ private List<Os> oses = new ArrayList<>();
@XmlTransient
private boolean valid = true;
@@ -67,7 +68,7 @@ public class RepositoryXml implements Validable{
}
@XmlTransient
- private Set<String> errorSet = new HashSet<String>();
+ private Set<String> errorSet = new HashSet<>();
@Override
public void addError(String error) {
@@ -151,6 +152,10 @@ public class RepositoryXml implements Validable{
private String latest = null;
private boolean unique = false;
+ @XmlElementWrapper(name="tags")
+ @XmlElement(name="tag")
+ private Set<RepoTag> tags = new HashSet<>();
+
private Repo() {
}
@@ -207,6 +212,13 @@ public class RepositoryXml implements Validable{
public void setUnique(boolean unique) {
this.unique = unique;
}
+
+ /**
+ * @return the repo tags
+ */
+ public Set<RepoTag> getTags() {
+ return tags;
+ }
}
/**
@@ -231,6 +243,7 @@ public class RepositoryXml implements Validable{
ri.setComponents(r.getComponents());
ri.setLatestBaseUrl(r.getBaseUrl());
ri.setUnique(r.isUnique());
+ ri.setTags(r.tags);
repos.add(ri);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
index a77caab..5c12922 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
@@ -20,6 +20,7 @@ package org.apache.ambari.server.state.stack.upgrade;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -57,6 +58,7 @@ import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.repository.ClusterVersionSummary;
import org.apache.ambari.server.state.repository.VersionDefinitionXml;
import org.apache.ambari.server.state.stack.OsFamily;
+import org.apache.ambari.server.state.stack.RepoTag;
import org.apache.ambari.server.state.stack.UpgradePack;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
@@ -196,6 +198,7 @@ public class RepositoryVersionHelper {
if (repositoryJson.getAsJsonObject().get(RepositoryResourceProvider.REPOSITORY_UNIQUE_PROPERTY_ID) != null) {
repositoryEntity.setUnique(repositoryJson.getAsJsonObject().get(RepositoryResourceProvider.REPOSITORY_UNIQUE_PROPERTY_ID).getAsBoolean());
}
+
if (repositoryJson.get(RepositoryResourceProvider.REPOSITORY_APPLICABLE_SERVICES_PROPERTY_ID) != null) {
List<String> applicableServices = new LinkedList<>();
JsonArray jsonArray = repositoryJson.get(RepositoryResourceProvider.REPOSITORY_APPLICABLE_SERVICES_PROPERTY_ID).getAsJsonArray();
@@ -204,6 +207,17 @@ public class RepositoryVersionHelper {
}
repositoryEntity.setApplicableServices(applicableServices);
}
+
+ if (null != repositoryJson.get(RepositoryResourceProvider.REPOSITORY_TAGS_PROPERTY_ID)) {
+ Set<RepoTag> tags = new HashSet<>();
+
+ JsonArray jsonArray = repositoryJson.get(RepositoryResourceProvider.REPOSITORY_TAGS_PROPERTY_ID).getAsJsonArray();
+ for(JsonElement je : jsonArray) {
+ tags.add(RepoTag.valueOf(je.getAsString()));
+ }
+ repositoryEntity.setTags(tags);
+ }
+
operatingSystemEntity.getRepositories().add(repositoryEntity);
}
operatingSystems.add(operatingSystemEntity);
@@ -256,6 +270,11 @@ public class RepositoryVersionHelper {
repositoryJson.addProperty(RepositoryResourceProvider.REPOSITORY_MIRRORS_LIST_PROPERTY_ID, repository.getMirrorsList());
repositoryJson.addProperty(RepositoryResourceProvider.REPOSITORY_UNIQUE_PROPERTY_ID, repository.isUnique());
+
+ // add the tags even if there are none
+ JsonArray tags = gson.toJsonTree(repository.getTags()).getAsJsonArray();
+ repositoryJson.add(RepositoryResourceProvider.REPOSITORY_TAGS_PROPERTY_ID, tags);
+
if(repository.getApplicableServices() != null) {
Gson gson = new GsonBuilder().create();
JsonArray applicableServicesJson = gson.toJsonTree(repository.getApplicableServices()).getAsJsonArray();
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/resources/version_definition.xsd
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/version_definition.xsd b/ambari-server/src/main/resources/version_definition.xsd
index eaed31d..db10d67 100644
--- a/ambari-server/src/main/resources/version_definition.xsd
+++ b/ambari-server/src/main/resources/version_definition.xsd
@@ -63,7 +63,13 @@
<xs:enumeration value="suse12" />
</xs:restriction>
</xs:simpleType>
-
+
+ <xs:simpleType name="tag-type">
+ <xs:restriction base="xs:NMTOKEN">
+ <xs:enumeration value="GPL" />
+ </xs:restriction>
+ </xs:simpleType>
+
<xs:complexType name="manifest-service-type">
<xs:annotation>
<xs:documentation>
@@ -113,6 +119,20 @@
</xs:element>
</xs:sequence>
</xs:complexType>
+
+ <xs:complexType name="tags-type">
+ <xs:annotation>
+ <xs:documentation>
+ Tags are a way to mark a repository for special purposes. There
+ may be logic surrounding a tag in the code, or used by agents.
+
+ Tags are NOT meant to be random strings, they must be defined
+ </xs:documentation>
+ </xs:annotation>
+ <xs:sequence>
+ <xs:element name="tag" type="tag-type" minOccurs="0" maxOccurs="unbounded" />
+ </xs:sequence>
+ </xs:complexType>
<xs:complexType name="repository-info-type">
<xs:sequence>
@@ -130,6 +150,7 @@
<xs:element name="components" type="xs:string" minOccurs="0" maxOccurs="1" />
<xs:element name="mirrorslist" type="xs:string" minOccurs="0" maxOccurs="1" />
<xs:element name="unique" type="xs:boolean" minOccurs="0" maxOccurs="1" />
+ <xs:element name="tags" type="tags-type" minOccurs="0" maxOccurs="1" />
</xs:sequence>
</xs:complexType>
</xs:element>
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryResourceProviderTest.java
index c46cce7..2c01809 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryResourceProviderTest.java
@@ -23,6 +23,7 @@ import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify;
+import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -37,6 +38,7 @@ import org.apache.ambari.server.controller.spi.Resource;
import org.apache.ambari.server.controller.spi.ResourceProvider;
import org.apache.ambari.server.controller.utilities.PredicateBuilder;
import org.apache.ambari.server.controller.utilities.PropertyHelper;
+import org.apache.ambari.server.state.stack.RepoTag;
import org.easymock.EasyMock;
import org.junit.Assert;
import org.junit.Test;
@@ -57,10 +59,11 @@ public class RepositoryResourceProviderTest {
AmbariManagementController managementController = EasyMock.createMock(AmbariManagementController.class);
RepositoryResponse rr = new RepositoryResponse(VAL_BASE_URL, VAL_OS,
- VAL_REPO_ID, VAL_REPO_NAME, VAL_DISTRIBUTION, VAL_COMPONENT_NAME, null, null, null, null);
+ VAL_REPO_ID, VAL_REPO_NAME, VAL_DISTRIBUTION, VAL_COMPONENT_NAME, null, null, null, null,
+ Collections.<RepoTag>emptySet());
rr.setStackName(VAL_STACK_NAME);
rr.setStackVersion(VAL_STACK_VERSION);
- Set<RepositoryResponse> allResponse = new HashSet<RepositoryResponse>();
+ Set<RepositoryResponse> allResponse = new HashSet<>();
allResponse.add(rr);
// set expectations
@@ -71,7 +74,7 @@ public class RepositoryResourceProviderTest {
ResourceProvider provider = new RepositoryResourceProvider(managementController);
- Set<String> propertyIds = new HashSet<String>();
+ Set<String> propertyIds = new HashSet<>();
propertyIds.add(RepositoryResourceProvider.REPOSITORY_STACK_NAME_PROPERTY_ID);
propertyIds.add(RepositoryResourceProvider.REPOSITORY_STACK_VERSION_PROPERTY_ID);
propertyIds.add(RepositoryResourceProvider.REPOSITORY_REPO_NAME_PROPERTY_ID);
@@ -169,8 +172,8 @@ public class RepositoryResourceProviderTest {
AmbariManagementController managementController = EasyMock.createMock(AmbariManagementController.class);
RepositoryResponse rr = new RepositoryResponse(VAL_BASE_URL, VAL_OS,
- VAL_REPO_ID, VAL_REPO_NAME, null, null, null, null ,null, null);
- Set<RepositoryResponse> allResponse = new HashSet<RepositoryResponse>();
+ VAL_REPO_ID, VAL_REPO_NAME, null, null, null, null ,null, null, Collections.<RepoTag>emptySet());
+ Set<RepositoryResponse> allResponse = new HashSet<>();
allResponse.add(rr);
// set expectations
@@ -183,7 +186,7 @@ public class RepositoryResourceProviderTest {
ResourceProvider provider = new RepositoryResourceProvider(managementController);
// add the property map to a set for the request.
- Map<String, Object> properties = new LinkedHashMap<String, Object>();
+ Map<String, Object> properties = new LinkedHashMap<>();
properties.put(RepositoryResourceProvider.REPOSITORY_BASE_URL_PROPERTY_ID, "http://garbage.eu.co");
// create the request
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/test/java/org/apache/ambari/server/state/repository/VersionDefinitionTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/repository/VersionDefinitionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/repository/VersionDefinitionTest.java
index 6f33c7e..9fe6146 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/repository/VersionDefinitionTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/repository/VersionDefinitionTest.java
@@ -40,7 +40,10 @@ import org.apache.ambari.server.state.RepositoryType;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.ServiceInfo;
import org.apache.ambari.server.state.StackInfo;
+import org.apache.ambari.server.state.stack.RepoTag;
import org.apache.ambari.server.state.stack.RepositoryXml;
+import org.apache.ambari.server.state.stack.RepositoryXml.Os;
+import org.apache.ambari.server.state.stack.RepositoryXml.Repo;
import org.apache.commons.io.FileUtils;
import org.junit.Test;
@@ -186,15 +189,41 @@ public class VersionDefinitionTest {
public void testSerialization() throws Exception {
File f = new File("src/test/resources/version_definition_test_all_services.xml");
-
VersionDefinitionXml xml = VersionDefinitionXml.load(f.toURI().toURL());
-
String xmlString = xml.toXml();
-
xml = VersionDefinitionXml.load(xmlString);
assertNotNull(xml.release.build);
assertEquals("1234", xml.release.build);
+
+ f = new File("src/test/resources/version_definition_with_tags.xml");
+ xml = VersionDefinitionXml.load(f.toURI().toURL());
+ xmlString = xml.toXml();
+
+ xml = VersionDefinitionXml.load(xmlString);
+
+ assertEquals(2, xml.repositoryInfo.getOses().size());
+ List<Repo> repos = null;
+ for (Os os : xml.repositoryInfo.getOses()) {
+ if (os.getFamily().equals("redhat6")) {
+ repos = os.getRepos();
+ }
+ }
+ assertNotNull(repos);
+ assertEquals(3, repos.size());
+
+ Repo found = null;
+ for (Repo repo : repos) {
+ if (repo.getRepoName().equals("HDP-GPL")) {
+ found = repo;
+ break;
+ }
+ }
+
+ assertNotNull(found);
+ assertNotNull(found.getTags());
+ assertEquals(1, found.getTags().size());
+ assertEquals(RepoTag.GPL, found.getTags().iterator().next());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelperTest.java
index 70ff801..171eb88 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelperTest.java
@@ -53,6 +53,6 @@ public class RepositoryVersionHelperTest {
repositories.add(repository);
final String serialized = helper.serializeOperatingSystems(repositories);
- Assert.assertEquals("[{\"OperatingSystems/ambari_managed_repositories\":true,\"repositories\":[{\"Repositories/base_url\":\"baseurl\",\"Repositories/repo_id\":\"repoId\",\"Repositories/unique\":true,\"Repositories/applicable_services\":[]}],\"OperatingSystems/os_type\":\"os\"}]", serialized);
+ Assert.assertEquals("[{\"OperatingSystems/ambari_managed_repositories\":true,\"repositories\":[{\"Repositories/base_url\":\"baseurl\",\"Repositories/repo_id\":\"repoId\",\"Repositories/unique\":true,\"Repositories/tags\":[],\"Repositories/applicable_services\":[]}],\"OperatingSystems/os_type\":\"os\"}]", serialized);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/test/resources/version_definition_with_tags.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/version_definition_with_tags.xml b/ambari-server/src/test/resources/version_definition_with_tags.xml
new file mode 100644
index 0000000..fe4266b
--- /dev/null
+++ b/ambari-server/src/test/resources/version_definition_with_tags.xml
@@ -0,0 +1,86 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<repository-version xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:noNamespaceSchemaLocation="version_definition.xsd">
+
+ <release>
+ <type>PATCH</type>
+ <stack-id>HDP-2.3</stack-id>
+ <version>2.3.4.1</version>
+ <build>1234</build>
+ <compatible-with>2.3.4.[1-9]</compatible-with>
+ <release-notes>http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.4/</release-notes>
+ </release>
+
+ <manifest>
+ <service id="HDFS-271" name="HDFS" version="2.7.1" version-id="10" />
+ <service id="HIVE-110" name="HIVE" version="1.1.0" />
+ <service id="HIVE-200" name="HIVE" version="2.0.0" />
+ <service id="HBASE-899" name="HBASE" version="8.9.9" />
+ </manifest>
+
+ <available-services />
+
+ <repository-info>
+ <os family="redhat6">
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.0.0</baseurl>
+ <repoid>HDP-2.3</repoid>
+ <reponame>HDP</reponame>
+ <unique>true</unique>
+ </repo>
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6</baseurl>
+ <repoid>HDP-UTILS-1.1.0.20</repoid>
+ <reponame>HDP-UTILS</reponame>
+ <unique>false</unique>
+ </repo>
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP-GPL/repos/centos6</baseurl>
+ <repoid>HDP-GPL</repoid>
+ <reponame>HDP-GPL</reponame>
+ <unique>false</unique>
+ <tags>
+ <tag>GPL</tag>
+ </tags>
+ </repo>
+ </os>
+ <os family="redhat7">
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP/centos7/2.x/updates/2.3.0.0</baseurl>
+ <repoid>HDP-2.3</repoid>
+ <reponame>HDP</reponame>
+ <unique>true</unique>
+ </repo>
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos7</baseurl>
+ <repoid>HDP-UTILS-1.1.0.20</repoid>
+ <reponame>HDP-UTILS</reponame>
+ <unique>false</unique>
+ </repo>
+ </os>
+
+ </repository-info>
+
+ <upgrade>
+ <configuration type="hdfs-site">
+ <set key="foo" value="bar" />
+ </configuration>
+ </upgrade>
+</repository-version>
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/contrib/version-builder/version_builder.py
----------------------------------------------------------------------
diff --git a/contrib/version-builder/version_builder.py b/contrib/version-builder/version_builder.py
index 5bf5430..ac7ddd4 100644
--- a/contrib/version-builder/version_builder.py
+++ b/contrib/version-builder/version_builder.py
@@ -176,7 +176,7 @@ class VersionBuilder:
e = ET.SubElement(service_element, 'component')
e.text = component
- def add_repo(self, os_family, repo_id, repo_name, base_url, unique):
+ def add_repo(self, os_family, repo_id, repo_name, base_url, unique, tags):
"""
Adds a repository
"""
@@ -211,6 +211,13 @@ class VersionBuilder:
e = ET.SubElement(repo_element, 'unique')
e.text = unique
+ if tags is not None:
+ e = ET.SubElement(repo_element, 'tags')
+ tag_names = tags.split(',')
+ for tag in tag_names:
+ t = ET.SubElement(e, 'tag')
+ t.text = tag
+
def _check_xmllint(self):
"""
@@ -326,7 +333,8 @@ def process_repo(vb, options):
if not options.repo:
return
- vb.add_repo(options.repo_os, options.repo_id, options.repo_name, options.repo_url, options.unique)
+ vb.add_repo(options.repo_os, options.repo_id, options.repo_name, options.repo_url,
+ options.unique, options.repo_tags)
def validate_manifest(parser, options):
"""
@@ -444,6 +452,7 @@ def main(argv):
help="Indicates base url should be unique")
parser.add_option('--repo-id', dest='repo_id', help="The ID of the repo")
parser.add_option('--repo-name', dest='repo_name', help="The name of the repo")
+ parser.add_option('--repo-tags', dest='repo_tags', help="The CSV tags for the repo")
(options, args) = parser.parse_args()
[12/24] ambari git commit: AMBARI-22486 - Conditionally Rebuild
MapReduce and Tez Tarballs with LZO if Enabled (jonathanhurley)
Posted by jo...@apache.org.
AMBARI-22486 - Conditionally Rebuild MapReduce and Tez Tarballs with LZO if Enabled (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/58c7f784
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/58c7f784
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/58c7f784
Branch: refs/heads/branch-2.6
Commit: 58c7f784abe9b6f05a1358c14a8f94afc6725f79
Parents: 519f527
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Mon Nov 20 14:23:20 2017 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Tue Nov 21 20:32:13 2017 -0500
----------------------------------------------------------------------
.../libraries/functions/copy_tarball.py | 99 +++++++++++++++++++-
.../TEZ/0.4.0.2.1/package/scripts/tez.py | 5 +-
.../configuration-mapred/mapred-site.xml | 2 +-
.../2.2/services/TEZ/configuration/tez-site.xml | 4 +-
.../YARN/configuration-mapred/mapred-site.xml | 2 +-
.../stacks/HDP/2.6/upgrades/config-upgrade.xml | 6 +-
6 files changed, 106 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/58c7f784/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
index b05c97c..bf0701c 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
@@ -28,6 +28,8 @@ from contextlib import closing
from resource_management.libraries.script.script import Script
from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.functions import component_version
+from resource_management.libraries.functions import lzo_utils
from resource_management.libraries.functions.default import default
from resource_management.core import shell
from resource_management.core import sudo
@@ -45,6 +47,9 @@ STACK_VERSION_PATTERN = "{{ stack_version }}"
def _prepare_tez_tarball():
"""
Prepares the Tez tarball by adding the Hadoop native libraries found in the mapreduce tarball.
+ It's very important to use the version of mapreduce which matches tez here.
+ Additionally, this will also copy native LZO to the tez tarball if LZO is enabled and the
+ GPL license has been accepted.
:return: the full path of the newly created tez tarball to use
"""
import tempfile
@@ -79,12 +84,31 @@ def _prepare_tez_tarball():
if not os.path.exists(tez_lib_dir):
raise Fail("Unable to seed the Tez tarball with native libraries since the target Tez lib directory {0} does not exist".format(tez_lib_dir))
- # ensure that the tez/lib directory is readable by non-root (which it typically is not)
- sudo.chmod(tez_lib_dir, 0755)
-
# copy native libraries from hadoop to tez
Execute(("cp", "-a", hadoop_lib_native_dir, tez_lib_dir), sudo = True)
+ # if enabled, LZO GPL libraries must be copied as well
+ if lzo_utils.should_install_lzo():
+ stack_root = Script.get_stack_root()
+ tez_version = component_version.get_component_repository_version("TEZ")
+ hadoop_lib_native_lzo_dir = os.path.join(stack_root, tez_version, "hadoop", "lib", "native")
+
+ if not sudo.path_isdir(hadoop_lib_native_lzo_dir):
+ Logger.warning("Unable to located native LZO libraries at {0}, falling back to hadoop home".format(hadoop_lib_native_lzo_dir))
+ hadoop_lib_native_lzo_dir = os.path.join(stack_root, "current", "hadoop-client", "lib", "native")
+
+ if not sudo.path_isdir(hadoop_lib_native_lzo_dir):
+ raise Fail("Unable to seed the Tez tarball with native libraries since LZO is enabled but the native LZO libraries could not be found at {0}".format(hadoop_lib_native_lzo_dir))
+
+ Execute(("cp", "-a", hadoop_lib_native_lzo_dir, tez_lib_dir), sudo = True)
+
+
+ # ensure that the tez/lib directory is readable by non-root (which it typically is not)
+ Directory(tez_lib_dir,
+ mode = 0755,
+ cd_access = 'a',
+ recursive_ownership = True)
+
# create the staging directory so that non-root agents can write to it
tez_native_tarball_staging_dir = os.path.join(temp_dir, "tez-native-tarball-staging")
if not os.path.exists(tez_native_tarball_staging_dir):
@@ -111,6 +135,72 @@ def _prepare_tez_tarball():
return tez_tarball_with_native_lib
+def _prepare_mapreduce_tarball():
+ """
+ Prepares the mapreduce tarball by including the native LZO libraries if necessary. If LZO is
+ not enabled or has not been opted-in, then this will do nothing and return the original
+ tarball to upload to HDFS.
+ :return: the full path of the newly created mapreduce tarball to use or the original path
+ if no changes were made
+ """
+ # get the mapreduce tarball to crack open and add LZO libraries to
+ _, mapreduce_source_file, _, _ = get_tarball_paths("mapreduce")
+
+ if not lzo_utils.should_install_lzo():
+ return mapreduce_source_file
+
+ Logger.info("Preparing the mapreduce tarball with native LZO libraries...")
+
+ temp_dir = Script.get_tmp_dir()
+
+ # create the temp staging directories ensuring that non-root agents using tarfile can work with them
+ mapreduce_temp_dir = tempfile.mkdtemp(prefix="mapreduce-tarball-", dir=temp_dir)
+ sudo.chmod(mapreduce_temp_dir, 0777)
+
+ # calculate the source directory for LZO
+ hadoop_lib_native_source_dir = os.path.join(os.path.dirname(mapreduce_source_file), "lib", "native")
+ if not sudo.path_exists(hadoop_lib_native_source_dir):
+ raise Fail("Unable to seed the mapreduce tarball with native LZO libraries since the source Hadoop native lib directory {0} does not exist".format(hadoop_lib_native_source_dir))
+
+ Logger.info("Extracting {0} to {1}".format(mapreduce_source_file, mapreduce_temp_dir))
+ tar_archive.extract_archive(mapreduce_source_file, mapreduce_temp_dir)
+
+ mapreduce_lib_dir = os.path.join(mapreduce_temp_dir, "hadoop", "lib")
+
+ # copy native libraries from source hadoop to target
+ Execute(("cp", "-af", hadoop_lib_native_source_dir, mapreduce_lib_dir), sudo = True)
+
+ # ensure that the hadoop/lib/native directory is readable by non-root (which it typically is not)
+ Directory(mapreduce_lib_dir,
+ mode = 0755,
+ cd_access = 'a',
+ recursive_ownership = True)
+
+ # create the staging directory so that non-root agents can write to it
+ mapreduce_native_tarball_staging_dir = os.path.join(temp_dir, "mapreduce-native-tarball-staging")
+ if not os.path.exists(mapreduce_native_tarball_staging_dir):
+ Directory(mapreduce_native_tarball_staging_dir,
+ mode = 0777,
+ cd_access = 'a',
+ create_parents = True,
+ recursive_ownership = True)
+
+ mapreduce_tarball_with_native_lib = os.path.join(mapreduce_native_tarball_staging_dir, "mapreduce-native.tar.gz")
+ Logger.info("Creating a new mapreduce tarball at {0}".format(mapreduce_tarball_with_native_lib))
+
+ # tar up mapreduce, making sure to specify nothing for the arcname so that it does not include an absolute path
+ with closing(tarfile.open(mapreduce_tarball_with_native_lib, "w:gz")) as new_tarball:
+ new_tarball.add(mapreduce_temp_dir, arcname = os.path.sep)
+
+ # ensure that the tarball can be read and uploaded
+ sudo.chmod(mapreduce_tarball_with_native_lib, 0744)
+
+ # cleanup
+ sudo.rmtree(mapreduce_temp_dir)
+
+ return mapreduce_tarball_with_native_lib
+
+
# TODO, in the future, each stack can define its own mapping of tarballs
# inside the stack definition directory in some sort of xml file.
# PLEASE DO NOT put this in cluster-env since it becomes much harder to change,
@@ -163,7 +253,8 @@ TARBALL_MAP = {
"mapreduce": {
"dirs": ("{0}/{1}/hadoop/mapreduce.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN),
"/{0}/apps/{1}/mapreduce/mapreduce.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)),
- "service": "MAPREDUCE2"
+ "service": "MAPREDUCE2",
+ "prepare_function": _prepare_mapreduce_tarball
},
"spark": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/58c7f784/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py
index dfa6501..35647e4 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py
@@ -23,8 +23,8 @@ import os
# Local Imports
from resource_management.core.resources.system import Directory, File
+from resource_management.libraries.functions import lzo_utils
from resource_management.libraries.resources.xml_config import XmlConfig
-from resource_management.libraries.functions.format import format
from resource_management.core.source import InlineTemplate
from ambari_commons import OSConst
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
@@ -37,6 +37,9 @@ def tez(config_dir):
"""
import params
+ # ensure that matching LZO libraries are installed for Tez
+ lzo_utils.install_lzo_if_needed()
+
Directory(params.tez_etc_dir, mode=0755)
Directory(config_dir,
http://git-wip-us.apache.org/repos/asf/ambari/blob/58c7f784/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
index 3438c45..398c9d7 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
@@ -438,7 +438,7 @@
</property>
<property>
<name>mapreduce.admin.user.env</name>
- <value>LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:{{hadoop_lib_home}}/native/Linux-{{architecture}}-64</value>
+ <value>LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:./mr-framework/hadoop/lib/native/Linux-{{architecture}}-64:{{hadoop_lib_home}}/native/Linux-{{architecture}}-64</value>
<description>
Additional execution environment entries for map and reduce task processes.
This is not an additive property. You must preserve the original value if
http://git-wip-us.apache.org/repos/asf/ambari/blob/58c7f784/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml
index 4ffb7a4..5513ab1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml
@@ -78,7 +78,7 @@
</property>
<property>
<name>tez.am.launch.env</name>
- <value>LD_LIBRARY_PATH=./tezlib/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value>
+ <value>LD_LIBRARY_PATH=./tezlib/lib/native:./tezlib/lib/native/Linux-{{architecture}}-64:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value>
<description>
Additional execution environment entries for tez. This is not an additive property. You must preserve the original value if
you want to have access to native libraries.
@@ -124,7 +124,7 @@
</property>
<property>
<name>tez.task.launch.env</name>
- <value>LD_LIBRARY_PATH=./tezlib/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value>
+ <value>LD_LIBRARY_PATH=./tezlib/lib/native:./tezlib/lib/native/Linux-{{architecture}}-64:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value>
<description>
Additional execution environment entries for tez. This is not an additive property. You must preserve the original value if
you want to have access to native libraries.
http://git-wip-us.apache.org/repos/asf/ambari/blob/58c7f784/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml
index 084e912..099e388 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml
@@ -20,7 +20,7 @@
<configuration xmlns:xi="http://www.w3.org/2001/XInclude" supports_final="true">
<property>
<name>mapreduce.admin.user.env</name>
- <value>LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value>
+ <value>LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:./mr-framework/hadoop/lib/native/Linux-{{architecture}}-64:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value>
<description>
Additional execution environment entries for map and reduce task processes.
This is not an additive property. You must preserve the original value if
http://git-wip-us.apache.org/repos/asf/ambari/blob/58c7f784/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
index 4538072..5c672ba 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
@@ -269,8 +269,8 @@
<changes>
<definition xsi:type="configure" id="hdp_2_6_tez_tarball_ld_library">
<type>tez-site</type>
- <set key="tez.am.launch.env" value="LD_LIBRARY_PATH=./tezlib/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64"/>
- <set key="tez.task.launch.env" value="LD_LIBRARY_PATH=./tezlib/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64"/>
+ <set key="tez.am.launch.env" value="LD_LIBRARY_PATH=./tezlib/lib/native:./tezlib/lib/native/Linux-{{architecture}}-64:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64"/>
+ <set key="tez.task.launch.env" value="LD_LIBRARY_PATH=./tezlib/lib/native:./tezlib/lib/native/Linux-{{architecture}}-64:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64"/>
</definition>
</changes>
</component>
@@ -281,7 +281,7 @@
<changes>
<definition xsi:type="configure" id="hdp_2_6_mapreduce_tarball_ld_library">
<type>mapred-site</type>
- <set key="mapreduce.admin.user.env" value="LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:{{hadoop_lib_home}}/native/Linux-{{architecture}}-64"/>
+ <set key="mapreduce.admin.user.env" value="LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:./mr-framework/hadoop/lib/native/Linux-{{architecture}}-64:{{hadoop_lib_home}}/native/Linux-{{architecture}}-64"/>
</definition>
</changes>
</component>
[13/24] ambari git commit: Merge branch 'branch-2.6' into
branch-feature-AMBARI-22457
Posted by jo...@apache.org.
Merge branch 'branch-2.6' into branch-feature-AMBARI-22457
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c7457a82
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c7457a82
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c7457a82
Branch: refs/heads/branch-2.6
Commit: c7457a820046392dfaf5ca06202143d06c4f143b
Parents: 58c7f78 0615fa0
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Wed Nov 22 10:44:42 2017 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Wed Nov 22 10:44:42 2017 +0200
----------------------------------------------------------------------
.../libraries/functions/copy_tarball.py | 60 ++++----
ambari-metrics/ambari-metrics-assembly/pom.xml | 1 -
ambari-server/pom.xml | 12 ++
.../ambari/server/actionmanager/Stage.java | 3 +-
.../internal/UpgradeResourceProvider.java | 2 +
.../serveraction/ServerActionExecutor.java | 147 +++++++++++++++++--
.../ambari/server/stack/ServiceDirectory.java | 29 ++++
.../ambari/server/stack/ServiceModule.java | 8 +
.../apache/ambari/server/state/ServiceInfo.java | 14 ++
.../stack/upgrade/ServerSideActionTask.java | 15 ++
.../state/stack/upgrade/TaskParameter.java | 41 ++++++
.../stacks/HDP/2.6/services/stack_advisor.py | 11 +-
.../src/main/resources/upgrade-pack.xsd | 9 ++
.../ambari/server/stack/ServiceModuleTest.java | 30 ++++
.../server/stack/StackManagerExtensionTest.java | 6 +
.../main/service/reassign/step3_controller.js | 3 +
.../main/service/reassign/step4_controller.js | 12 +-
.../service/reassign/step4_controller_test.js | 9 +-
18 files changed, 355 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c7457a82/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
----------------------------------------------------------------------
[02/24] ambari git commit: AMBARI-22453. ambari-server setup should
surface GPL software agreement (aonishuk)
Posted by jo...@apache.org.
AMBARI-22453. ambari-server setup should surface GPL software agreement (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/46237fe3
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/46237fe3
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/46237fe3
Branch: refs/heads/branch-2.6
Commit: 46237fe37b518a4eb0f3094d8c91867877b68094
Parents: 81439af
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Nov 16 18:00:09 2017 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Nov 16 18:00:09 2017 +0200
----------------------------------------------------------------------
.../ambari/server/configuration/Configuration.java | 11 +++++------
ambari-server/src/main/python/ambari-server.py | 4 ++--
.../controller/AmbariManagementControllerImplTest.java | 3 ++-
.../internal/ClientConfigResourceProviderTest.java | 2 ++
4 files changed, 11 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/46237fe3/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index 085f420..2590a83 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -726,13 +726,12 @@ public class Configuration {
"server.version.file", null);
/**
- * The full path to the file which contains the Ambari Server version.
+ * Whether user accepted GPL license
*/
@Markdown(
- description = "Whether user accepted GPL license.",
- examples = { "true", "false" })
- public static final ConfigurationProperty<String> GPL_LICENSE_ACCEPTED = new ConfigurationProperty<>(
- "gpl.license.accepted", null);
+ description = "Whether user accepted GPL license.")
+ public static final ConfigurationProperty<Boolean> GPL_LICENSE_ACCEPTED = new ConfigurationProperty<>(
+ "gpl.license.accepted", false);
/**
* The location of the JDK on the Ambari Agent hosts.
@@ -5329,7 +5328,7 @@ public class Configuration {
}
public Boolean getGplLicenseAccepted(){
- return getProperty(GPL_LICENSE_ACCEPTED).toLowerCase().equals("true");
+ return Boolean.valueOf(getProperty(GPL_LICENSE_ACCEPTED));
}
public String getAgentStackRetryOnInstallCount(){
http://git-wip-us.apache.org/repos/asf/ambari/blob/46237fe3/ambari-server/src/main/python/ambari-server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py
index 6d242a3..3a67532 100755
--- a/ambari-server/src/main/python/ambari-server.py
+++ b/ambari-server/src/main/python/ambari-server.py
@@ -409,7 +409,7 @@ def init_parser_options(parser):
parser.add_option('--skip-database-check', action="store_true", default=False, help="Skip database consistency check", dest="skip_database_check")
parser.add_option('--skip-view-extraction', action="store_true", default=False, help="Skip extraction of system views", dest="skip_view_extraction")
parser.add_option('--auto-fix-database', action="store_true", default=False, help="Automatically fix database consistency issues", dest="fix_database_consistency")
- parser.add_option('--accept-gpl', action="store_true", default=False, help="Automatically accepts GPL license", dest="accept_gpl")
+ parser.add_option('--enable-lzo-under-gpl-license', action="store_true", default=False, help="Automatically accepts GPL license", dest="accept_gpl")
add_parser_options('--mpack',
default=None,
help="Specify the path for management pack to be installed/upgraded",
@@ -513,7 +513,7 @@ def init_parser_options(parser):
parser.add_option('--skip-database-check', action="store_true", default=False, help="Skip database consistency check", dest="skip_database_check")
parser.add_option('--skip-view-extraction', action="store_true", default=False, help="Skip extraction of system views", dest="skip_view_extraction")
parser.add_option('--auto-fix-database', action="store_true", default=False, help="Automatically fix database consistency issues", dest="fix_database_consistency")
- parser.add_option('--accept-gpl', action="store_true", default=False, help="Automatically accepts GPL license", dest="accept_gpl")
+ parser.add_option('--enable-lzo-under-gpl-license', action="store_true", default=False, help="Automatically accepts GPL license", dest="accept_gpl")
parser.add_option('--force-version', action="store_true", default=False, help="Force version to current", dest="force_repo_version")
parser.add_option('--version', dest="stack_versions", default=None, action="append", type="string",
help="Specify stack version that needs to be enabled. All other stacks versions will be disabled")
http://git-wip-us.apache.org/repos/asf/ambari/blob/46237fe3/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
index c857008..00b87a6 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
@@ -2082,6 +2082,7 @@ public class AmbariManagementControllerImplTest {
expect(configuration.getServerDBName()).andReturn(SERVER_DB_NAME);
expect(configuration.getJavaVersion()).andReturn(8);
expect(configuration.areHostsSysPrepped()).andReturn("true");
+ expect(configuration.getGplLicenseAccepted()).andReturn(false);
expect(configuration.getDatabaseConnectorNames()).andReturn(new HashMap<String, String>()).anyTimes();
expect(configuration.getPreviousDatabaseConnectorNames()).andReturn(new HashMap<String, String>()).anyTimes();
expect(repositoryVersionEntity.getVersion()).andReturn("1234").anyTimes();
@@ -2129,7 +2130,7 @@ public class AmbariManagementControllerImplTest {
Map<String, String> defaultHostParams = helper.createDefaultHostParams(cluster, repositoryVersionEntity.getStackId());
- assertEquals(15, defaultHostParams.size());
+ assertEquals(16, defaultHostParams.size());
assertEquals(MYSQL_JAR, defaultHostParams.get(DB_DRIVER_FILENAME));
assertEquals(SOME_STACK_NAME, defaultHostParams.get(STACK_NAME));
assertEquals(SOME_STACK_VERSION, defaultHostParams.get(STACK_VERSION));
http://git-wip-us.apache.org/repos/asf/ambari/blob/46237fe3/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
index 9de5795..8c89daa 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
@@ -276,6 +276,7 @@ public class ClientConfigResourceProviderTest {
expect(configuration.areHostsSysPrepped()).andReturn("false");
expect(configuration.isAgentStackRetryOnInstallEnabled()).andReturn("false");
expect(configuration.getAgentStackRetryOnInstallCount()).andReturn("5");
+ expect(configuration.getGplLicenseAccepted()).andReturn(Configuration.GPL_LICENSE_ACCEPTED.getDefaultValue());
expect(configuration.getExternalScriptThreadPoolSize()).andReturn(Configuration.THREAD_POOL_SIZE_FOR_EXTERNAL_SCRIPT.getDefaultValue());
expect(configuration.getExternalScriptTimeout()).andReturn(Configuration.EXTERNAL_SCRIPT_TIMEOUT.getDefaultValue());
Map<String,String> props = new HashMap<String, String>();
@@ -528,6 +529,7 @@ public class ClientConfigResourceProviderTest {
expect(configuration.areHostsSysPrepped()).andReturn("false");
expect(configuration.isAgentStackRetryOnInstallEnabled()).andReturn("false");
expect(configuration.getAgentStackRetryOnInstallCount()).andReturn("5");
+ expect(configuration.getGplLicenseAccepted()).andReturn(Configuration.GPL_LICENSE_ACCEPTED.getDefaultValue());
expect(configuration.getExternalScriptThreadPoolSize()).andReturn(Configuration.THREAD_POOL_SIZE_FOR_EXTERNAL_SCRIPT.getDefaultValue());
expect(configuration.getExternalScriptTimeout()).andReturn(Configuration.EXTERNAL_SCRIPT_TIMEOUT.getDefaultValue());
[19/24] ambari git commit: AMBARI-22495. Installer - Select Version
page should show HDP-GPL repo if necessary (vsubramanian)
Posted by jo...@apache.org.
AMBARI-22495. Installer - Select Version page should show HDP-GPL repo if necessary (vsubramanian)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/85418298
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/85418298
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/85418298
Branch: refs/heads/branch-2.6
Commit: 8541829876ae3edcfbe96587270ecedad86f36e8
Parents: 059fadf
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
Authored: Wed Nov 22 17:55:46 2017 -0800
Committer: Vivek Ratnavel Subramanian <vi...@gmail.com>
Committed: Wed Nov 22 17:55:46 2017 -0800
----------------------------------------------------------------------
ambari-web/app/controllers/installer.js | 53 +++++++++++---------
.../app/controllers/wizard/step8_controller.js | 12 +++--
.../app/mappers/repository_version_mapper.js | 3 +-
ambari-web/app/mappers/stack_mapper.js | 3 +-
ambari-web/app/models/repository.js | 17 +++++++
.../app/models/stack_version/repository.js | 3 +-
ambari-web/app/templates/wizard/step1.hbs | 2 +
ambari-web/test/controllers/installer_test.js | 14 ++++--
8 files changed, 69 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/85418298/ambari-web/app/controllers/installer.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/installer.js b/ambari-web/app/controllers/installer.js
index a45ac57..afce91a 100644
--- a/ambari-web/app/controllers/installer.js
+++ b/ambari-web/app/controllers/installer.js
@@ -807,6 +807,7 @@ App.InstallerController = App.WizardController.extend(App.UserPref, {
"repo_id": repository.get('repoId'),
"repo_name": repository.get('repoName'),
"components": repository.get('components'),
+ "tags": repository.get('tags'),
"distribution": repository.get('distribution')
}
});
@@ -827,7 +828,7 @@ App.InstallerController = App.WizardController.extend(App.UserPref, {
var dfd = $.Deferred();
if (selectedStack && selectedStack.get('operatingSystems')) {
this.set('validationCnt', selectedStack.get('operatingSystems').filterProperty('isSelected').filterProperty('isEmpty', false).map(function (os) {
- return os.get('repositories.length');
+ return os.get('repositories').filterProperty('showRepo', true).length;
}).reduce(Em.sum, 0));
var verifyBaseUrl = !wizardStep1Controller.get('skipValidationChecked') && !wizardStep1Controller.get('selectedStack.useRedhatSatellite');
if (!verifyBaseUrl) {
@@ -836,32 +837,34 @@ App.InstallerController = App.WizardController.extend(App.UserPref, {
selectedStack.get('operatingSystems').forEach(function (os) {
if (os.get('isSelected') && !os.get('isEmpty')) {
os.get('repositories').forEach(function (repo) {
- repo.setProperties({
- errorTitle: '',
- errorContent: '',
- validation: App.Repository.validation.INPROGRESS
- });
- this.set('content.isCheckInProgress', true);
- App.ajax.send({
- name: 'wizard.advanced_repositories.valid_url',
- sender: this,
- data: {
- stackName: stackName,
- stackVersion: stackVersion,
- repoId: repo.get('repoId'),
- osType: os.get('osType'),
- osId: os.get('id'),
- dfd: dfd,
+ if (repo.get('showRepo')) {
+ repo.setProperties({
+ errorTitle: '',
+ errorContent: '',
+ validation: App.Repository.validation.INPROGRESS
+ });
+ this.set('content.isCheckInProgress', true);
+ App.ajax.send({
+ name: 'wizard.advanced_repositories.valid_url',
+ sender: this,
data: {
- 'Repositories': {
- 'base_url': repo.get('baseUrl'),
- "verify_base_url": verifyBaseUrl
+ stackName: stackName,
+ stackVersion: stackVersion,
+ repoId: repo.get('repoId'),
+ osType: os.get('osType'),
+ osId: os.get('id'),
+ dfd: dfd,
+ data: {
+ 'Repositories': {
+ 'base_url': repo.get('baseUrl'),
+ "verify_base_url": verifyBaseUrl
+ }
}
- }
- },
- success: 'checkRepoURLSuccessCallback',
- error: 'checkRepoURLErrorCallback'
- });
+ },
+ success: 'checkRepoURLSuccessCallback',
+ error: 'checkRepoURLErrorCallback'
+ });
+ }
}, this);
} else if (os.get('isSelected') && os.get('isEmpty')) {
os.set('isSelected', false);
http://git-wip-us.apache.org/repos/asf/ambari/blob/85418298/ambari-web/app/controllers/wizard/step8_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/wizard/step8_controller.js b/ambari-web/app/controllers/wizard/step8_controller.js
index 4bedde0..be86b21 100644
--- a/ambari-web/app/controllers/wizard/step8_controller.js
+++ b/ambari-web/app/controllers/wizard/step8_controller.js
@@ -307,11 +307,13 @@ App.WizardStep8Controller = Em.Controller.extend(App.AddSecurityConfigs, App.wiz
selectedStack.get('operatingSystems').forEach(function (os) {
if (os.get('isSelected')) {
os.get('repositories').forEach(function(repo) {
- allRepos.push(Em.Object.create({
- base_url: repo.get('baseUrl'),
- os_type: repo.get('osType'),
- repo_id: repo.get('repoId')
- }));
+ if (repo.get('showRepo')) {
+ allRepos.push(Em.Object.create({
+ base_url: repo.get('baseUrl'),
+ os_type: repo.get('osType'),
+ repo_id: repo.get('repoId')
+ }));
+ }
}, this);
}
}, this);
http://git-wip-us.apache.org/repos/asf/ambari/blob/85418298/ambari-web/app/mappers/repository_version_mapper.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mappers/repository_version_mapper.js b/ambari-web/app/mappers/repository_version_mapper.js
index 90086f2..1cd0c12 100644
--- a/ambari-web/app/mappers/repository_version_mapper.js
+++ b/ambari-web/app/mappers/repository_version_mapper.js
@@ -73,7 +73,8 @@ App.repoVersionMapper = App.QuickDataMapper.create({
repo_id : 'Repositories.repo_id',
repo_name : 'Repositories.repo_name',
stack_name : 'Repositories.stack_name',
- stack_version : 'Repositories.stack_version'
+ stack_version : 'Repositories.stack_version',
+ tags: 'Repositories.tags'
},
map: function (json, loadAll, isCurrentStackOnly) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/85418298/ambari-web/app/mappers/stack_mapper.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mappers/stack_mapper.js b/ambari-web/app/mappers/stack_mapper.js
index 822d35b..64a8e8c 100644
--- a/ambari-web/app/mappers/stack_mapper.js
+++ b/ambari-web/app/mappers/stack_mapper.js
@@ -86,7 +86,8 @@ App.stackMapper = App.QuickDataMapper.create({
stack_version: 'stack_version',
operating_system_id: 'os_id',
components: 'components',
- distribution: 'distribution'
+ distribution: 'distribution',
+ tags: 'tags'
},
map: function(json) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/85418298/ambari-web/app/models/repository.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/models/repository.js b/ambari-web/app/models/repository.js
index 91f0083..42393c2 100644
--- a/ambari-web/app/models/repository.js
+++ b/ambari-web/app/models/repository.js
@@ -34,6 +34,7 @@ App.Repository = DS.Model.extend({
operatingSystem: DS.belongsTo('App.OperatingSystem'),
components: DS.attr('string'),
distribution: DS.attr('string'),
+ tags: DS.attr('array'),
validation: DS.attr('string', {defaultValue: ''}),
errorContent: DS.attr('string', {defaultValue: ''}),
@@ -60,6 +61,22 @@ App.Repository = DS.Model.extend({
return this.get('repoName').contains('UTILS');
}.property('repoName'),
+ /**
+ * @type {boolean}
+ */
+ isGPL: function () {
+ return this.get('tags').contains('GPL');
+ }.property('tags'),
+
+ /**
+ * Determines whether a repo needs to be displayed in the UI or not
+ * @type {boolean}
+ */
+ showRepo: function () {
+ const isGPLAccepted = App.router.get('clusterController.ambariProperties')['gpl.license.accepted'] === 'true';
+ return isGPLAccepted || !this.get('isGPL');
+ }.property('isGPL'),
+
undo: Em.computed.notEqualProperties('baseUrl', 'baseUrlInit'),
notEmpty: Em.computed.notEqual('baseUrl', ''),
http://git-wip-us.apache.org/repos/asf/ambari/blob/85418298/ambari-web/app/models/stack_version/repository.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/models/stack_version/repository.js b/ambari-web/app/models/stack_version/repository.js
index 8bd4323..6c93416 100644
--- a/ambari-web/app/models/stack_version/repository.js
+++ b/ambari-web/app/models/stack_version/repository.js
@@ -29,7 +29,8 @@ App.Repository = DS.Model.extend({
repoName : DS.attr('string'),
stackName : DS.attr('string'),
stackVersion : DS.attr('string'),
- operatingSystem: DS.belongsTo('App.OS')
+ operatingSystem: DS.belongsTo('App.OS'),
+ tags: DS.attr('array')
});
App.Repository.FIXTURES = [];
http://git-wip-us.apache.org/repos/asf/ambari/blob/85418298/ambari-web/app/templates/wizard/step1.hbs
----------------------------------------------------------------------
diff --git a/ambari-web/app/templates/wizard/step1.hbs b/ambari-web/app/templates/wizard/step1.hbs
index 485bd5d..2cf5646 100644
--- a/ambari-web/app/templates/wizard/step1.hbs
+++ b/ambari-web/app/templates/wizard/step1.hbs
@@ -148,6 +148,7 @@
</div>
<div class="span9 repo-name-url">
{{#each repository in operatingSystem.repositories}}
+ {{#if repository.showRepo}}
<div {{bindAttr class=":clearfix :repo-name-url-inner repository.repoName"}}>
<div class="span3">
<label class="repo-name-label control-label">{{repository.repoId}}</label>
@@ -176,6 +177,7 @@
{{/if}}
</div>
</div>
+ {{/if}}
{{/each}}
</div>
<div {{bindAttr class=":span1 :remove-icon controller.selectedStack.useRedhatSatellite:disabled"}} {{action "removeOS" operatingSystem target="controller"}}>
http://git-wip-us.apache.org/repos/asf/ambari/blob/85418298/ambari-web/test/controllers/installer_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/controllers/installer_test.js b/ambari-web/test/controllers/installer_test.js
index 3e1c622..9a17486 100644
--- a/ambari-web/test/controllers/installer_test.js
+++ b/ambari-web/test/controllers/installer_test.js
@@ -92,7 +92,8 @@ describe('App.InstallerController', function () {
isEmpty: false,
errorTitle: '1',
errorContent: '1',
- validation: ''
+ validation: '',
+ showRepo: true
})
])
})
@@ -131,7 +132,8 @@ describe('App.InstallerController', function () {
"isEmpty": false,
"errorTitle": "",
"errorContent": "",
- "validation": "icon-repeat"
+ "validation": "icon-repeat",
+ "showRepo": true
}
]
}
@@ -172,7 +174,8 @@ describe('App.InstallerController', function () {
isEmpty: false,
errorTitle: '1',
errorContent: '1',
- validation: ''
+ validation: '',
+ showRepo: true
})
])
})
@@ -189,7 +192,7 @@ describe('App.InstallerController', function () {
}
}
};
- it ('Should check stacks for sucess', function() {
+ it ('Should check stacks for success', function() {
installerController.set('content.stacks', stacks);
installerController.checkRepoURLSuccessCallback(null,null,data);
@@ -220,7 +223,8 @@ describe('App.InstallerController', function () {
"isEmpty": false,
"errorTitle": "1",
"errorContent": "1",
- "validation": "icon-ok"
+ "validation": "icon-ok",
+ "showRepo": true
}
]
}
[20/24] ambari git commit: AMBARI-22467. YARN, MapReduce2, Hive,
and Oozie Should Conditionally Install LZO (aonishuk)
Posted by jo...@apache.org.
AMBARI-22467. YARN, MapReduce2, Hive, and Oozie Should Conditionally Install LZO (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bc8b3916
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bc8b3916
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bc8b3916
Branch: refs/heads/branch-2.6
Commit: bc8b3916b8899b0629768ad9f308ac013edcf01c
Parents: 8541829
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Nov 23 10:26:46 2017 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Nov 23 10:36:29 2017 +0200
----------------------------------------------------------------------
.../common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/bc8b3916/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
index 2316f79..9297337 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
@@ -192,7 +192,6 @@ def oozie(is_server=False, upgrade_type=None):
if params.lzo_enabled:
install_lzo_if_needed()
Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
- not_if = no_op_test,
)
if is_server:
[09/24] ambari git commit: AMBARI-22480. Validate blueprint does not
allow lzo enable without setup with license agreement. (Myroslav Papirkovskyy
via swagle)
Posted by jo...@apache.org.
AMBARI-22480. Validate blueprint does not allow lzo enable without setup with license agreement. (Myroslav Papirkovskyy via swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1fce5650
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1fce5650
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1fce5650
Branch: refs/heads/branch-2.6
Commit: 1fce5650ba92f68a49e8750369aea8e9d3dcf6bf
Parents: 096fedb
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Mon Nov 20 10:56:04 2017 -0800
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Mon Nov 20 10:56:04 2017 -0800
----------------------------------------------------------------------
.../internal/BlueprintResourceProvider.java | 36 +++++-----
.../ambari/server/topology/Blueprint.java | 40 ++++++------
.../ambari/server/topology/BlueprintImpl.java | 17 ++---
.../server/topology/BlueprintValidator.java | 4 +-
.../server/topology/BlueprintValidatorImpl.java | 21 +++++-
.../server/topology/BlueprintImplTest.java | 69 ++++++++++++++++++--
.../topology/BlueprintValidatorImplTest.java | 18 +++--
7 files changed, 143 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/1fce5650/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
index 1c127c0..b313c8b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
@@ -18,11 +18,17 @@
package org.apache.ambari.server.controller.internal;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.gson.Gson;
-import org.apache.ambari.server.orm.entities.BlueprintSettingEntity;
-import org.apache.ambari.server.utils.SecretReference;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.DuplicateResourceException;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
@@ -41,28 +47,24 @@ import org.apache.ambari.server.orm.dao.BlueprintDAO;
import org.apache.ambari.server.orm.entities.BlueprintConfigEntity;
import org.apache.ambari.server.orm.entities.BlueprintConfiguration;
import org.apache.ambari.server.orm.entities.BlueprintEntity;
+import org.apache.ambari.server.orm.entities.BlueprintSettingEntity;
import org.apache.ambari.server.orm.entities.HostGroupComponentEntity;
import org.apache.ambari.server.orm.entities.HostGroupEntity;
import org.apache.ambari.server.orm.entities.StackEntity;
import org.apache.ambari.server.stack.NoSuchStackException;
-import org.apache.ambari.server.state.StackInfo;
import org.apache.ambari.server.state.SecurityType;
+import org.apache.ambari.server.state.StackInfo;
import org.apache.ambari.server.topology.Blueprint;
import org.apache.ambari.server.topology.BlueprintFactory;
+import org.apache.ambari.server.topology.GPLLicenseNotAcceptedException;
import org.apache.ambari.server.topology.InvalidTopologyException;
import org.apache.ambari.server.topology.SecurityConfiguration;
import org.apache.ambari.server.topology.SecurityConfigurationFactory;
+import org.apache.ambari.server.utils.SecretReference;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.gson.Gson;
/**
@@ -514,7 +516,7 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
try {
blueprint.validateRequiredProperties();
- } catch (InvalidTopologyException e) {
+ } catch (InvalidTopologyException | GPLLicenseNotAcceptedException e) {
throw new IllegalArgumentException("Blueprint configuration validation failed: " + e.getMessage(), e);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/1fce5650/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java
index c9dc728..8cacd22 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java
@@ -18,12 +18,12 @@
package org.apache.ambari.server.topology;
-import org.apache.ambari.server.controller.internal.Stack;
-import org.apache.ambari.server.orm.entities.BlueprintEntity;
-
import java.util.Collection;
import java.util.Map;
+import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.orm.entities.BlueprintEntity;
+
/**
* Blueprint representation.
*/
@@ -34,13 +34,13 @@ public interface Blueprint {
*
* @return blueprint name
*/
- public String getName();
+ String getName();
/**
* Get the hot groups contained in the blueprint.
* @return map of host group name to host group
*/
- public Map<String, HostGroup> getHostGroups();
+ Map<String, HostGroup> getHostGroups();
/**
* Get a hostgroup specified by name.
@@ -49,7 +49,7 @@ public interface Blueprint {
*
* @return the host group with the given name or null
*/
- public HostGroup getHostGroup(String name);
+ HostGroup getHostGroup(String name);
/**
* Get the Blueprint cluster scoped configuration.
@@ -59,7 +59,7 @@ public interface Blueprint {
*
* @return blueprint cluster scoped configuration
*/
- public Configuration getConfiguration();
+ Configuration getConfiguration();
/**
* Get the Blueprint cluster scoped setting.
@@ -68,14 +68,14 @@ public interface Blueprint {
*
* @return blueprint cluster scoped setting
*/
- public Setting getSetting();
+ Setting getSetting();
/**
* Get all of the services represented in the blueprint.
*
* @return collection of all represented service names
*/
- public Collection<String> getServices();
+ Collection<String> getServices();
/**
* Get the components that are included in the blueprint for the specified service.
@@ -84,7 +84,7 @@ public interface Blueprint {
*
* @return collection of component names for the service. Will not return null.
*/
- public Collection<String> getComponents(String service);
+ Collection<String> getComponents(String service);
/**
* Get whether a component is enabled for auto start.
@@ -94,7 +94,7 @@ public interface Blueprint {
*
* @return null if value is not specified; true or false if specified.
*/
- public String getRecoveryEnabled(String serviceName, String componentName);
+ String getRecoveryEnabled(String serviceName, String componentName);
/**
* Get whether a service is enabled for credential store use.
@@ -103,20 +103,20 @@ public interface Blueprint {
*
* @return null if value is not specified; true or false if specified.
*/
- public String getCredentialStoreEnabled(String serviceName);
+ String getCredentialStoreEnabled(String serviceName);
/**
* Check if auto skip failure is enabled.
* @return true if enabled, otherwise false.
*/
- public boolean shouldSkipFailure();
+ boolean shouldSkipFailure();
/**
* Get the stack associated with the blueprint.
*
* @return associated stack
*/
- public Stack getStack();
+ Stack getStack();
/**
* Get the host groups which contain components for the specified service.
@@ -126,7 +126,7 @@ public interface Blueprint {
* @return collection of host groups containing components for the specified service;
* will not return null
*/
- public Collection<HostGroup> getHostGroupsForService(String service);
+ Collection<HostGroup> getHostGroupsForService(String service);
/**
* Get the host groups which contain the give component.
@@ -135,23 +135,23 @@ public interface Blueprint {
*
* @return collection of host groups containing the specified component; will not return null
*/
- public Collection<HostGroup> getHostGroupsForComponent(String component);
+ Collection<HostGroup> getHostGroupsForComponent(String component);
- public SecurityConfiguration getSecurity();
+ SecurityConfiguration getSecurity();
/**
* Validate the blueprint topology.
*
* @throws InvalidTopologyException if the topology is invalid
*/
- public void validateTopology() throws InvalidTopologyException;
+ void validateTopology() throws InvalidTopologyException;
/**
* Validate that the blueprint contains all of the required properties.
*
* @throws InvalidTopologyException if the blueprint doesn't contain all required properties
*/
- public void validateRequiredProperties() throws InvalidTopologyException;
+ void validateRequiredProperties() throws InvalidTopologyException, GPLLicenseNotAcceptedException;
/**
*
@@ -166,5 +166,5 @@ public interface Blueprint {
*
* @return entity representation of the blueprint
*/
- public BlueprintEntity toEntity();
+ BlueprintEntity toEntity();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/1fce5650/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java
index 1d917e0..06fd9f2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java
@@ -19,7 +19,13 @@
package org.apache.ambari.server.topology;
-import com.google.gson.Gson;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.StackAccessException;
import org.apache.ambari.server.controller.AmbariServer;
@@ -36,12 +42,7 @@ import org.apache.ambari.server.stack.NoSuchStackException;
import org.apache.ambari.server.state.ConfigHelper;
import org.apache.commons.lang.StringUtils;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
+import com.google.gson.Gson;
/**
* Blueprint implementation.
@@ -351,7 +352,7 @@ public class BlueprintImpl implements Blueprint {
* @throws InvalidTopologyException if the blueprint configuration is invalid
*/
@Override
- public void validateRequiredProperties() throws InvalidTopologyException {
+ public void validateRequiredProperties() throws InvalidTopologyException, GPLLicenseNotAcceptedException {
validator.validateRequiredProperties();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/1fce5650/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java
index 206d161..c201a7f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java
@@ -29,7 +29,7 @@ public interface BlueprintValidator {
*
* @throws InvalidTopologyException if the topology is invalid
*/
- public void validateTopology() throws InvalidTopologyException;
+ void validateTopology() throws InvalidTopologyException;
/**
* Validate that required properties are provided.
@@ -37,5 +37,5 @@ public interface BlueprintValidator {
*
* @throws InvalidTopologyException if required properties are not set in blueprint
*/
- public void validateRequiredProperties() throws InvalidTopologyException;
+ void validateRequiredProperties() throws InvalidTopologyException, GPLLicenseNotAcceptedException;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/1fce5650/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
index 1aaa5ce..7b7f7d7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import org.apache.ambari.server.StaticallyInject;
import org.apache.ambari.server.controller.internal.Stack;
import org.apache.ambari.server.state.AutoDeployInfo;
import org.apache.ambari.server.state.DependencyInfo;
@@ -31,15 +32,24 @@ import org.apache.ambari.server.utils.VersionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.inject.Inject;
+
/**
* Default blueprint validator.
*/
+@StaticallyInject
public class BlueprintValidatorImpl implements BlueprintValidator {
private static final Logger LOGGER = LoggerFactory.getLogger(BlueprintValidatorImpl.class);
private final Blueprint blueprint;
private final Stack stack;
+ public static final String LZO_CODEC_CLASS_PROPERTY_NAME = "io.compression.codec.lzo.class";
+ public static final String LZO_CODEC_CLASS = "com.hadoop.compression.lzo.LzoCodec";
+
+ @Inject
+ private static org.apache.ambari.server.configuration.Configuration configuration;
+
public BlueprintValidatorImpl(Blueprint blueprint) {
this.blueprint = blueprint;
this.stack = blueprint.getStack();
@@ -80,13 +90,17 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
}
@Override
- public void validateRequiredProperties() throws InvalidTopologyException {
+ public void validateRequiredProperties() throws InvalidTopologyException, GPLLicenseNotAcceptedException {
// we don't want to include default stack properties so we can't just use hostGroup full properties
Map<String, Map<String, String>> clusterConfigurations = blueprint.getConfiguration().getProperties();
// we need to have real passwords, not references
if (clusterConfigurations != null) {
+
+ // need to reject blueprints that have LZO enabled if the Ambari Server hasn't been configured for it
+ boolean gplEnabled = configuration.getGplLicenseAccepted();
+
StringBuilder errorMessage = new StringBuilder();
boolean containsSecretReferences = false;
for (Map.Entry<String, Map<String, String>> configEntry : clusterConfigurations.entrySet()) {
@@ -96,6 +110,11 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
String propertyName = propertyEntry.getKey();
String propertyValue = propertyEntry.getValue();
if (propertyValue != null) {
+ if (!gplEnabled && configType.equals("core-site") && propertyName.equals(LZO_CODEC_CLASS_PROPERTY_NAME)
+ && propertyValue.contains(LZO_CODEC_CLASS)) {
+ throw new GPLLicenseNotAcceptedException("Your Ambari server has not been configured to download LZO GPL software. " +
+ "Please refer to documentation to configure Ambari before proceeding.");
+ }
if (SecretReference.isSecret(propertyValue)) {
errorMessage.append(" Config:" + configType + " Property:" + propertyName + "\n");
containsSecretReferences = true;
http://git-wip-us.apache.org/repos/asf/ambari/blob/1fce5650/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
index fdbc080..1a9fd1f 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
@@ -21,11 +21,13 @@ package org.apache.ambari.server.topology;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.mock;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import java.lang.reflect.Field;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -105,13 +107,14 @@ public class BlueprintImplTest {
@Test
public void testValidateConfigurations__basic_positive() throws Exception {
+ org.apache.ambari.server.configuration.Configuration serverConfig = setupConfigurationWithGPLLicense(true);
expect(group1.getCardinality()).andReturn("1").atLeastOnce();
expect(group1.getComponents()).andReturn(Arrays.asList(new Component("c1"), new Component("c2"))).atLeastOnce();
expect(group2.getCardinality()).andReturn("1").atLeastOnce();
expect(group2.getComponents()).andReturn(Arrays.asList(new Component("c1"), new Component("c3"))).atLeastOnce();
expect(group2.getConfiguration()).andReturn(EMPTY_CONFIGURATION).atLeastOnce();
- replay(stack, group1, group2);
+ replay(stack, group1, group2, serverConfig);
Map<String, String> category2Props = new HashMap<>();
properties.put("category2", category2Props);
@@ -122,7 +125,7 @@ public class BlueprintImplTest {
blueprint.validateRequiredProperties();
BlueprintEntity entity = blueprint.toEntity();
- verify(stack, group1, group2);
+ verify(stack, group1, group2, serverConfig);
assertTrue(entity.getSecurityType() == SecurityType.KERBEROS);
assertTrue(entity.getSecurityDescriptorReference().equals("testRef"));
}
@@ -133,6 +136,8 @@ public class BlueprintImplTest {
Map<String, String> group2Category2Props = new HashMap<>();
group2Props.put("category2", group2Category2Props);
group2Category2Props.put("prop2", "val");
+
+ org.apache.ambari.server.configuration.Configuration serverConfig = setupConfigurationWithGPLLicense(true);
// set config for group2 which contains a required property
Configuration group2Configuration = new Configuration(group2Props, EMPTY_ATTRIBUTES, configuration);
expect(group2.getConfiguration()).andReturn(group2Configuration).atLeastOnce();
@@ -142,29 +147,67 @@ public class BlueprintImplTest {
expect(group2.getCardinality()).andReturn("1").atLeastOnce();
expect(group2.getComponents()).andReturn(Arrays.asList(new Component("c1"), new Component("c3"))).atLeastOnce();
- replay(stack, group1, group2);
+ replay(stack, group1, group2, serverConfig);
Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null);
blueprint.validateRequiredProperties();
BlueprintEntity entity = blueprint.toEntity();
- verify(stack, group1, group2);
+ verify(stack, group1, group2, serverConfig);
assertTrue(entity.getSecurityType() == SecurityType.NONE);
assertTrue(entity.getSecurityDescriptorReference() == null);
}
@Test(expected = InvalidTopologyException.class)
- public void testValidateConfigurations__secretReference() throws InvalidTopologyException {
+ public void testValidateConfigurations__secretReference() throws InvalidTopologyException,
+ GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException {
Map<String, Map<String, String>> group2Props = new HashMap<>();
Map<String, String> group2Category2Props = new HashMap<>();
+
+ org.apache.ambari.server.configuration.Configuration serverConfig = setupConfigurationWithGPLLicense(true);
group2Props.put("category2", group2Category2Props);
group2Category2Props.put("prop2", "val");
hdfsProps.put("secret", "SECRET:hdfs-site:1:test");
- replay(stack, group1, group2);
+ replay(stack, group1, group2, serverConfig);
Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null);
blueprint.validateRequiredProperties();
- verify(stack, group1, group2);
+ verify(stack, group1, group2, serverConfig);
+ }
+
+ @Test(expected = GPLLicenseNotAcceptedException.class)
+ public void testValidateConfigurations__gplIsNotAllowed() throws InvalidTopologyException,
+ GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException {
+ Map<String, Map<String, String>> lzoProperties = new HashMap<>();
+ lzoProperties.put("core-site", new HashMap<String, String>(){{
+ put(BlueprintValidatorImpl.LZO_CODEC_CLASS_PROPERTY_NAME, BlueprintValidatorImpl.LZO_CODEC_CLASS);
+ }});
+ Configuration lzoUsageConfiguration = new Configuration(lzoProperties, EMPTY_ATTRIBUTES, EMPTY_CONFIGURATION);
+
+ org.apache.ambari.server.configuration.Configuration serverConfig = setupConfigurationWithGPLLicense(false);
+ replay(stack, group1, group2, serverConfig);
+
+ Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, lzoUsageConfiguration, null);
+ blueprint.validateRequiredProperties();
+ verify(stack, group1, group2, serverConfig);
+ }
+
+ @Test
+ public void testValidateConfigurations__gplISAllowed() throws InvalidTopologyException,
+ GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException {
+ Map<String, Map<String, String>> lzoProperties = new HashMap<>();
+ lzoProperties.put("core-site", new HashMap<String, String>(){{
+ put(BlueprintValidatorImpl.LZO_CODEC_CLASS_PROPERTY_NAME, BlueprintValidatorImpl.LZO_CODEC_CLASS);
+ }});
+ Configuration lzoUsageConfiguration = new Configuration(lzoProperties, EMPTY_ATTRIBUTES, EMPTY_CONFIGURATION);
+
+ org.apache.ambari.server.configuration.Configuration serverConfig = setupConfigurationWithGPLLicense(true);
+ expect(group2.getConfiguration()).andReturn(EMPTY_CONFIGURATION).atLeastOnce();
+ replay(stack, group1, group2, serverConfig);
+
+ Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, lzoUsageConfiguration, null);
+ blueprint.validateRequiredProperties();
+ verify(stack, group1, group2, serverConfig);
}
@Test
@@ -191,6 +234,18 @@ public class BlueprintImplTest {
verify(stack, setting);
}
+ public static org.apache.ambari.server.configuration.Configuration setupConfigurationWithGPLLicense(boolean isGPLAllowed)
+ throws NoSuchFieldException, IllegalAccessException {
+ org.apache.ambari.server.configuration.Configuration serverConfig =
+ mock(org.apache.ambari.server.configuration.Configuration.class);
+ expect(serverConfig.getGplLicenseAccepted()).andReturn(isGPLAllowed).atLeastOnce();
+
+ Field field = BlueprintValidatorImpl.class.getDeclaredField("configuration");
+ field.setAccessible(true);
+ field.set(null, serverConfig);
+ return serverConfig;
+ }
+
//todo: ensure coverage for these existing tests
// private void validateEntity(BlueprintEntity entity, boolean containsConfig) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/1fce5650/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
index ea6bee3..e7268e7 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
@@ -18,6 +18,11 @@
package org.apache.ambari.server.topology;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+import static org.easymock.EasyMock.verify;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -38,11 +43,6 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.reset;
-import static org.easymock.EasyMock.verify;
-
/**
* BlueprintValidatorImpl unit tests.
*/
@@ -216,6 +216,8 @@ public class BlueprintValidatorImplTest {
services.addAll(Arrays.asList("HIVE"));
+ org.apache.ambari.server.configuration.Configuration serverConfig =
+ BlueprintImplTest.setupConfigurationWithGPLLicense(true);
expect(group1.getConfiguration()).andReturn(new Configuration(new HashMap(), new HashMap())).anyTimes();
expect(stack.getComponents("HIVE")).andReturn(Collections.singleton("HIVE_METASTORE")).anyTimes();
@@ -224,7 +226,7 @@ public class BlueprintValidatorImplTest {
expect(blueprint.getHostGroupsForComponent("HIVE_METASTORE")).andReturn(Collections.singleton(group1)).anyTimes();
- replay(blueprint, stack, group1, group2, dependency1);
+ replay(blueprint, stack, group1, group2, dependency1, serverConfig);
BlueprintValidator validator = new BlueprintValidatorImpl(blueprint);
validator.validateRequiredProperties();
}
@@ -239,6 +241,8 @@ public class BlueprintValidatorImplTest {
services.addAll(Arrays.asList("OOZIE"));
+ org.apache.ambari.server.configuration.Configuration serverConfig =
+ BlueprintImplTest.setupConfigurationWithGPLLicense(true);
expect(group1.getConfiguration()).andReturn(new Configuration(new HashMap(), new HashMap())).anyTimes();
expect(stack.getComponents("OOZIE")).andReturn(Collections.singleton("OOZIE_SERVER")).anyTimes();
@@ -247,7 +251,7 @@ public class BlueprintValidatorImplTest {
expect(blueprint.getHostGroupsForComponent("OOZIE_SERVER")).andReturn(Collections.singleton(group1)).anyTimes();
- replay(blueprint, stack, group1, group2, dependency1);
+ replay(blueprint, stack, group1, group2, dependency1, serverConfig);
BlueprintValidator validator = new BlueprintValidatorImpl(blueprint);
validator.validateRequiredProperties();
}
[21/24] ambari git commit: Merge branch 'branch-2.6' into
branch-feature-AMBARI-22457
Posted by jo...@apache.org.
Merge branch 'branch-2.6' into branch-feature-AMBARI-22457
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/606d876b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/606d876b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/606d876b
Branch: refs/heads/branch-2.6
Commit: 606d876b62cafe64e51022d722128475643116da
Parents: bc8b391 c6825e9
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Mon Nov 27 14:58:58 2017 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Mon Nov 27 14:58:58 2017 +0200
----------------------------------------------------------------------
.../src/main/python/ambari_agent/Hardware.py | 130 +++---
.../src/main/python/ambari_agent/Heartbeat.py | 8 +-
.../src/main/python/ambari_agent/HostInfo.py | 83 ++--
.../test/python/ambari_agent/TestHardware.py | 56 ++-
.../python/ambari_agent/TestRegistration.py | 5 +-
.../core/providers/package/yumrpm.py | 4 +-
.../controller/AmbariManagementController.java | 6 +-
.../AmbariManagementControllerImpl.java | 104 ++---
.../controller/AmbariManagementHelper.java | 201 ++++++++++
.../internal/ExtensionLinkResourceProvider.java | 15 +
.../ambari/server/orm/dao/ExtensionLinkDAO.java | 36 +-
.../orm/entities/ExtensionLinkEntity.java | 1 +
.../ambari/server/stack/ExtensionHelper.java | 88 ++++-
.../ambari/server/stack/ExtensionModule.java | 2 +
.../ambari/server/stack/StackManager.java | 81 +++-
.../apache/ambari/server/stack/StackModule.java | 8 +-
.../ambari/server/state/ExtensionInfo.java | 26 +-
.../apache/ambari/server/state/StackInfo.java | 27 +-
.../state/stack/ExtensionMetainfoXml.java | 11 +
.../server/upgrade/UpgradeCatalog260.java | 171 +++++++-
.../HIVE/0.12.0.2.0/package/scripts/hive.py | 12 +-
.../package/scripts/hive_server_interactive.py | 20 +-
.../stacks/HDP/2.5/services/YARN/kerberos.json | 12 +-
.../stacks/HDP/2.6/services/YARN/kerberos.json | 24 +-
.../stack/StackManagerCommonServicesTest.java | 4 +-
.../server/stack/StackManagerExtensionTest.java | 116 ++++--
.../server/stack/StackManagerMiscTest.java | 13 +-
.../ambari/server/stack/StackManagerMock.java | 5 +-
.../ambari/server/stack/StackManagerTest.java | 13 +-
.../server/upgrade/UpgradeCatalog260Test.java | 129 +++++-
.../stacks/2.0.6/HIVE/test_hive_server.py | 2 +-
.../HIVE/running_withMOTDmsg_andTrailingMsg.txt | 46 +++
.../stacks/2.5/HIVE/test_hive_server_int.py | 21 +
.../resources/extensions/EXT/0.1/metainfo.xml | 2 +-
.../resources/extensions/EXT/0.2/metainfo.xml | 3 +-
.../resources/extensions/EXT/0.3/metainfo.xml | 32 ++
.../EXT/0.3/services/OOZIE2/metainfo.xml | 118 ++++++
.../services/OOZIE2/themes/broken_theme.json | 3 +
.../resources/extensions/EXT/0.4/metainfo.xml | 32 ++
.../EXT/0.4/services/OOZIE2/metainfo.xml | 118 ++++++
.../services/OOZIE2/themes/broken_theme.json | 3 +
.../test_kerberos_descriptor_ranger_kms.json | 286 ++++++++++++++
.../stacks_with_extensions/HDP/0.3/metainfo.xml | 22 ++
.../HDP/0.3/repos/repoinfo.xml | 63 +++
.../HDP/0.3/services/HBASE/metainfo.xml | 26 ++
.../0.3/services/HDFS/configuration/global.xml | 145 +++++++
.../services/HDFS/configuration/hadoop-env.xml | 223 +++++++++++
.../services/HDFS/configuration/hbase-site.xml | 137 +++++++
.../services/HDFS/configuration/hdfs-log4j.xml | 199 ++++++++++
.../services/HDFS/configuration/hdfs-site.xml | 396 +++++++++++++++++++
.../HDP/0.3/services/HDFS/metainfo.xml | 30 ++
.../0.3/services/HDFS/package/dummy-script.py | 20 +
.../HDP/0.3/services/HIVE/metainfo.xml | 26 ++
.../HDP/0.3/services/MAPREDUCE/metainfo.xml | 23 ++
.../HDP/0.3/services/ZOOKEEPER/metainfo.xml | 26 ++
.../stacks_with_extensions/HDP/0.4/metainfo.xml | 22 ++
.../HDP/0.4/repos/repoinfo.xml | 63 +++
.../HDP/0.4/services/HBASE/metainfo.xml | 26 ++
.../0.4/services/HDFS/configuration/global.xml | 145 +++++++
.../services/HDFS/configuration/hadoop-env.xml | 223 +++++++++++
.../services/HDFS/configuration/hbase-site.xml | 137 +++++++
.../services/HDFS/configuration/hdfs-log4j.xml | 199 ++++++++++
.../services/HDFS/configuration/hdfs-site.xml | 396 +++++++++++++++++++
.../HDP/0.4/services/HDFS/metainfo.xml | 30 ++
.../0.4/services/HDFS/package/dummy-script.py | 20 +
.../HDP/0.4/services/HIVE/metainfo.xml | 26 ++
.../HDP/0.4/services/MAPREDUCE/metainfo.xml | 23 ++
.../HDP/0.4/services/ZOOKEEPER/metainfo.xml | 26 ++
.../main/admin/kerberos/step4_controller.js | 6 +
.../app/controllers/wizard/step3_controller.js | 2 +-
ambari-web/app/messages.js | 5 +-
ambari-web/app/mixins/common/serverValidator.js | 41 +-
ambari-web/app/styles/application.less | 3 +-
.../config_recommendation_popup.hbs | 102 +++--
.../config_validation_popup.js | 7 +-
.../test/mixins/common/serverValidator_test.js | 15 +-
.../apache/ambari/view/utils/hdfs/HdfsApi.java | 16 +-
.../apache/ambari/view/utils/hdfs/HdfsUtil.java | 17 +-
.../ui/app/domain/workflow-importer.js | 3 +-
79 files changed, 4584 insertions(+), 382 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/606d876b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/606d876b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
----------------------------------------------------------------------