You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ad...@apache.org on 2018/07/10 20:27:45 UTC
[ambari] branch branch-feature-AMBARI-14714 updated: AMBARI-14714.
Fix merge issues (#1740)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch branch-feature-AMBARI-14714
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by this push:
new c1f8558 AMBARI-14714. Fix merge issues (#1740)
c1f8558 is described below
commit c1f8558940cdec77ac0bc1fd416747f9dfb87780
Author: Doroszlai, Attila <64...@users.noreply.github.com>
AuthorDate: Tue Jul 10 22:27:43 2018 +0200
AMBARI-14714. Fix merge issues (#1740)
---
.../execution_command/execution_command.py | 2 +-
.../resource_management/libraries/script/script.py | 30 ++++----
.../ambari/server/agent/ExecutionCommand.java | 2 +
.../ambari/server/configuration/AmbariConfig.java | 2 +-
.../controller/AmbariManagementControllerImpl.java | 11 ---
.../server/metadata/ClusterMetadataGenerator.java | 84 +++++++++++-----------
.../stack-hooks/after-INSTALL/scripts/params.py | 5 +-
.../server/upgrade/UpgradeCatalog270Test.java | 5 +-
8 files changed, 66 insertions(+), 75 deletions(-)
diff --git a/ambari-common/src/main/python/resource_management/libraries/execution_command/execution_command.py b/ambari-common/src/main/python/resource_management/libraries/execution_command/execution_command.py
index 98bd62f..72f7c21 100644
--- a/ambari-common/src/main/python/resource_management/libraries/execution_command/execution_command.py
+++ b/ambari-common/src/main/python/resource_management/libraries/execution_command/execution_command.py
@@ -367,7 +367,7 @@ class ExecutionCommand(object):
return self.__get_value('commandParams/phase')
def get_dfs_type(self):
- return self.__get_value('clusterLevelParams/dfs_type')
+ return self.__get_value('stackSettings/dfs_type')
def get_module_package_folder(self):
return self.__get_value('commandParams/service_package_folder')
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index 93fe672..c8d0197 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -801,21 +801,21 @@ class Script(object):
self.available_packages_in_repos = []
return self.available_packages_in_repos
-def create_component_instance(self):
- # should be used only when mpack-instance-manager is available
- from resource_management.libraries.functions.mpack_manager_helper import create_component_instance
- config = self.get_config()
- execution_command = self.get_execution_command()
- mpack_name = execution_command.get_mpack_name()
- mpack_version = execution_command.get_mpack_version()
- mpack_instance_name = execution_command.get_servicegroup_name()
- module_name = execution_command.get_module_name()
- component_type = execution_command.get_component_type()
- component_instance_name = execution_command.get_component_instance_name()
-
- create_component_instance(mpack_name=mpack_name, mpack_version=mpack_version, instance_name=mpack_instance_name,
- module_name=module_name, components_instance_type=component_type,
- component_instance_name=component_instance_name)
+ def create_component_instance(self):
+ # should be used only when mpack-instance-manager is available
+ from resource_management.libraries.functions.mpack_manager_helper import create_component_instance
+ config = self.get_config()
+ execution_command = self.get_execution_command()
+ mpack_name = execution_command.get_mpack_name()
+ mpack_version = execution_command.get_mpack_version()
+ mpack_instance_name = execution_command.get_servicegroup_name()
+ module_name = execution_command.get_module_name()
+ component_type = execution_command.get_component_type()
+ component_instance_name = execution_command.get_component_instance_name()
+
+ create_component_instance(mpack_name=mpack_name, mpack_version=mpack_version, instance_name=mpack_instance_name,
+ module_name=module_name, components_instance_type=component_type,
+ component_instance_name=component_instance_name)
def install_packages(self, env):
"""
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
index 9639964..1009709 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
@@ -614,6 +614,8 @@ public class ExecutionCommand extends AgentCommand {
*/
String UPGRADE_SUSPENDED = "upgrade_suspended";
+ String CLUSTER_NAME = "cluster_name";
+
/**
* The version of the component to send down with the command. Normally,
* this is simply the repository version of the component. However, during
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/AmbariConfig.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/AmbariConfig.java
index 74bd2e9..8e55805 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/AmbariConfig.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/AmbariConfig.java
@@ -24,7 +24,7 @@ import org.apache.http.client.utils.URIBuilder;
public class AmbariConfig {
- private static final String JDK_RESOURCE_LOCATION = "/resources/";
+ private static final String JDK_RESOURCE_LOCATION = "/resources";
private final String masterHostname;
private final Integer masterPort;
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 6aed720..dfdfd74 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -5686,17 +5686,6 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
return commandParams;
}
- public SortedMap<String, SortedMap<String,String>> getMetadataAgentConfigs() {
- SortedMap<String, SortedMap<String,String>> agentConfigs = new TreeMap<>();
- Map<String, Map<String,String>> agentConfigsMap = configs.getAgentConfigsMap();
-
- for (String key : agentConfigsMap.keySet()) {
- agentConfigs.put(key, new TreeMap<>(agentConfigsMap.get(key)));
- }
-
- return agentConfigs;
- }
-
@Override
public HostRepositories retrieveHostRepositories(Cluster cluster, Host host) throws AmbariException {
List<ServiceComponentHost> hostComponents = cluster.getServiceComponentHosts(host.getHostName());
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metadata/ClusterMetadataGenerator.java b/ambari-server/src/main/java/org/apache/ambari/server/metadata/ClusterMetadataGenerator.java
index cbe4dbe..d3acb35 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/metadata/ClusterMetadataGenerator.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metadata/ClusterMetadataGenerator.java
@@ -22,8 +22,10 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STA
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_SERVER_HOST;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_SERVER_PORT;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_SERVER_USE_SSL;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLUSTER_NAME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DFS_TYPE;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GPL_LICENSE_ACCEPTED;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
@@ -56,6 +58,7 @@ import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.configuration.AmbariConfig;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.events.MetadataUpdateEvent;
+import org.apache.ambari.server.events.UpdateEventType;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.CommandScriptDefinition;
@@ -71,7 +74,6 @@ import org.apache.ambari.server.utils.StageUtils;
import com.google.gson.Gson;
-// TODO: [AMP] Revisit and fix
public class ClusterMetadataGenerator {
private final Configuration configs;
@@ -100,7 +102,7 @@ public class ClusterMetadataGenerator {
// STACK_NAME is part of stack settings, but STACK_VERSION is not
stackLevelParams.put(STACK_VERSION, stackId.getStackVersion());
- Map<String, DesiredConfig> clusterDesiredConfigs = cluster.getDesiredConfigs();
+ Map<String, DesiredConfig> clusterDesiredConfigs = cluster.getDesiredConfigs(false);
Set<PropertyInfo> stackProperties = ambariMetaInfo.getStackProperties(stackId.getStackName(), stackId.getStackVersion());
Map<String, ServiceInfo> servicesMap = ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion());
Set<PropertyInfo> clusterProperties = ambariMetaInfo.getClusterProperties();
@@ -124,6 +126,14 @@ public class ClusterMetadataGenerator {
String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet);
stackLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
+ Map<String, ServiceInfo> serviceInfos = ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion());
+ for (ServiceInfo serviceInfoInstance : serviceInfos.values()) {
+ if (serviceInfoInstance.getServiceType() != null) {
+ stackLevelParams.put(DFS_TYPE, serviceInfoInstance.getServiceType());
+ break;
+ }
+ }
+
return stackLevelParams;
}
@@ -138,67 +148,47 @@ public class ClusterMetadataGenerator {
MetadataCluster metadataCluster = new MetadataCluster(securityType,
getMetadataServiceLevelParams(cl),
- false,
+ true,
getMetadataClusterLevelParams(cl),
- new TreeMap<>());
+ null);
metadataClusters.put(Long.toString(cl.getClusterId()), metadataCluster);
}
- return new MetadataUpdateEvent(metadataClusters, getMetadataAmbariLevelParams(), null, null);
+ return new MetadataUpdateEvent(metadataClusters, getMetadataAmbariLevelParams(), getMetadataAgentConfigs(), UpdateEventType.CREATE);
}
public MetadataUpdateEvent getClusterMetadata(Cluster cl) throws AmbariException {
SortedMap<String, MetadataCluster> metadataClusters = new TreeMap<>();
- StackId stackId = cl.getDesiredStackVersion();
-
- SecurityType securityType = cl.getSecurityType();
-
- MetadataCluster metadataCluster = new MetadataCluster(securityType,
- getMetadataServiceLevelParams(cl),
- false,
- getMetadataClusterLevelParams(cl),
- new TreeMap<>());
+ MetadataCluster metadataCluster = new MetadataCluster(cl.getSecurityType(), getMetadataServiceLevelParams(cl), true, getMetadataClusterLevelParams(cl), null);
metadataClusters.put(Long.toString(cl.getClusterId()), metadataCluster);
-
- return new MetadataUpdateEvent(metadataClusters, null, null, null);
+ return new MetadataUpdateEvent(metadataClusters, null, getMetadataAgentConfigs(), UpdateEventType.UPDATE);
}
public MetadataUpdateEvent getClusterMetadataOnConfigsUpdate(Cluster cl) {
SortedMap<String, MetadataCluster> metadataClusters = new TreeMap<>();
-
- MetadataCluster metadataCluster = new MetadataCluster(null,
- new TreeMap<>(),
- false,
- getMetadataClusterLevelParams(cl), new TreeMap<>());
- metadataClusters.put(Long.toString(cl.getClusterId()), metadataCluster);
-
- return new MetadataUpdateEvent(metadataClusters, null, null, null);
+ metadataClusters.put(Long.toString(cl.getClusterId()), MetadataCluster.clusterLevelParamsMetadataCluster(null, getMetadataClusterLevelParams(cl)));
+ return new MetadataUpdateEvent(metadataClusters, null, getMetadataAgentConfigs(), UpdateEventType.UPDATE);
}
public MetadataUpdateEvent getClusterMetadataOnRepoUpdate(Cluster cl) throws AmbariException {
SortedMap<String, MetadataCluster> metadataClusters = new TreeMap<>();
-
- MetadataCluster metadataCluster = new MetadataCluster(null,
- getMetadataServiceLevelParams(cl), false,
- new TreeMap<>(), new TreeMap<>());
- metadataClusters.put(Long.toString(cl.getClusterId()), metadataCluster);
-
- return new MetadataUpdateEvent(metadataClusters, null, null, null);
+ metadataClusters.put(Long.toString(cl.getClusterId()), MetadataCluster.serviceLevelParamsMetadataCluster(null, getMetadataServiceLevelParams(cl), true));
+ return new MetadataUpdateEvent(metadataClusters, null, getMetadataAgentConfigs(), UpdateEventType.UPDATE);
}
public MetadataUpdateEvent getClusterMetadataOnServiceInstall(Cluster cl, String serviceName) throws AmbariException {
- SortedMap<String, MetadataCluster> metadataClusters = new TreeMap<>();
-
- MetadataCluster metadataCluster = new MetadataCluster(null,
- getMetadataServiceLevelParams(cl.getService(serviceName)),
- false, new TreeMap<>(), new TreeMap<>());
- metadataClusters.put(Long.toString(cl.getClusterId()), metadataCluster);
+ return getClusterMetadataOnServiceCredentialStoreUpdate(cl, serviceName);
+ }
- return new MetadataUpdateEvent(metadataClusters, null, null, null);
+ public MetadataUpdateEvent getClusterMetadataOnServiceCredentialStoreUpdate(Cluster cl, String serviceName) throws AmbariException {
+ final SortedMap<String, MetadataCluster> metadataClusters = new TreeMap<>();
+ metadataClusters.put(Long.toString(cl.getClusterId()), MetadataCluster.serviceLevelParamsMetadataCluster(null, getMetadataServiceLevelParams(cl), false));
+ return new MetadataUpdateEvent(metadataClusters, null, getMetadataAgentConfigs(), UpdateEventType.UPDATE);
}
private SortedMap<String, String> getMetadataClusterLevelParams(Cluster cluster) {
TreeMap<String, String> clusterLevelParams = new TreeMap<>();
+ clusterLevelParams.put(CLUSTER_NAME, cluster.getClusterName());
clusterLevelParams.put(HOOKS_FOLDER, configs.getProperty(Configuration.HOOKS_FOLDER));
return clusterLevelParams;
}
@@ -215,7 +205,6 @@ public class ClusterMetadataGenerator {
SortedMap<String, MetadataServiceInfo> serviceLevelParams = new TreeMap<>();
StackId serviceStackId = service.getStackId();
-
ServiceInfo serviceInfo = ambariMetaInfo.getService(serviceStackId.getStackName(),
serviceStackId.getStackVersion(), service.getName());
Long statusCommandTimeout = null;
@@ -224,14 +213,15 @@ public class ClusterMetadataGenerator {
}
String servicePackageFolder = serviceInfo.getServicePackageFolder();
+ Map<String, Map<String, String>> configCredentials = configHelper.getCredentialStoreEnabledProperties(serviceStackId, service);
serviceLevelParams.put(serviceInfo.getName(), new MetadataServiceInfo(serviceInfo.getVersion(),
- serviceInfo.isCredentialStoreEnabled(), null, statusCommandTimeout, servicePackageFolder));
+ service.isCredentialStoreEnabled(), configCredentials, statusCommandTimeout, servicePackageFolder));
return serviceLevelParams;
}
- public TreeMap<String, String> getMetadataAmbariLevelParams() throws AmbariException {
+ public TreeMap<String, String> getMetadataAmbariLevelParams() {
TreeMap<String, String> ambariLevelParams = new TreeMap<>();
ambariLevelParams.put(JDK_LOCATION, ambariConfig.getJdkResourceUrl());
ambariLevelParams.put(JAVA_HOME, ambariConfig.getJavaHome());
@@ -288,4 +278,16 @@ public class ClusterMetadataGenerator {
}
return commandTimeout;
}
+
+ public SortedMap<String, SortedMap<String,String>> getMetadataAgentConfigs() {
+ SortedMap<String, SortedMap<String,String>> agentConfigs = new TreeMap<>();
+ Map<String, Map<String,String>> agentConfigsMap = configs.getAgentConfigsMap();
+
+ for (String key : agentConfigsMap.keySet()) {
+ agentConfigs.put(key, new TreeMap<>(agentConfigsMap.get(key)));
+ }
+
+ return agentConfigs;
+ }
+
}
diff --git a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py
index e52f8f3..f8eb445 100644
--- a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py
+++ b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py
@@ -23,16 +23,13 @@ from ambari_commons.constants import AMBARI_SUDO_BINARY
from ambari_commons.constants import LOGFEEDER_CONF_DIR
from resource_management.libraries.script import Script
from resource_management.libraries.script.script import get_config_lock_file
-from resource_management.libraries.functions import default
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import stack_select
from resource_management.libraries.functions.format_jvm_option import format_jvm_option_value
from resource_management.libraries.functions.version import format_stack_version, get_major_version
from resource_management.libraries.functions.cluster_settings import get_cluster_setting_value
-from resource_management.libraries.execution_command import execution_command
-from resource_management.libraries.execution_command import module_configs
-from string import lower
+config = Script.get_config()
execution_command = Script.get_execution_command()
module_configs = Script.get_module_configs()
module_name = execution_command.get_module_name()
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog270Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog270Test.java
index b037b86..1585b11 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog270Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog270Test.java
@@ -1748,12 +1748,13 @@ public class UpgradeCatalog270Test {
.createNiceMock();
Injector injector2 = easyMockSupport.createNiceMock(Injector.class);
- Capture<Map> propertiesCapture = EasyMock.newCapture();
+ Capture<Map<String, String>> propertiesCapture = EasyMock.newCapture();
+ Capture<Map<String, Map<String, String>>> attributesCapture = EasyMock.newCapture();
expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes();
expect(controller.getClusters()).andReturn(clusters).anyTimes();
expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(), capture(propertiesCapture), anyString(),
- anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
+ capture(attributesCapture), anyLong())).andReturn(createNiceMock(Config.class)).once();
replay(controller, injector2);
new UpgradeCatalog270(injector2).clearHadoopMetrics2Content();