You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2016/07/12 20:37:41 UTC
ambari git commit: AMBARI-17662. Atlas HA fails to come up with error
finding ids (alejandro)
Repository: ambari
Updated Branches:
refs/heads/branch-2.4 121dc7fa8 -> 7c809e278
AMBARI-17662. Atlas HA fails to come up with error finding ids (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7c809e27
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7c809e27
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7c809e27
Branch: refs/heads/branch-2.4
Commit: 7c809e278e7820e5ff90362209d98bd7d797e086
Parents: 121dc7f
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Fri Jul 8 17:45:58 2016 -0700
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Tue Jul 12 13:39:32 2016 -0700
----------------------------------------------------------------------
.../BlueprintConfigurationProcessor.java | 57 ------------
.../ATLAS/0.1.0.2.3/package/scripts/params.py | 91 +++++++++++--------
.../0.1.0.2.3/package/scripts/status_params.py | 13 +--
.../configuration/application-properties.xml | 28 ------
.../BlueprintConfigurationProcessorTest.java | 94 --------------------
.../stacks/2.5/common/test_stack_advisor.py | 4 +-
6 files changed, 63 insertions(+), 224 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c809e27/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index c3af4eb..cf12800 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -2129,60 +2129,6 @@ public class BlueprintConfigurationProcessor {
}
/**
- * Custom PropertyUpdater that handles the updating of the Atlas HA related properties.
- */
- private static class AtlasHAPropertyUpdater extends MultipleHostTopologyUpdater {
-
- public AtlasHAPropertyUpdater() {
- super("ATLAS_SERVER");
- }
-
- @Override
- public String updateForClusterCreate(String propertyName, String origValue, Map<String,
- Map<String, String>> properties, ClusterTopology topology) {
-
- int serverId = 1;
-
- StringBuilder sb = new StringBuilder();
-
- Collection<String> hosts = topology.getHostAssignmentsForComponent("ATLAS_SERVER");
-
- switch (propertyName) {
- case "atlas.server.address.id1":
-
- Map<String, String> applicationProperties = properties.get("application-properties");
-
- Boolean ssl_enabled = Boolean.parseBoolean(applicationProperties.get("atlas.enableTLS"));
-
- String port = ssl_enabled ? applicationProperties.get("atlas.server.https.port") :
- applicationProperties.get("atlas.server.http.port");
-
- for (String host : hosts) {
-
- if (serverId > 1) {
- sb.append("\n").append("atlas.server.address.id").append(serverId).append("=");
- }
- sb.append(host).append(":").append(port);
- ++serverId;
- }
- break;
- case "atlas.server.ids":
-
- while (serverId <= hosts.size()) {
- if (serverId > 1) {
- sb.append(",");
- }
- sb.append("id" + serverId++);
- }
- break;
- default:
- return origValue;
- }
- return sb.toString();
- }
- }
-
- /**
* Custom PropertyUpdater that handles the parsing and updating of the
* "templeton.hive.properties" configuration property for WebHCat.
* This particular configuration property uses a format of
@@ -2660,9 +2606,6 @@ public class BlueprintConfigurationProcessor {
// ATLAS
atlasPropsMap.put("atlas.server.bind.address", new SingleHostTopologyUpdater("ATLAS_SERVER"));
- PropertyUpdater atlasHAUpdater = new AtlasHAPropertyUpdater();
- atlasPropsMap.put("atlas.server.ids", atlasHAUpdater);
- atlasPropsMap.put("atlas.server.address.id1", atlasHAUpdater);
atlasPropsMap.put("atlas.kafka.bootstrap.servers", new MultipleHostTopologyUpdater("KAFKA_BROKER"));
atlasPropsMap.put("atlas.kafka.zookeeper.connect", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER"));
atlasPropsMap.put("atlas.graph.index.search.solr.zookeeper-url", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER", ',', false, true, true));
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c809e27/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
index 2119268..8445214 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
@@ -17,38 +17,70 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
+# Python Imports
import os
import sys
+
+# Local Imports
+from status_params import *
from resource_management import format_stack_version, Script
from resource_management.libraries.functions import format
from resource_management.libraries.functions.default import default
-
-import status_params
from resource_management.libraries.functions.stack_features import check_stack_feature
from resource_management.libraries.functions import StackFeature
from resource_management.libraries.functions.is_empty import is_empty
from resource_management.libraries.functions.expect import expect
+
+def configs_for_ha(atlas_hosts, metadata_port, is_atlas_ha_enabled):
+ """
+ Return a dictionary of additional configs to merge if Atlas HA is enabled.
+ :param atlas_hosts: List of hostnames that contain Atlas
+ :param metadata_port: Port number
+ :param is_atlas_ha_enabled: None, True, or False
+ :return: Dictionary with additional configs to merge to application-properties if HA is enabled.
+ """
+ additional_props = {}
+ if atlas_hosts is None or len(atlas_hosts) == 0 or metadata_port is None:
+ return additional_props
+
+ # Sort to guarantee each host sees the same values, assuming restarted at the same time.
+ atlas_hosts = sorted(atlas_hosts)
+
+ # E.g., id1,id2,id3,...,idn
+ _server_id_list = ["id" + str(i) for i in range(1, len(atlas_hosts) + 1)]
+ atlas_server_ids = ",".join(_server_id_list)
+ additional_props["atlas.server.ids"] = atlas_server_ids
+
+ i = 0
+ for curr_hostname in atlas_hosts:
+ id = _server_id_list[i]
+ prop_name = "atlas.server.address." + id
+ prop_value = curr_hostname + ":" + metadata_port
+ additional_props[prop_name] = prop_value
+ i += 1
+
+ # This may override the existing property
+ if i == 1 or (i > 1 and is_atlas_ha_enabled is False):
+ additional_props["atlas.server.ha.enabled"] = "false"
+ elif i > 1:
+ additional_props["atlas.server.ha.enabled"] = "true"
+
+ return additional_props
+
# server configurations
config = Script.get_config()
stack_root = Script.get_stack_root()
-tmp_dir = Script.get_tmp_dir()
-
cluster_name = config['clusterName']
java_version = expect("/hostLevelParams/java_version", int)
-# security enabled
-security_enabled = status_params.security_enabled
-
if security_enabled:
_hostname_lowercase = config['hostname'].lower()
_atlas_principal_name = config['configurations']['application-properties']['atlas.authentication.principal']
atlas_jaas_principal = _atlas_principal_name.replace('_HOST',_hostname_lowercase)
atlas_keytab_path = config['configurations']['application-properties']['atlas.authentication.keytab']
-stack_name = status_params.stack_name
-
# New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade
version = default("/commandParams/version", None)
@@ -65,8 +97,6 @@ metadata_stop_script = format("{metadata_bin}/atlas_stop.py")
# metadata local directory structure
log_dir = config['configurations']['atlas-env']['metadata_log_dir']
-conf_dir = status_params.conf_dir # "/etc/metadata/conf"
-conf_file = status_params.conf_file
# service locations
hadoop_conf_dir = os.path.join(os.environ["HADOOP_HOME"], "conf") if 'HADOOP_HOME' in os.environ else '/etc/hadoop/conf'
@@ -74,11 +104,8 @@ hadoop_conf_dir = os.path.join(os.environ["HADOOP_HOME"], "conf") if 'HADOOP_HOM
# some commands may need to supply the JAAS location when running as atlas
atlas_jaas_file = format("{conf_dir}/atlas_jaas.conf")
-# user and status
-metadata_user = status_params.metadata_user
+# user
user_group = config['configurations']['cluster-env']['user_group']
-pid_dir = status_params.pid_dir
-pid_file = format("{pid_dir}/atlas.pid")
# metadata env
java64_home = config['hostLevelParams']['java_home']
@@ -104,6 +131,18 @@ metadata_host = config['hostname']
application_properties = dict(config['configurations']['application-properties'])
application_properties['atlas.server.bind.address'] = metadata_host
+# Atlas HA should populate
+# atlas.server.ids = id1,id2,...,idn
+# atlas.server.address.id# = host#:port
+atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', [])
+# User should not have to modify this property, but still allow overriding it to False if multiple Atlas servers exist
+# This can be None, True, or False
+is_atlas_ha_enabled = default("/configurations/application-properties/atlas.server.ha.enabled", None)
+additional_ha_props = configs_for_ha(atlas_hosts, metadata_port, is_atlas_ha_enabled)
+for k,v in additional_ha_props.iteritems():
+ application_properties[k] = v
+
+
metadata_env_content = config['configurations']['atlas-env']['content']
metadata_opts = config['configurations']['atlas-env']['metadata_opts']
@@ -124,7 +163,6 @@ smoke_test_password = 'smoke'
smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
smokeuser_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
-kinit_path_local = status_params.kinit_path_local
security_check_status_file = format('{log_dir}/security_check.status')
if security_enabled:
@@ -135,26 +173,6 @@ else:
# hbase
hbase_conf_dir = "/etc/hbase/conf"
-# atlas HA
-atlas_hosts = sorted(default('/clusterHostInfo/atlas_server_hosts', []))
-
-id = 1
-server_ids = ""
-server_hosts = ""
-first_id = True
-for host in atlas_hosts:
- server_id = "id" + str(id)
- server_host = host + ":" + metadata_port
- if first_id:
- server_ids = server_id
- server_hosts = server_host
- else:
- server_ids += "," + server_id
- server_hosts += "\n" + "atlas.server.address." + server_id + "=" + server_host
-
- id += 1
- first_id = False
-
atlas_search_backend = default("/configurations/application-properties/atlas.graph.index.search.backend", "")
search_backend_solr = atlas_search_backend.startswith('solr')
@@ -199,7 +217,6 @@ atlas_server_max_new_size = config['configurations']['atlas-env']['atlas_server_
if has_ranger_admin and stack_supports_atlas_ranger_plugin:
# for create_hdfs_directory
- hadoop_bin_dir = status_params.hadoop_bin_dir
namenode_host = set(default("/clusterHostInfo/namenode_host", []))
has_namenode = not len(namenode_host) == 0
hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c809e27/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
index 0b0d2ae..7ea70c0 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
@@ -18,8 +18,10 @@ limitations under the License.
"""
import os
+
from resource_management import Script
-from resource_management.libraries.functions import get_kinit_path, format
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions import format
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import stack_select
@@ -27,15 +29,16 @@ from resource_management.libraries.functions import stack_select
config = Script.get_config()
-conf_file = config['configurations']['atlas-env']['metadata_conf_file']
+conf_file = default("/configurations/atlas-env/metadata_conf_file", "atlas-application.properties")
conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf'
-pid_dir = config['configurations']['atlas-env']['metadata_pid_dir']
+pid_dir = default("/configurations/atlas-env/metadata_pid_dir", "/var/run/atlas")
pid_file = format("{pid_dir}/atlas.pid")
-metadata_user = config['configurations']['atlas-env']['metadata_user']
+
+metadata_user = default("/configurations/atlas-env/metadata_user", None)
# Security related/required params
hostname = config['hostname']
-security_enabled = config['configurations']['cluster-env']['security_enabled']
+security_enabled = default("/configurations/cluster-env/security_enabled", None)
kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
tmp_dir = Script.get_tmp_dir()
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c809e27/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml
index 0fe0827..aa9940e 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml
@@ -21,34 +21,6 @@
-->
<configuration supports_final="false">
- <!-- HA properties -->
- <property>
- <name>atlas.server.ha.enabled</name>
- <value>false</value>
- <description>Atlas high availability feature toggle.</description>
- <on-ambari-upgrade add="false"/>
- </property>
- <property>
- <name>atlas.server.ids</name>
- <value>{{server_ids}}</value>
- <description>List of Atlas server ids for HA feature.</description>
- <value-attributes>
- <overridable>false</overridable>
- <visible>false</visible>
- </value-attributes>
- <on-ambari-upgrade add="false"/>
- </property>
- <property>
- <name>atlas.server.address.id1</name>
- <value>{{server_hosts}}</value>
- <description>Mapping of Atlas server ids to hosts.</description>
- <value-attributes>
- <overridable>false</overridable>
- <visible>false</visible>
- </value-attributes>
- <on-ambari-upgrade add="false"/>
- </property>
-
<!-- Misc properties -->
<property>
<name>atlas.audit.hbase.zookeeper.quorum</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c809e27/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index b595868..692fac6 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -4857,100 +4857,6 @@ public class BlueprintConfigurationProcessorTest {
}
@Test
- public void testAtlasHA() throws Exception {
- final String expectedHostGroupName = "host_group_1";
- final String host1 = "c6401.ambari.apache.org";
- final String host2 = "c6402.ambari.apache.org";
- final String host3 = "c6403.ambari.apache.org";
-
- Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
- Map<String, String> atlasProperties = new HashMap<String, String>();
- properties.put("application-properties", atlasProperties);
-
- // setup properties that include host information
- atlasProperties.put("atlas.server.ids", "");
- atlasProperties.put("atlas.server.address.id1", "");
- atlasProperties.put("atlas.server.http.port", "21000");
- atlasProperties.put("atlas.server.https.port", "21443");
- atlasProperties.put("atlas.enableTLS", "false");
-
- Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap());
-
- Collection<String> hgComponents = new HashSet<String>();
- hgComponents.add("ATLAS_SERVER");
- List<String> hosts = new ArrayList<String>();
- hosts.add(host1);
- hosts.add(host2);
- hosts.add(host3);
- TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, hosts);
-
- Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>();
- hostGroups.add(group1);
-
- ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
- BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology);
-
- // call top-level cluster config update method
- updater.doUpdateForClusterCreate();
-
- assertEquals("id1,id2,id3", atlasProperties.get("atlas.server.ids"));
-
- List<String> hostArray =
- Arrays.asList(atlasProperties.get("atlas.server.address.id1").split("\natlas.server.address.id.="));
- List<String> expected =
- Arrays.asList("c6401.ambari.apache.org:21000","c6402.ambari.apache.org:21000", "c6403.ambari.apache.org:21000");
-
- Assert.assertTrue(hostArray.containsAll(expected) && expected.containsAll(hostArray));
- }
-
- @Test
- public void testAtlasHAEnableTLS() throws Exception {
- final String expectedHostGroupName = "host_group_1";
- final String host1 = "c6401.ambari.apache.org";
- final String host2 = "c6402.ambari.apache.org";
- final String host3 = "c6403.ambari.apache.org";
-
- Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
- Map<String, String> atlasProperties = new HashMap<String, String>();
- properties.put("application-properties", atlasProperties);
-
- // setup properties that include host information
- atlasProperties.put("atlas.server.ids", "");
- atlasProperties.put("atlas.server.address.id1", "");
- atlasProperties.put("atlas.server.http.port", "21000");
- atlasProperties.put("atlas.server.https.port", "21443");
- atlasProperties.put("atlas.enableTLS", "true");
-
- Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap());
-
- Collection<String> hgComponents = new HashSet<String>();
- hgComponents.add("ATLAS_SERVER");
- List<String> hosts = new ArrayList<String>();
- hosts.add(host1);
- hosts.add(host2);
- hosts.add(host3);
- TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, hosts);
-
- Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>();
- hostGroups.add(group1);
-
- ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
- BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology);
-
- // call top-level cluster config update method
- updater.doUpdateForClusterCreate();
-
- assertEquals("id1,id2,id3", atlasProperties.get("atlas.server.ids"));
-
- List<String> hostArray =
- Arrays.asList(atlasProperties.get("atlas.server.address.id1").split("\natlas.server.address.id.="));
- List<String> expected =
- Arrays.asList("c6401.ambari.apache.org:21443","c6402.ambari.apache.org:21443", "c6403.ambari.apache.org:21443");
-
- Assert.assertTrue(hostArray.containsAll(expected) && expected.containsAll(hostArray));
- }
-
- @Test
public void testHiveConfigClusterUpdateExportedHostGroupValue() throws Exception {
final String expectedHostGroupName = "host_group_1";
final String expectedHostName = "c6401.ambari.apache.org";
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c809e27/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 58cce06..40f3367 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -6646,9 +6646,7 @@ class TestHDP25StackAdvisor(TestCase):
"atlas.audit.hbase.zookeeper.quorum": "",
"atlas.graph.storage.hostname": "",
"atlas.kafka.bootstrap.servers": "",
- "atlas.kafka.zookeeper.connect": "",
- 'atlas.server.address.id1': "",
- 'atlas.server.ids': ""
+ "atlas.kafka.zookeeper.connect": ""
}
},
"logsearch-solr-env": {