You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@knox.apache.org by pz...@apache.org on 2018/03/28 13:45:47 UTC
knox git commit: KNOX-1228 - Atlas HA ZooKeeper Config Discovery
Repository: knox
Updated Branches:
refs/heads/master a6d2f5240 -> 98dbfe990
KNOX-1228 - Atlas HA ZooKeeper Config Discovery
Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/98dbfe99
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/98dbfe99
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/98dbfe99
Branch: refs/heads/master
Commit: 98dbfe990887c2ce032015781cc99d4ae8198cbe
Parents: a6d2f52
Author: Phil Zampino <pz...@apache.org>
Authored: Thu Mar 22 13:22:13 2018 -0400
Committer: Phil Zampino <pz...@apache.org>
Committed: Wed Mar 28 09:29:04 2018 -0400
----------------------------------------------------------------------
.../discovery/ambari/AmbariCluster.java | 38 +++++++++++-
...rvice-discovery-zk-config-mapping.properties | 14 ++++-
.../discovery/ambari/AmbariClusterTest.java | 61 ++++++++++++++++++--
3 files changed, 106 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/knox/blob/98dbfe99/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java
index b5e4cff..9d2c92e 100644
--- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java
+++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java
@@ -19,6 +19,7 @@ package org.apache.knox.gateway.topology.discovery.ambari;
import org.apache.knox.gateway.i18n.messages.MessagesFactory;
import org.apache.knox.gateway.topology.discovery.ServiceDiscovery;
+import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.ArrayList;
@@ -33,6 +34,8 @@ class AmbariCluster implements ServiceDiscovery.Cluster {
private static final String ZK_CONFIG_MAPPING_FILE = "ambari-service-discovery-zk-config-mapping.properties";
+ private static final String ZK_CONFIG_MAPPING_OVERRIDE_FILE = "ambari-discovery-zk-config.properties";
+
static final String ZK_CONFIG_MAPPING_SYSTEM_PROPERTY =
"org.apache.knox.gateway.topology.discovery.ambari.zk.mapping";
@@ -48,7 +51,22 @@ class AmbariCluster implements ServiceDiscovery.Cluster {
}
// Attempt to apply overriding or additional mappings from external source
- String overridesPath = System.getProperty(ZK_CONFIG_MAPPING_SYSTEM_PROPERTY);
+ String overridesPath = null;
+
+ // First, check for the well-known overrides config file
+ String gatewayConfDir = System.getProperty(ServiceDiscovery.CONFIG_DIR_PROPERTY);
+ if (gatewayConfDir != null) {
+ File overridesFile = new File(gatewayConfDir, ZK_CONFIG_MAPPING_OVERRIDE_FILE);
+ if (overridesFile.exists()) {
+ overridesPath = overridesFile.getAbsolutePath();
+ }
+ }
+
+ // If no file in the config dir, check for the system property reference
+ if (overridesPath == null) {
+ overridesPath = System.getProperty(ZK_CONFIG_MAPPING_SYSTEM_PROPERTY);
+ }
+
if (overridesPath != null) {
Properties overrides = new Properties();
try (InputStream in = new FileInputStream(overridesPath)) {
@@ -150,6 +168,24 @@ class AmbariCluster implements ServiceDiscovery.Cluster {
String namespaceProp = zooKeeperHAConfigMappings.getProperty(serviceName + ".namespace");
Map<String, String> scProps = sc.getProperties();
if (scProps != null) {
+
+ if (ensembleProp != null) {
+ // If there are multiple ensemble properties specified, then iteratively check for the first
+ // valid value, and use that one.
+ String[] ensembleProps = ensembleProp.split(",");
+ if (ensembleProps.length > 1) {
+ for (String prop : ensembleProps) {
+ if (!prop.isEmpty()) {
+ String value = scProps.get(prop);
+ if (value != null) {
+ ensembleProp = prop;
+ break;
+ }
+ }
+ }
+ }
+ }
+
result =
new ZooKeeperConfiguration(enabledProp != null ? scProps.get(enabledProp) : null,
ensembleProp != null ? scProps.get(ensembleProp) : null,
http://git-wip-us.apache.org/repos/asf/knox/blob/98dbfe99/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-zk-config-mapping.properties
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-zk-config-mapping.properties b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-zk-config-mapping.properties
index 0a5faae..752382e 100644
--- a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-zk-config-mapping.properties
+++ b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-zk-config-mapping.properties
@@ -48,4 +48,16 @@ YARN.ensemble=yarn.resourcemanager.zk-address
# WEBHCAT
WEBHCAT.config=WEBHCAT:webhcat-site
-WEBHCAT.ensemble=templeton.zookeeper.hosts
\ No newline at end of file
+WEBHCAT.ensemble=templeton.zookeeper.hosts
+
+# ATLAS
+ATLAS.config=ATLAS:application-properties
+ATLAS.enabled=atlas.server.ha.enabled
+ATLAS.ensemble=atlas.server.ha.zookeeper.connect,atlas.kafka.zookeeper.connect
+ATLAS.namespace=atlas.server.ha.zookeeper.zkroot
+
+# ATLAS-API
+ATLAS-API.config=ATLAS:application-properties
+ATLAS-API.enabled=atlas.server.ha.enabled
+ATLAS-API.ensemble=atlas.server.ha.zookeeper.connect,atlas.kafka.zookeeper.connect
+ATLAS-API.namespace=atlas.server.ha.zookeeper.zkroot
http://git-wip-us.apache.org/repos/asf/knox/blob/98dbfe99/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariClusterTest.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariClusterTest.java b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariClusterTest.java
index abec9d9..c51da6a 100644
--- a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariClusterTest.java
+++ b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariClusterTest.java
@@ -16,17 +16,12 @@
*/
package org.apache.knox.gateway.topology.discovery.ambari;
-import org.apache.commons.io.FileUtils;
import org.apache.knox.gateway.topology.discovery.ServiceDiscovery;
-import org.apache.knox.test.TestUtils;
import org.easymock.EasyMock;
import org.junit.Test;
-import java.io.File;
-import java.io.FileOutputStream;
import java.util.HashMap;
import java.util.Map;
-import java.util.Properties;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@@ -143,6 +138,62 @@ public class AmbariClusterTest {
}
+ /**
+ * The Atlas ZooKeeper ensemble determination is based on multiple properties with a prioritized search order.
+ * This test verifies that the default property is used when the primary property value is undefined.
+ */
+ @Test
+ public void testAtlasZooKeeperEnsemblePropsConfigurationSecondary() throws Exception {
+
+ final boolean isEnabled = true;
+ final String ensemble = "host1:2181,host2:2181,host3:2181";
+ //final String ensemblePropPrimary = "atlas.server.ha.zookeeper.connect";
+ final String ensemblePropSecondary = "atlas.kafka.zookeeper.connect";
+ final String namespace = "/apache_atlas";
+
+ Map<String, String> serviceConfigProps = new HashMap<>();
+ // Since the primary property is undefined, the ZooKeeper config should query the value of the secondary property
+ serviceConfigProps.put(ensemblePropSecondary, ensemble);
+ serviceConfigProps.put("atlas.server.ha.zookeeper.zkroot", namespace);
+ serviceConfigProps.put("atlas.server.ha.enabled", "true");
+
+ AmbariCluster.ZooKeeperConfig config =
+ getZooKeeperConfiguration("ATLAS", "ATLAS", "application-properties", serviceConfigProps);
+ assertNotNull(config);
+ assertEquals(isEnabled, config.isEnabled());
+ assertEquals(ensemble, config.getEnsemble());
+ assertEquals(namespace, config.getNamespace());
+ }
+
+
+ /**
+ * The Atlas ZooKeeper ensemble determination is based on multiple properties with a prioritized search order.
+ * This test verifies that the primary property value is used when it's defined.
+ */
+ @Test
+ public void testAtlasZooKeeperEnsemblePropsConfigurationPrimary() throws Exception {
+
+ final boolean isEnabled = true;
+ final String ensemble = "host1:2181,host2:2181,host3:2181";
+ final String ensemblePropPrimary = "atlas.server.ha.zookeeper.connect";
+ final String ensemblePropSecondary = "atlas.kafka.zookeeper.connect";
+ final String namespace = "/apache_atlas";
+
+ Map<String, String> serviceConfigProps = new HashMap<>();
+ serviceConfigProps.put(ensemblePropSecondary, "invalid");
+ serviceConfigProps.put(ensemblePropPrimary, ensemble); // Since the primary property is defined, its value should be used
+ serviceConfigProps.put("atlas.server.ha.zookeeper.zkroot", namespace);
+ serviceConfigProps.put("atlas.server.ha.enabled", "true");
+
+ AmbariCluster.ZooKeeperConfig config =
+ getZooKeeperConfiguration("ATLAS", "ATLAS", "application-properties", serviceConfigProps);
+ assertNotNull(config);
+ assertEquals(isEnabled, config.isEnabled());
+ assertEquals(ensemble, config.getEnsemble());
+ assertEquals(namespace, config.getNamespace());
+ }
+
+
private ServiceDiscovery.Cluster.ZooKeeperConfig getZooKeeperConfiguration(final String serviceName,
final String configType,