knox git commit: KNOX-1335 - Spark service discovery when SSL is configured

[pz...@apache.org]

Repository: knox
Updated Branches:
  refs/heads/master 08cf0cbca -> 2a6d72c34


KNOX-1335 - Spark service discovery when SSL is configured


Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/2a6d72c3
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/2a6d72c3
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/2a6d72c3

Branch: refs/heads/master
Commit: 2a6d72c34c90faf861b714c4ad3260571fb328a8
Parents: 08cf0cb
Author: Phil Zampino <pz...@apache.org>
Authored: Fri May 25 15:32:11 2018 -0400
Committer: Phil Zampino <pz...@apache.org>
Committed: Fri May 25 15:32:11 2018 -0400

----------------------------------------------------------------------
 .../discovery/ambari/LivyServiceURLCreator.java |   6 +
 .../ambari/SparkCommonServiceURLCreator.java    |  18 ++-
 .../ambari/SparkHistoryUIServiceURLCreator.java |  32 +++++
 .../ambari/SparkServiceURLCreatorTest.java      | 118 +++++++++++++++++++
 4 files changed, 171 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/knox/blob/2a6d72c3/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/LivyServiceURLCreator.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/LivyServiceURLCreator.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/LivyServiceURLCreator.java
index 6c12b18..c098693 100644
--- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/LivyServiceURLCreator.java
+++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/LivyServiceURLCreator.java
@@ -33,4 +33,10 @@ public class LivyServiceURLCreator extends SparkCommonServiceURLCreator {
     return RESOURCE_ROLE;
   }
 
+  @Override
+  boolean isSSL(AmbariComponent comp) {
+    String keystore = comp.getConfigProperty("livy.keystore");
+    return (keystore != null && !keystore.isEmpty());
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/knox/blob/2a6d72c3/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkCommonServiceURLCreator.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkCommonServiceURLCreator.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkCommonServiceURLCreator.java
index ae223d3..7f98872 100644
--- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkCommonServiceURLCreator.java
+++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkCommonServiceURLCreator.java
@@ -22,7 +22,10 @@ import java.util.Map;
 
 public abstract class SparkCommonServiceURLCreator implements ServiceURLCreator {
 
-  private static final String URL_TEMPLATE = "http://%s:%s";
+  private static final String SCHEME_HTTP  = "http";
+  private static final String SCHEME_HTTPS = "https";
+
+  private static final String URL_TEMPLATE = "%s://%s:%s";
 
   protected AmbariCluster cluster = null;
 
@@ -37,6 +40,15 @@ public abstract class SparkCommonServiceURLCreator implements ServiceURLCreator
     this.cluster = cluster;
   }
 
+  boolean isSSL(AmbariComponent comp) {
+    return false;
+  }
+
+  String getPort(AmbariComponent comp) {
+    return comp.getConfigProperty(portConfigProperty);
+  }
+
+
   @Override
   public List<String> create(String service, Map<String, String> serviceParams) {
     List<String> urls = new ArrayList<>();
@@ -48,10 +60,10 @@ public abstract class SparkCommonServiceURLCreator implements ServiceURLCreator
       }
 
       if (comp != null) {
-        String port = comp.getConfigProperty(portConfigProperty);
+        String port = getPort(comp);
         List<String> hostNames = comp.getHostNames();
         for (String host : hostNames) {
-          urls.add(String.format(URL_TEMPLATE, host, port));
+          urls.add(String.format(URL_TEMPLATE, (isSSL(comp) ? SCHEME_HTTPS : SCHEME_HTTP), host, port));
         }
       }
     }

http://git-wip-us.apache.org/repos/asf/knox/blob/2a6d72c3/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkHistoryUIServiceURLCreator.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkHistoryUIServiceURLCreator.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkHistoryUIServiceURLCreator.java
index a0de1d5..09506c9 100644
--- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkHistoryUIServiceURLCreator.java
+++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkHistoryUIServiceURLCreator.java
@@ -20,6 +20,14 @@ public class SparkHistoryUIServiceURLCreator extends SparkCommonServiceURLCreato
 
   private static final String RESOURCE_ROLE = "SPARKHISTORYUI";
 
+
+  private static final String SSL_FLAG_PRIMARY   = "spark.ssl.historyServer.enabled";
+  private static final String SSL_FLAG_SECONDARY = "spark.ssl.enabled";
+
+  private static final String SSL_PORT_PROPERTY = "spark.ssl.historyServer.port";
+
+  private static final int SSL_PORT_OFFSET = 400;
+
   @Override
   public void init(AmbariCluster cluster) {
     super.init(cluster);
@@ -33,4 +41,28 @@ public class SparkHistoryUIServiceURLCreator extends SparkCommonServiceURLCreato
     return RESOURCE_ROLE;
   }
 
+
+  @Override
+  String getPort(AmbariComponent comp) {
+    String port;
+
+    if (isSSL(comp)) {
+      String sslPort = comp.getConfigProperty(SSL_PORT_PROPERTY);
+      if (sslPort == null || sslPort.isEmpty()) {
+        int p = Integer.valueOf(comp.getConfigProperty(portConfigProperty)) + SSL_PORT_OFFSET;
+        sslPort = String.valueOf(p);
+      }
+      port = sslPort;
+    } else {
+      port = comp.getConfigProperty(portConfigProperty);
+    }
+
+    return port;
+  }
+
+  @Override
+  boolean isSSL(AmbariComponent comp) {
+    return Boolean.valueOf(comp.getConfigProperty(SSL_FLAG_PRIMARY)) || Boolean.valueOf(comp.getConfigProperty(SSL_FLAG_SECONDARY));
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/knox/blob/2a6d72c3/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java
index 9ee6dc8..e346166 100644
--- a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java
+++ b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java
@@ -64,6 +64,90 @@ public class SparkServiceURLCreatorTest {
 
 
   @Test
+  public void testSparkHistoryUI_SSL() {
+    doTestSparkHistoryUI_SSL("SPARK_JOBHISTORYSERVER", true, false, "4321");
+  }
+
+  @Test
+  public void testSparkHistoryUI_SSL_FALLBACK_FLAG() {
+    doTestSparkHistoryUI_SSL("SPARK_JOBHISTORYSERVER", false, true, "4321");
+  }
+
+  @Test
+  public void testSparkHistoryUI_SSL_FALLBACK_PORT() {
+    doTestSparkHistoryUI_SSL("SPARK_JOBHISTORYSERVER", true, false, null);
+  }
+
+  @Test
+  public void testSparkHistoryUI_SSL_FALLBACK_FLAG_AND_PORT() {
+    doTestSparkHistoryUI_SSL("SPARK_JOBHISTORYSERVER", false, true, null);
+  }
+
+  @Test
+  public void testSpark2HistoryUI_SSL() {
+    doTestSparkHistoryUI_SSL("SPARK2_JOBHISTORYSERVER", true, false, "4321");
+  }
+
+  @Test
+  public void testSparkwHistoryUI_SSL_FALLBACK_FLAG() {
+    doTestSparkHistoryUI_SSL("SPARK2_JOBHISTORYSERVER", false, true, "4321");
+  }
+
+  @Test
+  public void testSpark2HistoryUI_SSL_FALLBACK_PORT() {
+    doTestSparkHistoryUI_SSL("SPARK2_JOBHISTORYSERVER", true, false, null);
+  }
+
+  @Test
+  public void testSpark2HistoryUI_SSL_FALLBACK_FLAG_AND_PORT() {
+    doTestSparkHistoryUI_SSL("SPARK2_JOBHISTORYSERVER", false, true, null);
+  }
+
+  private void doTestSparkHistoryUI_SSL(String componentName, Boolean sslSHS, Boolean sslSpark, String sslSHSPort) {
+    final String PORT = "4545";
+
+    boolean isSSLConfigured = false;
+
+    AmbariComponent ac = EasyMock.createNiceMock(AmbariComponent.class);
+    List<String> hostNames = Arrays.asList("host1", "host2");
+    EasyMock.expect(ac.getHostNames()).andReturn(hostNames).anyTimes();
+    EasyMock.expect(ac.getConfigProperty("spark.history.ui.port")).andReturn(PORT).anyTimes();
+    if (sslSHS != null) {
+      isSSLConfigured = true;
+      EasyMock.expect(ac.getConfigProperty("spark.ssl.historyServer.enabled")).andReturn(String.valueOf(sslSHS)).anyTimes();
+    }
+    if (sslSpark != null) {
+      isSSLConfigured = true;
+      EasyMock.expect(ac.getConfigProperty("spark.ssl.enabled")).andReturn(String.valueOf(sslSpark)).anyTimes();
+    }
+    if (sslSHSPort != null) {
+      EasyMock.expect(ac.getConfigProperty("spark.ssl.historyServer.port")).andReturn(sslSHSPort).anyTimes();
+    }
+    EasyMock.replay(ac);
+
+    AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+    EasyMock.expect(cluster.getComponent(componentName)).andReturn(ac).anyTimes();
+    EasyMock.replay(cluster);
+
+    SparkHistoryUIServiceURLCreator c = new SparkHistoryUIServiceURLCreator();
+    c.init(cluster);
+    List<String> urls = c.create("SPARKHISTORYUI", null);
+    assertNotNull(urls);
+    assertFalse(urls.isEmpty());
+    assertEquals(2, urls.size());
+
+    if (isSSLConfigured) {
+      String expectedPort = sslSHSPort != null ? sslSHSPort : String.valueOf(Integer.valueOf(PORT) + 400);
+      assertEquals("https://host1:" + expectedPort, urls.get(0));
+      assertEquals("https://host2:" + expectedPort, urls.get(1));
+    } else {
+      assertEquals("http://host1:" + PORT, urls.get(0));
+      assertEquals("http://host2:" + PORT, urls.get(1));
+    }
+  }
+
+
+  @Test
   public void testSparkAndSpark2HistoryUI() {
     final String PORT  = "4545";
     final String PORT2 = "6767";
@@ -128,6 +212,40 @@ public class SparkServiceURLCreatorTest {
   }
 
   @Test
+  public void testLivyServer_SSL() {
+    doTestLivyServerSSL("LIVY_SERVER");
+  }
+
+  @Test
+  public void testLivy2Server_SSL() {
+    doTestLivyServerSSL("LIVY2_SERVER");
+  }
+
+  private void doTestLivyServerSSL(String componentName) {
+    final String PORT = "4545";
+
+    AmbariComponent ac = EasyMock.createNiceMock(AmbariComponent.class);
+    List<String> hostNames = Arrays.asList("host1", "host2");
+    EasyMock.expect(ac.getHostNames()).andReturn(hostNames).anyTimes();
+    EasyMock.expect(ac.getConfigProperty("livy.server.port")).andReturn(PORT).anyTimes();
+    EasyMock.expect(ac.getConfigProperty("livy.keystore")).andReturn("mykeystore").anyTimes();
+    EasyMock.replay(ac);
+
+    AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+    EasyMock.expect(cluster.getComponent(componentName)).andReturn(ac).anyTimes();
+    EasyMock.replay(cluster);
+
+    LivyServiceURLCreator c = new LivyServiceURLCreator();
+    c.init(cluster);
+    List<String> urls = c.create("LIVYSERVER", null);
+    assertNotNull(urls);
+    assertFalse(urls.isEmpty());
+    assertEquals(2, urls.size());
+    assertEquals("https://host1:" + PORT, urls.get(0));
+    assertEquals("https://host2:" + PORT, urls.get(1));
+  }
+
+  @Test
   public void testLivyAndLivy2Server() {
     final String PORT  = "4545";
     final String PORT2 = "2323";