You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@knox.apache.org by mo...@apache.org on 2017/05/22 18:49:08 UTC
knox git commit: KNOX-928 - Topology Port Mapping Feature
Repository: knox
Updated Branches:
refs/heads/master 8c1c94b9e -> 111d89747
KNOX-928 - Topology Port Mapping Feature
Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/111d8974
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/111d8974
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/111d8974
Branch: refs/heads/master
Commit: 111d8974771f23976bc802ea9b9d12d5fbd6f527
Parents: 8c1c94b
Author: Sandeep More <mo...@apache.org>
Authored: Mon May 22 14:48:55 2017 -0400
Committer: Sandeep More <mo...@apache.org>
Committed: Mon May 22 14:48:55 2017 -0400
----------------------------------------------------------------------
.../apache/hadoop/gateway/GatewayMessages.java | 81 ++++-
.../apache/hadoop/gateway/GatewayServer.java | 323 ++++++++++++++++---
.../gateway/config/impl/GatewayConfigImpl.java | 47 +++
.../gateway/filter/DefaultTopologyHandler.java | 2 +-
.../filter/PortMappingHelperHandler.java | 98 ++++++
.../gateway/filter/RequestUpdateHandler.java | 136 ++++++++
.../gateway/GatewayPortMappingConfigTest.java | 204 ++++++++++++
.../hadoop/gateway/config/GatewayConfig.java | 15 +
.../hadoop/gateway/GatewayTestConfig.java | 22 ++
.../GatewayPortMappingDisableFeatureTest.java | 252 +++++++++++++++
.../gateway/GatewayPortMappingFuncTest.java | 298 +++++++++++++++++
.../hadoop/gateway/GatewayTestConfig.java | 34 ++
12 files changed, 1456 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/knox/blob/111d8974/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java
index deb0034..1f94584 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java
@@ -22,9 +22,7 @@ import org.apache.hadoop.gateway.i18n.messages.Message;
import org.apache.hadoop.gateway.i18n.messages.MessageLevel;
import org.apache.hadoop.gateway.i18n.messages.Messages;
import org.apache.hadoop.gateway.i18n.messages.StackTrace;
-import org.apache.hadoop.gateway.services.security.AliasServiceException;
import org.apache.hadoop.gateway.services.security.KeystoreServiceException;
-import org.apache.hadoop.gateway.util.urltemplate.Template;
import java.io.File;
import java.net.URI;
@@ -436,4 +434,83 @@ public interface GatewayMessages {
@Message( level = MessageLevel.INFO, text = "Cookie scoping feature enabled: {0}" )
void cookieScopingFeatureEnabled( boolean enabled );
+
+ /**
+ * Log whether Topology port mapping feature is enabled/disabled.
+ *
+ * @param enabled
+ */
+ @Message(level = MessageLevel.INFO,
+ text = "Topology port mapping feature enabled: {0}")
+ void gatewayTopologyPortMappingEnabled(final boolean enabled);
+
+ /**
+ * @param topology
+ * @param port
+ */
+ @Message(level = MessageLevel.DEBUG,
+ text = "Creating a connector for topology {0} listening on port {1}.")
+ void createJettyConnector(final String topology, final int port);
+
+ /**
+ * @param topology
+ */
+ @Message(level = MessageLevel.DEBUG,
+ text = "Creating a handler for topology {0}.")
+ void createJettyHandler(final String topology);
+
+ /**
+ * @param oldTarget
+ * @param newTarget
+ */
+ @Message(level = MessageLevel.INFO,
+ text = "Updating request context from {0} to {1}")
+ void topologyPortMappingAddContext(final String oldTarget,
+ final String newTarget);
+
+ /**
+ * @param oldTarget
+ * @param newTarget
+ */
+ @Message(level = MessageLevel.DEBUG,
+ text = "Updating request target from {0} to {1}")
+ void topologyPortMappingUpdateRequest(final String oldTarget,
+ final String newTarget);
+
+ /**
+ * Messages for Topology Port Mapping
+ *
+ * @param port
+ * @param topology
+ */
+ @Message(level = MessageLevel.ERROR,
+ text = "Port {0} configured for Topology - {1} is already in use.")
+ void portAlreadyInUse(final int port, final String topology);
+
+ /**
+ * Messages for Topology Port Mapping
+ *
+ * @param port
+ */
+ @Message(level = MessageLevel.ERROR,
+ text = "Port {0} is already in use.")
+ void portAlreadyInUse(final int port);
+
+ /**
+ * Log topology and port
+ *
+ * @param topology
+ * @param port
+ */
+ @Message(level = MessageLevel.INFO,
+ text = "Started gateway, topology \"{0}\" listening on port \"{1}\".")
+ void startedGateway(final String topology, final int port);
+
+ @Message(level = MessageLevel.ERROR,
+ text =
+ " Could not find topology \"{0}\" mapped to port \"{1}\" configured in gateway-config.xml. "
+ + "This invalid topology mapping will be ignored by the gateway. "
+ + "Gateway restart will be required if in the future \"{0}\" topology is added.")
+ void topologyPortMappingCannotFindTopology(final String topology,
+ final int port);
}
http://git-wip-us.apache.org/repos/asf/knox/blob/111d8974/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayServer.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayServer.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayServer.java
index d6aa43d..0c4d1ea 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayServer.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayServer.java
@@ -17,39 +17,13 @@
*/
package org.apache.hadoop.gateway;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.Serializable;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.security.cert.CertificateException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.ServiceLoader;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.regex.Pattern;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.TransformerException;
-
import net.lingala.zip4j.core.ZipFile;
import net.lingala.zip4j.exception.ZipException;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.gateway.audit.api.Action;
import org.apache.hadoop.gateway.audit.api.ActionOutcome;
import org.apache.hadoop.gateway.audit.api.AuditServiceFactory;
@@ -62,6 +36,8 @@ import org.apache.hadoop.gateway.deploy.DeploymentException;
import org.apache.hadoop.gateway.deploy.DeploymentFactory;
import org.apache.hadoop.gateway.filter.CorrelationHandler;
import org.apache.hadoop.gateway.filter.DefaultTopologyHandler;
+import org.apache.hadoop.gateway.filter.PortMappingHelperHandler;
+import org.apache.hadoop.gateway.filter.RequestUpdateHandler;
import org.apache.hadoop.gateway.i18n.messages.MessagesFactory;
import org.apache.hadoop.gateway.i18n.resources.ResourcesFactory;
import org.apache.hadoop.gateway.services.GatewayServices;
@@ -80,13 +56,13 @@ import org.apache.hadoop.gateway.util.XmlUtils;
import org.apache.hadoop.gateway.websockets.GatewayWebsocketHandler;
import org.apache.log4j.PropertyConfigurator;
import org.eclipse.jetty.server.Connector;
-import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.NetworkConnector;
import org.eclipse.jetty.server.SecureRequestCustomizer;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.handler.ContextHandler;
import org.eclipse.jetty.server.handler.ContextHandlerCollection;
import org.eclipse.jetty.server.handler.HandlerCollection;
import org.eclipse.jetty.server.handler.RequestLogHandler;
@@ -103,11 +79,43 @@ import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.TransformerException;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Serializable;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.ServiceLoader;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Pattern;
+
public class GatewayServer {
private static final GatewayResources res = ResourcesFactory.get(GatewayResources.class);
private static final GatewayMessages log = MessagesFactory.get(GatewayMessages.class);
private static final Auditor auditor = AuditServiceFactory.getAuditService().getAuditor(AuditConstants.DEFAULT_AUDITOR_NAME,
AuditConstants.KNOX_SERVICE_NAME, AuditConstants.KNOX_COMPONENT_NAME);
+ private static final String DEFAULT_CONNECTOR_NAME = "default";
+
private static GatewayServer server;
private static GatewayServices services;
@@ -295,7 +303,26 @@ public class GatewayServer {
server.start();
// Coverity CID 1352654
URI uri = server.jetty.getURI();
- log.startedGateway( uri != null ? uri.getPort() : -1 );
+
+ // Logging for topology <-> port
+ InetSocketAddress[] addresses = new InetSocketAddress[server.jetty
+ .getConnectors().length];
+ for (int i = 0, n = addresses.length; i < n; i++) {
+ NetworkConnector connector = (NetworkConnector) server.jetty
+ .getConnectors()[i];
+ if (connector != null) {
+
+ if (connector.getName() == null) {
+ log.startedGateway(
+ connector != null ? connector.getLocalPort() : -1);
+ } else {
+ log.startedGateway(connector != null ? connector.getName() : "",
+ connector != null ? connector.getLocalPort() : -1);
+ }
+
+ }
+ }
+
return server;
}
}
@@ -309,13 +336,35 @@ public class GatewayServer {
this.listener = new InternalTopologyListener();
}
- private static Connector createConnector( final Server server, final GatewayConfig config ) throws IOException, CertificateException, NoSuchAlgorithmException, KeyStoreException {
+ /**
+ * Create a connector for Gateway Server to listen on.
+ *
+ * @param server Jetty server
+ * @param config GatewayConfig
+ * @param port If value is > 0 then the given value is used else we
+ * use the port provided in GatewayConfig.
+ * @param topologyName Connector name, only used when not null
+ * @return
+ * @throws IOException
+ * @throws CertificateException
+ * @throws NoSuchAlgorithmException
+ * @throws KeyStoreException
+ */
+ private static Connector createConnector(final Server server,
+ final GatewayConfig config, final int port, final String topologyName)
+ throws IOException, CertificateException, NoSuchAlgorithmException,
+ KeyStoreException {
+
ServerConnector connector;
// Determine the socket address and check availability.
InetSocketAddress address = config.getGatewayAddress();
checkAddressAvailability( address );
+ final int connectorPort = port > 0 ? port : address.getPort();
+
+ checkPortConflict(connectorPort, topologyName, config);
+
HttpConfiguration httpConfig = new HttpConfiguration();
httpConfig.setRequestHeaderSize( config.getHttpServerRequestHeaderBuffer() );
//httpConfig.setRequestBufferSize( config.getHttpServerRequestBuffer() );
@@ -325,7 +374,7 @@ public class GatewayServer {
if (config.isSSLEnabled()) {
HttpConfiguration httpsConfig = new HttpConfiguration( httpConfig );
httpsConfig.setSecureScheme( "https" );
- httpsConfig.setSecurePort( address.getPort() );
+ httpsConfig.setSecurePort( connectorPort );
httpsConfig.addCustomizer( new SecureRequestCustomizer() );
SSLService ssl = services.getService("SSLService");
String keystoreFileName = config.getGatewaySecurityDir() + File.separatorChar + "keystores" + File.separatorChar + "gateway.jks";
@@ -335,7 +384,12 @@ public class GatewayServer {
connector = new ServerConnector( server );
}
connector.setHost( address.getHostName() );
- connector.setPort( address.getPort() );
+ connector.setPort( connectorPort );
+
+ if(!StringUtils.isBlank(topologyName)) {
+ connector.setName(topologyName);
+ }
+
long idleTimeout = config.getGatewayIdleTimeout();
if (idleTimeout > 0l) {
connector.setIdleTimeout(idleTimeout);
@@ -347,9 +401,11 @@ public class GatewayServer {
private static HandlerCollection createHandlers(
final GatewayConfig config,
final GatewayServices services,
- final ContextHandlerCollection contexts ) {
+ final ContextHandlerCollection contexts,
+ final Map<String, Integer> topologyPortMap) {
HandlerCollection handlers = new HandlerCollection();
RequestLogHandler logHandler = new RequestLogHandler();
+
logHandler.setRequestLog( new AccessHandler() );
TraceHandler traceHandler = new TraceHandler();
@@ -369,44 +425,122 @@ public class GatewayServer {
gzipHandler.addIncludedMimeTypes(mimeTypes);
gzipHandler.setHandler(correlationHandler);
+ // Used to correct the {target} part of request with Topology Port Mapping feature
+ final PortMappingHelperHandler portMappingHandler = new PortMappingHelperHandler(config);
+ portMappingHandler.setHandler(gzipHandler);
+
DefaultTopologyHandler defaultTopoHandler = new DefaultTopologyHandler(
config, services, contexts);
+
+ // If topology to port mapping feature is enabled then we add new Handler {RequestForwardHandler}
+ // to the chain, this handler listens on the configured port (in gateway-site.xml)
+ // and simply forwards requests to the correct context path.
+
+ // The reason for adding ContextHandler is so that we can add a connector
+ // to it on which the handler listens (exclusively).
+
+
+ if (config.isGatewayPortMappingEnabled()) {
+
+ for (final Map.Entry<String, Integer> entry : topologyPortMap
+ .entrySet()) {
+ log.createJettyHandler(entry.getKey());
+ final ContextHandler topologyContextHandler = new ContextHandler();
+
+ final RequestUpdateHandler updateHandler = new RequestUpdateHandler(
+ config, entry.getKey(), services);
+
+ topologyContextHandler.setHandler(updateHandler);
+ topologyContextHandler.setVirtualHosts(
+ new String[] { "@" + entry.getKey().toLowerCase() });
+
+ handlers.addHandler(topologyContextHandler);
+ }
+
+ }
+
+ handlers.addHandler(defaultTopoHandler);
+ handlers.addHandler(logHandler);
+
if (config.isWebsocketEnabled()) {
- GatewayWebsocketHandler websockethandler = new GatewayWebsocketHandler(
+ final GatewayWebsocketHandler websocketHandler = new GatewayWebsocketHandler(
config, services);
- websockethandler.setHandler(gzipHandler);
-
- /*
- * Chaining the gzipHandler to correlationHandler. The expected flow here
- * is defaultTopoHandler -> logHandler -> gzipHandler ->
- * correlationHandler -> traceHandler -> websockethandler
- */
- handlers.setHandlers(
- new Handler[] { defaultTopoHandler, logHandler, websockethandler });
+ websocketHandler.setHandler(portMappingHandler);
+
+ handlers.addHandler(websocketHandler);
+
} else {
- /*
- * Chaining the gzipHandler to correlationHandler. The expected flow here
- * is defaultTopoHandler -> logHandler -> gzipHandler ->
- * correlationHandler -> traceHandler
- */
- handlers.setHandlers(
- new Handler[] { defaultTopoHandler, logHandler, gzipHandler });
+ handlers.addHandler(portMappingHandler);
}
return handlers;
}
+ /**
+ * Sanity Check to make sure configured ports are free and there is not port
+ * conflict.
+ *
+ * @param port
+ * @param topologyName
+ * @param config
+ * @throws IOException
+ */
+ public static void checkPortConflict(final int port,
+ final String topologyName, final GatewayConfig config)
+ throws IOException {
+
+ // Throw an exception if port in use
+ if (isPortInUse(port)) {
+ if (topologyName == null) {
+ log.portAlreadyInUse(port);
+ } else {
+ log.portAlreadyInUse(port, topologyName);
+ }
+ throw new IOException(String.format(" Port %d already in use. ", port));
+ }
+
+ // if topology name is blank which means we have all topologies listening on this port
+ if (StringUtils.isBlank(topologyName)) {
+ if (config.getGatewayPortMappings().containsValue(new Integer(port))) {
+ log.portAlreadyInUse(port);
+ throw new IOException(
+ String.format(" Cannot use port %d, it has to be unique. ", port));
+ }
+ } else {
+ // Topology name is not blank so check amongst other ports if we have a conflict
+ for (final Map.Entry<String, Integer> entry : config
+ .getGatewayPortMappings().entrySet()) {
+ if (entry.getKey().equalsIgnoreCase(topologyName)) {
+ continue;
+ }
+
+ if (entry.getValue() == port) {
+ log.portAlreadyInUse(port, topologyName);
+ throw new IOException(String.format(
+ " Topologies %s and %s use the same port %d, ports for topologies (if defined) have to be unique. ",
+ entry.getKey(), topologyName, port));
+ }
+
+ }
+
+ }
+
+ }
+
private synchronized void start() throws Exception {
// Create the global context handler.
contexts = new ContextHandlerCollection();
+
// A map to keep track of current deployments by cluster name.
deployments = new ConcurrentHashMap<String, WebAppContext>();
// Start Jetty.
jetty = new Server( new QueuedThreadPool( config.getThreadPoolMax() ) );
- jetty.addConnector( createConnector( jetty, config ) );
- jetty.setHandler( createHandlers( config, services, contexts ) );
+
+ /* topologyName is null because all topology listen on this port */
+ jetty.addConnector( createConnector( jetty, config, config.getGatewayPort(), null) );
+
// Add Annotations processing into the Jetty server to support JSPs
Configuration.ClassList classlist = Configuration.ClassList.setServerDefault( jetty );
@@ -421,6 +555,40 @@ public class GatewayServer {
monitor.addTopologyChangeListener(listener);
monitor.reloadTopologies();
+ final Collection<Topology> topologies = monitor.getTopologies();
+ final Map<String, Integer> topologyPortMap = config.getGatewayPortMappings();
+
+ // List of all the topology that are deployed
+ final List<String> deployedTopologyList = new ArrayList<String>();
+
+ for (final Topology t : topologies) {
+ deployedTopologyList.add(t.getName());
+ }
+
+
+ // Check whether the configured topologies for port mapping exist, if not
+ // log WARN message and continue
+ checkMappedTopologiesExist(topologyPortMap, deployedTopologyList);
+
+ final HandlerCollection handlers = createHandlers( config, services, contexts, topologyPortMap);
+
+ // Check whether a topology wants dedicated port,
+ // if yes then we create a connector that listens on the provided port.
+
+ log.gatewayTopologyPortMappingEnabled(config.isGatewayPortMappingEnabled());
+ if (config.isGatewayPortMappingEnabled()) {
+ for (Map.Entry<String, Integer> entry : topologyPortMap.entrySet()) {
+ // Add connector for only valid topologies, i.e. deployed topologies.
+ if(deployedTopologyList.contains(entry.getKey())) {
+ log.createJettyConnector(entry.getKey().toLowerCase(), entry.getValue());
+ jetty.addConnector(createConnector(jetty, config, entry.getValue(),
+ entry.getKey().toLowerCase()));
+ }
+ }
+ }
+
+ jetty.setHandler(handlers);
+
try {
jetty.start();
}
@@ -445,6 +613,51 @@ public class GatewayServer {
log.stoppedGateway();
}
+ /**
+ * Check whether a port is free
+ *
+ * @param port
+ * @return true if port in use else false
+ */
+ public static boolean isPortInUse(final int port) {
+
+ Socket socket = null;
+ try {
+ socket = new Socket("localhost", port);
+ return true;
+ } catch (final UnknownHostException e) {
+ return false;
+ } catch (final IOException e) {
+ return false;
+ } finally {
+ IOUtils.closeQuietly(socket);
+ }
+
+ }
+
+ /**
+ * Checks whether the topologies defined in gateway-xml as part of Topology
+ * Port mapping feature exists. If it does not Log a message and move on.
+ *
+ * @param configTopologies
+ * @param topologies
+ * @return
+ */
+ private void checkMappedTopologiesExist(
+ final Map<String, Integer> configTopologies,
+ final List<String> topologies) throws IOException {
+
+ for(final Map.Entry<String, Integer> entry : configTopologies.entrySet()) {
+
+ // If the topologies defined in gateway-config.xml are not found in gateway
+ if (!topologies.contains(entry.getKey())) {
+ log.topologyPortMappingCannotFindTopology(entry.getKey(), entry.getValue());
+ }
+
+ }
+
+ }
+
public URI getURI() {
return jetty.getURI();
}
@@ -487,6 +700,7 @@ public class GatewayServer {
context.setTempDirectory( FileUtils.getFile( warFile, "META-INF", "temp" ) );
context.setErrorHandler( createErrorHandler() );
context.setInitParameter("org.eclipse.jetty.servlet.Default.dirAllowed", "false");
+
return context;
}
@@ -585,12 +799,14 @@ public class GatewayServer {
if( contexts.isRunning() && !newContext.isRunning() ) {
newContext.start();
}
+
} catch( Exception e ) {
auditor.audit( Action.DEPLOY, topology.getName(), ResourceType.TOPOLOGY, ActionOutcome.FAILURE );
log.failedToDeployTopology( topology.getName(), e );
}
}
+
private synchronized void internalDeactivateTopology( Topology topology ) {
log.deactivatingTopology( topology.getName() );
@@ -614,6 +830,7 @@ public class GatewayServer {
}
}
}
+
// Deactivate the required deployed contexts.
for( WebAppContext context : deactivate ) {
String contextPath = context.getContextPath();
http://git-wip-us.apache.org/repos/asf/knox/blob/111d8974/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java
index 368787a..0af105f 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.gateway.config.impl;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.gateway.GatewayMessages;
@@ -33,8 +34,10 @@ import java.net.URL;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
/**
* The configuration for the Gateway.
@@ -150,6 +153,13 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig {
public static final String WEBSOCKET_IDLE_TIMEOUT = GATEWAY_CONFIG_FILE_PREFIX + ".websocket.idle.timeout";
/**
+ * Properties for for gateway port mapping feature
+ */
+ public static final String GATEWAY_PORT_MAPPING_PREFIX = GATEWAY_CONFIG_FILE_PREFIX + ".port.mapping.";
+ public static final String GATEWAY_PORT_MAPPING_REGEX = GATEWAY_CONFIG_FILE_PREFIX + "\\.port\\.mapping\\..*";
+ public static final String GATEWAY_PORT_MAPPING_ENABLED = GATEWAY_PORT_MAPPING_PREFIX + "enabled";
+
+ /**
* Comma seperated list of MIME Types to be compressed by Knox on the way out.
*
* @since 0.12
@@ -183,6 +193,8 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig {
public static final int DEFAULT_WEBSOCKET_ASYNC_WRITE_TIMEOUT = 60000;
public static final int DEFAULT_WEBSOCKET_IDLE_TIMEOUT = 300000;
+ public static final boolean DEFAULT_GATEWAY_PORT_MAPPING_ENABLED = true;
+
/**
* Default list of MIME Type to be compressed.
* @since 0.12
@@ -808,6 +820,41 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig {
return mimeTypes;
}
+ /**
+ * Map of Topology names and their ports.
+ *
+ * @return
+ */
+ @Override
+ public Map<String, Integer> getGatewayPortMappings() {
+
+ final Map<String, Integer> result = new ConcurrentHashMap<String, Integer>();
+ final Map<String, String> properties = getValByRegex(GATEWAY_PORT_MAPPING_REGEX);
+
+ // Convert port no. from string to int
+ for(final Map.Entry<String, String> e : properties.entrySet()) {
+ // ignore the GATEWAY_PORT_MAPPING_ENABLED property
+ if(!e.getKey().equalsIgnoreCase(GATEWAY_PORT_MAPPING_ENABLED)) {
+ // extract the topology name and use it as a key
+ result.put(StringUtils.substringAfter(e.getKey(), GATEWAY_PORT_MAPPING_PREFIX), Integer.parseInt(e.getValue()) );
+ }
+
+ }
+
+ return Collections.unmodifiableMap(result);
+ }
+
+ /**
+ * Is the Port Mapping feature on ?
+ *
+ * @return
+ */
+ @Override
+ public boolean isGatewayPortMappingEnabled() {
+ final String result = get( GATEWAY_PORT_MAPPING_ENABLED, Boolean.toString(DEFAULT_GATEWAY_PORT_MAPPING_ENABLED));
+ return Boolean.parseBoolean(result);
+ }
+
private static long parseNetworkTimeout(String s ) {
PeriodFormatter f = new PeriodFormatterBuilder()
.appendMinutes().appendSuffix("m"," min")
http://git-wip-us.apache.org/repos/asf/knox/blob/111d8974/gateway-server/src/main/java/org/apache/hadoop/gateway/filter/DefaultTopologyHandler.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/filter/DefaultTopologyHandler.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/filter/DefaultTopologyHandler.java
index 707083d..2b91b19 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/filter/DefaultTopologyHandler.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/filter/DefaultTopologyHandler.java
@@ -86,7 +86,7 @@ public class DefaultTopologyHandler extends HandlerWrapper {
}
}
- private static class ForwardedRequest extends HttpServletRequestWrapper {
+ static class ForwardedRequest extends HttpServletRequestWrapper {
private String contextPath;
http://git-wip-us.apache.org/repos/asf/knox/blob/111d8974/gateway-server/src/main/java/org/apache/hadoop/gateway/filter/PortMappingHelperHandler.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/filter/PortMappingHelperHandler.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/filter/PortMappingHelperHandler.java
new file mode 100644
index 0000000..ace4c11
--- /dev/null
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/filter/PortMappingHelperHandler.java
@@ -0,0 +1,98 @@
+package org.apache.hadoop.gateway.filter;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.gateway.GatewayMessages;
+import org.apache.hadoop.gateway.config.GatewayConfig;
+import org.apache.hadoop.gateway.i18n.messages.MessagesFactory;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.handler.HandlerWrapper;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This is a helper handler that adjusts the "target" patch of the request.
+ * Used when Topology Port Mapping feature is used.
+ * See KNOX-928
+ *
+ */
+public class PortMappingHelperHandler extends HandlerWrapper {
+
+ private static final GatewayMessages LOG = MessagesFactory
+ .get(GatewayMessages.class);
+
+ final GatewayConfig config;
+
+ public PortMappingHelperHandler(final GatewayConfig config) {
+
+ this.config = config;
+ }
+
+ @Override
+ public void handle(final String target, final Request baseRequest,
+ final HttpServletRequest request, final HttpServletResponse response)
+ throws IOException, ServletException {
+
+ // If Port Mapping feature enabled
+ if (config.isGatewayPortMappingEnabled()) {
+ int targetIndex;
+ String context = "";
+ String baseURI = baseRequest.getUri().toString();
+
+ // extract the gateway specific part i.e. {/gatewayName/}
+ String originalContextPath = "";
+ targetIndex = StringUtils.ordinalIndexOf(target, "/", 2);
+
+ // Match found e.g. /{string}/
+ if (targetIndex > 0) {
+ originalContextPath = target.substring(0, targetIndex + 1);
+ } else if (targetIndex == -1) {
+ targetIndex = StringUtils.ordinalIndexOf(target, "/", 1);
+ // For cases "/" and "/hive"
+ if(targetIndex == 0) {
+ originalContextPath = target;
+ }
+ }
+
+ // Match "/{gatewayName}/{topologyName/foo" or "/".
+ // There could be a case where content is served from the root
+ // i.e. https://host:port/
+
+ if (!baseURI.startsWith(originalContextPath)) {
+ final int index = StringUtils.ordinalIndexOf(baseURI, "/", 3);
+ if (index > 0) {
+ context = baseURI.substring(0, index);
+ }
+ }
+
+ if(!StringUtils.isBlank(context)) {
+ LOG.topologyPortMappingAddContext(target, context + target);
+ }
+ // Move on to the next handler in chain with updated path
+ super.handle(context + target, baseRequest, request, response);
+ } else {
+ super.handle(target, baseRequest, request, response);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/111d8974/gateway-server/src/main/java/org/apache/hadoop/gateway/filter/RequestUpdateHandler.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/filter/RequestUpdateHandler.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/filter/RequestUpdateHandler.java
new file mode 100644
index 0000000..22dd6b9
--- /dev/null
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/filter/RequestUpdateHandler.java
@@ -0,0 +1,136 @@
+package org.apache.hadoop.gateway.filter;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.gateway.GatewayMessages;
+import org.apache.hadoop.gateway.config.GatewayConfig;
+import org.apache.hadoop.gateway.i18n.messages.MessagesFactory;
+import org.apache.hadoop.gateway.services.GatewayServices;
+import org.eclipse.jetty.http.HttpURI;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.handler.ScopedHandler;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletRequestWrapper;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+
+/**
+ * This handler will be ONLY registered with a specific connector listening on a
+ * port that is configured through a property gateway.port.mapping.{topologyName}
+ * in gateway-site.xml
+ * <p>
+ * The function of this connector is to append the right context path and
+ * forward the request to the default port.
+ * <p>
+ * See KNOX-928
+ *
+ */
+public class RequestUpdateHandler extends ScopedHandler {
+
+ private static final GatewayMessages LOG = MessagesFactory
+ .get(GatewayMessages.class);
+
+ private String redirectContext = null;
+
+ public RequestUpdateHandler(final GatewayConfig config,
+ final String topologyName, final GatewayServices services) {
+ super();
+
+ if (config == null) {
+ throw new IllegalArgumentException("config==null");
+ }
+ if (services == null) {
+ throw new IllegalArgumentException("services==null");
+ }
+ if (topologyName == null) {
+ throw new IllegalArgumentException("topologyName==null");
+ }
+
+ redirectContext = "/" + config.getGatewayPath() + "/" + topologyName;
+
+ }
+
+ @Override
+ public void doScope(final String target, final Request baseRequest,
+ final HttpServletRequest request, final HttpServletResponse response)
+ throws IOException, ServletException {
+ nextScope(target, baseRequest, request, response);
+ }
+
+ @Override
+ public void doHandle(final String target, final Request baseRequest,
+ final HttpServletRequest request, final HttpServletResponse response)
+ throws IOException, ServletException {
+
+ final String newTarget = redirectContext + target;
+
+ RequestUpdateHandler.ForwardedRequest newRequest = new RequestUpdateHandler.ForwardedRequest(
+ request, redirectContext, newTarget);
+
+ // if the request already has the /{gatewaypath}/{topology} part then skip
+ if (!StringUtils.startsWithIgnoreCase(target, redirectContext)) {
+ baseRequest.setPathInfo(redirectContext + baseRequest.getPathInfo());
+ baseRequest.setUri(
+ new HttpURI(redirectContext + baseRequest.getUri().toString()));
+
+ LOG.topologyPortMappingUpdateRequest(target, newTarget);
+ nextHandle(newTarget, baseRequest, newRequest, response);
+ } else {
+ nextHandle(target, baseRequest, newRequest, response);
+ }
+
+ }
+
+ /**
+ * A request wrapper class that wraps a request and adds the context path if
+ * needed.
+ */
+ static class ForwardedRequest extends HttpServletRequestWrapper {
+
+ private String newURL;
+ private String contextpath;
+
+ public ForwardedRequest(final HttpServletRequest request,
+ final String contextpath, final String newURL) {
+ super(request);
+ this.newURL = newURL;
+ this.contextpath = contextpath;
+ }
+
+ @Override
+ public StringBuffer getRequestURL() {
+ return new StringBuffer(newURL);
+ }
+
+ @Override
+ public String getRequestURI() {
+ return contextpath + super.getRequestURI();
+ }
+
+ @Override
+ public String getContextPath() {
+ return this.contextpath;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/111d8974/gateway-server/src/test/java/org/apache/hadoop/gateway/GatewayPortMappingConfigTest.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/test/java/org/apache/hadoop/gateway/GatewayPortMappingConfigTest.java b/gateway-server/src/test/java/org/apache/hadoop/gateway/GatewayPortMappingConfigTest.java
new file mode 100644
index 0000000..5b439d1
--- /dev/null
+++ b/gateway-server/src/test/java/org/apache/hadoop/gateway/GatewayPortMappingConfigTest.java
@@ -0,0 +1,204 @@
+package org.apache.hadoop.gateway;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.hadoop.gateway.config.GatewayConfig;
+import org.apache.hadoop.gateway.services.DefaultGatewayServices;
+import org.apache.hadoop.gateway.services.topology.TopologyService;
+import org.apache.velocity.VelocityContext;
+import org.apache.velocity.app.VelocityEngine;
+import org.easymock.EasyMock;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.handler.ContextHandler;
+import org.eclipse.jetty.server.handler.HandlerCollection;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.greaterThan;
+
+/**
+ * Test the Gateway Topology Port Mapping config
+ *
+ */
+public class GatewayPortMappingConfigTest {
+
+ /**
+ * Mock gateway config
+ */
+ private static GatewayConfig gatewayConfig;
+
+ private static int eeriePort;
+ private static int ontarioPort;
+ private static int huronPort;
+
+ private static int defaultPort;
+
+ private static DefaultGatewayServices services;
+ private static TopologyService topos;
+
+ private static VelocityEngine velocity;
+ private static VelocityContext context;
+
+ private static Server gatewayServer;
+
+ private static Properties params;
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ public GatewayPortMappingConfigTest() {
+ super();
+ }
+
+ @BeforeClass
+ public static void init() throws Exception {
+
+ Map<String, Integer> topologyPortMapping = new ConcurrentHashMap<String, Integer>();
+
+ // get unique ports
+ eeriePort = getAvailablePort(1240, 49151);
+ ontarioPort = getAvailablePort(eeriePort + 1, 49151);
+ huronPort = getAvailablePort(ontarioPort + 1, 49151);
+
+ defaultPort = getAvailablePort(huronPort + 1, 49151);
+
+ topologyPortMapping.put("eerie", eeriePort);
+ topologyPortMapping.put("ontario", ontarioPort);
+ topologyPortMapping.put("huron", huronPort);
+
+ gatewayConfig = EasyMock.createNiceMock(GatewayConfig.class);
+ EasyMock.expect(gatewayConfig.getGatewayPortMappings())
+ .andReturn(topologyPortMapping).anyTimes();
+
+ EasyMock.expect(gatewayConfig.getGatewayPort()).andReturn(defaultPort)
+ .anyTimes();
+
+ EasyMock.replay(gatewayConfig);
+
+ // Start gateway to check port conflicts
+ startGatewayServer();
+
+ }
+
+ @AfterClass
+ public static void stopServers() {
+ try {
+ gatewayServer.stop();
+ } catch (final Exception e) {
+ e.printStackTrace(System.err);
+ }
+ }
+
+ /**
+ * This utility method will return the next available port
+ * that can be used.
+ *
+ * @return Port that is available.
+ */
+ public static int getAvailablePort(final int min, final int max) {
+
+ for (int i = min; i <= max; i++) {
+
+ if (!GatewayServer.isPortInUse(i)) {
+ return i;
+ }
+ }
+ // too bad
+ return -1;
+ }
+
+
+
+ /**
+ * This method simply tests the configs
+ */
+ @Test
+ public void testGatewayConfig() {
+ assertThat(gatewayConfig.getGatewayPortMappings().get("eerie"),
+ greaterThan(-1));
+ assertThat(gatewayConfig.getGatewayPortMappings().get("ontario"),
+ greaterThan(-1));
+ assertThat(gatewayConfig.getGatewayPortMappings().get("huron"),
+ greaterThan(-1));
+ }
+
+ /**
+ * Test case where topologies "eerie" and "huron" use same ports.
+ */
+ @Test
+ public void testCheckPortConflict() throws IOException {
+ /* Check port conflict with default port */
+ exception.expect(IOException.class);
+ exception.expectMessage(String.format(
+ " Topologies %s and %s use the same port %d, ports for topologies (if defined) have to be unique. ",
+ "huron", "eerie", huronPort));
+
+ GatewayServer.checkPortConflict(huronPort, "eerie", gatewayConfig);
+
+ }
+
+ /**
+ * Test a case where gateway is already running and same port is used to start
+ * another gateway.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testDefaultPortInUse() throws IOException {
+
+ exception.expect(IOException.class);
+ exception
+ .expectMessage(String.format("Port %d already in use.", defaultPort));
+
+ GatewayServer.checkPortConflict(defaultPort, null, gatewayConfig);
+
+ }
+
+ private static void startGatewayServer() throws Exception {
+ // use default Max threads
+ gatewayServer = new Server(defaultPort);
+ final ServerConnector connector = new ServerConnector(gatewayServer);
+ gatewayServer.addConnector(connector);
+
+ // workaround so we can add our handler later at runtime
+ HandlerCollection handlers = new HandlerCollection(true);
+
+ // add some initial handlers
+ ContextHandler context = new ContextHandler();
+ context.setContextPath("/");
+ handlers.addHandler(context);
+
+ gatewayServer.setHandler(handlers);
+
+ // Start Server
+ gatewayServer.start();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/111d8974/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java
----------------------------------------------------------------------
diff --git a/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java b/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java
index af03bbd..ed868b1 100644
--- a/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java
+++ b/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.gateway.config;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.List;
+import java.util.Map;
public interface GatewayConfig {
@@ -278,4 +279,18 @@ public interface GatewayConfig {
*/
String getKeyLength();
+ /**
+ * Map of Topology names and their ports.
+ *
+ * @return
+ */
+ Map<String, Integer> getGatewayPortMappings();
+
+ /**
+ * Is the Port Mapping feature on ?
+ * @return
+ */
+ boolean isGatewayPortMappingEnabled();
+
+
}
http://git-wip-us.apache.org/repos/asf/knox/blob/111d8974/gateway-test-release-utils/src/main/java/org/apache/hadoop/gateway/GatewayTestConfig.java
----------------------------------------------------------------------
diff --git a/gateway-test-release-utils/src/main/java/org/apache/hadoop/gateway/GatewayTestConfig.java b/gateway-test-release-utils/src/main/java/org/apache/hadoop/gateway/GatewayTestConfig.java
index f41046b..e703919 100644
--- a/gateway-test-release-utils/src/main/java/org/apache/hadoop/gateway/GatewayTestConfig.java
+++ b/gateway-test-release-utils/src/main/java/org/apache/hadoop/gateway/GatewayTestConfig.java
@@ -25,6 +25,8 @@ import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
public class GatewayTestConfig extends Configuration implements GatewayConfig {
@@ -438,6 +440,26 @@ public class GatewayTestConfig extends Configuration implements GatewayConfig {
return new ArrayList<String>();
}
+ /**
+ * Map of Topology names and their ports.
+ *
+ * @return
+ */
+ @Override
+ public Map<String, Integer> getGatewayPortMappings() {
+ return new ConcurrentHashMap<String, Integer>();
+ }
+
+ /**
+ * Is the Port Mapping feature on ?
+ *
+ * @return
+ */
+ @Override
+ public boolean isGatewayPortMappingEnabled() {
+ return true;
+ }
+
@Override
public boolean isMetricsEnabled() {
return false;
http://git-wip-us.apache.org/repos/asf/knox/blob/111d8974/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayPortMappingDisableFeatureTest.java
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayPortMappingDisableFeatureTest.java b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayPortMappingDisableFeatureTest.java
new file mode 100644
index 0000000..248be57
--- /dev/null
+++ b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayPortMappingDisableFeatureTest.java
@@ -0,0 +1,252 @@
+package org.apache.hadoop.gateway;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.mycila.xmltool.XMLDoc;
+import com.mycila.xmltool.XMLTag;
+import org.apache.hadoop.test.TestUtils;
+import org.apache.hadoop.test.category.ReleaseTest;
+import org.apache.hadoop.test.mock.MockServer;
+import org.apache.http.HttpStatus;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.ExpectedException;
+
+import java.io.IOException;
+import java.net.ConnectException;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static com.jayway.restassured.RestAssured.given;
+import static org.apache.hadoop.test.TestUtils.LOG_ENTER;
+import static org.apache.hadoop.test.TestUtils.LOG_EXIT;
+import static org.hamcrest.CoreMatchers.is;
+
+/**
+ * Test that the Gateway Topology Port Mapping feature is disabled properly.
+ *
+ */
+@Category(ReleaseTest.class)
+public class GatewayPortMappingDisableFeatureTest {
+
+ // Specifies if the test requests should go through the gateway or directly to the services.
+ // This is frequently used to verify the behavior of the test both with and without the gateway.
+ private static final boolean USE_GATEWAY = true;
+
+ // Specifies if the test requests should be sent to mock services or the real services.
+ // This is frequently used to verify the behavior of the test both with and without mock services.
+ private static final boolean USE_MOCK_SERVICES = true;
+
+ private static GatewayFuncTestDriver driver = new GatewayFuncTestDriver();
+
+ private static MockServer masterServer;
+
+ private int eeriePort;
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+
+ public GatewayPortMappingDisableFeatureTest() {
+ super();
+ }
+
+ /**
+ * Creates a deployment of a gateway instance that all test methods will share. This method also creates a
+ * registry of sorts for all of the services that will be used by the test methods.
+ * The createTopology method is used to create the topology file that would normally be read from disk.
+ * The driver.setupGateway invocation is where the creation of GATEWAY_HOME occurs.
+ * <p/>
+ * This would normally be done once for this suite but the failure tests start affecting each other depending
+ * on the state the last 'active' url
+ *
+ * @throws Exception Thrown if any failure occurs.
+ */
+ @Before
+ public void setup() throws Exception {
+ LOG_ENTER();
+
+ eeriePort = getAvailablePort(1240, 49151);
+
+ ConcurrentHashMap<String, Integer> topologyPortMapping = new ConcurrentHashMap<String, Integer>();
+ topologyPortMapping.put("eerie", eeriePort);
+
+ masterServer = new MockServer("master", true);
+ GatewayTestConfig config = new GatewayTestConfig();
+ config.setGatewayPath("gateway");
+ config.setTopologyPortMapping(topologyPortMapping);
+ // disable the feature
+ config.setGatewayPortMappingEnabled(false);
+
+ driver.setResourceBase(WebHdfsHaFuncTest.class);
+ driver.setupLdap(0);
+
+ driver.setupService("WEBHDFS", "http://vm.local:50070/webhdfs", "/eerie/webhdfs", USE_MOCK_SERVICES);
+
+ driver.setupGateway(config, "eerie", createTopology("WEBHDFS"), USE_GATEWAY);
+
+ LOG_EXIT();
+ }
+
+ @After
+ public void cleanup() throws Exception {
+ LOG_ENTER();
+ driver.cleanup();
+ driver.reset();
+ masterServer.reset();
+ LOG_EXIT();
+ }
+
+ /**
+ * Test the standard case
+ *
+ * @throws IOException
+ */
+ @Test(timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testBasicListOperation() throws IOException {
+ LOG_ENTER();
+ test(driver.getUrl("WEBHDFS") );
+ LOG_EXIT();
+ }
+
+ /**
+ * Test the multi port fail scenario when the feature is disabled.
+ * @throws IOException
+ */
+ @Test(timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testMultiPortFailOperation() throws IOException {
+ LOG_ENTER();
+ exception.expect(ConnectException.class);
+ exception.expectMessage("Connection refused");
+
+ test("http://localhost:" + eeriePort + "/webhdfs" );
+ LOG_EXIT();
+ }
+
+
+ private void test (final String url) throws IOException {
+ String password = "hdfs-password";
+ String username = "hdfs";
+
+ masterServer.expect()
+ .method("GET")
+ .pathInfo("/webhdfs/v1/")
+ .queryParam("op", "LISTSTATUS")
+ .queryParam("user.name", username)
+ .respond()
+ .status(HttpStatus.SC_OK)
+ .content(driver.getResourceBytes("webhdfs-liststatus-success.json"))
+ .contentType("application/json");
+
+ given()
+ .auth().preemptive().basic(username, password)
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .queryParam("op", "LISTSTATUS")
+ .expect()
+ .log().ifError()
+ .statusCode(HttpStatus.SC_OK)
+ .content("FileStatuses.FileStatus[0].pathSuffix", is("app-logs"))
+ .when().get(url + "/v1/");
+ masterServer.isEmpty();
+ }
+
+
+ /**
+ * Creates a topology that is deployed to the gateway instance for the test suite.
+ * Note that this topology is shared by all of the test methods in this suite.
+ *
+ * @return A populated XML structure for a topology file.
+ */
+ private static XMLTag createTopology(final String role) {
+ XMLTag xml = XMLDoc.newDocument(true)
+ .addRoot("topology")
+ .addTag("gateway")
+ .addTag("provider")
+ .addTag("role").addText("webappsec")
+ .addTag("name").addText("WebAppSec")
+ .addTag("enabled").addText("true")
+ .addTag("param")
+ .addTag("name").addText("csrf.enabled")
+ .addTag("value").addText("true").gotoParent().gotoParent()
+ .addTag("provider")
+ .addTag("role").addText("authentication")
+ .addTag("name").addText("ShiroProvider")
+ .addTag("enabled").addText("true")
+ .addTag("param")
+ .addTag("name").addText("main.ldapRealm")
+ .addTag("value").addText("org.apache.hadoop.gateway.shirorealm.KnoxLdapRealm").gotoParent()
+ .addTag("param")
+ .addTag("name").addText("main.ldapRealm.userDnTemplate")
+ .addTag("value").addText("uid={0},ou=people,dc=hadoop,dc=apache,dc=org").gotoParent()
+ .addTag("param")
+ .addTag("name").addText("main.ldapRealm.contextFactory.url")
+ .addTag("value").addText(driver.getLdapUrl()).gotoParent()
+ .addTag("param")
+ .addTag("name").addText("main.ldapRealm.contextFactory.authenticationMechanism")
+ .addTag("value").addText("simple").gotoParent()
+ .addTag("param")
+ .addTag("name").addText("urls./**")
+ .addTag("value").addText("authcBasic").gotoParent().gotoParent()
+ .addTag("provider")
+ .addTag("role").addText("identity-assertion")
+ .addTag("enabled").addText("true")
+ .addTag("name").addText("Default").gotoParent()
+ .addTag("provider")
+ .addTag("role").addText("authorization")
+ .addTag("enabled").addText("true")
+ .addTag("name").addText("AclsAuthz").gotoParent()
+ .addTag("param")
+ .addTag("name").addText("webhdfs-acl")
+ .addTag("value").addText("hdfs;*;*").gotoParent()
+ .addTag("provider")
+ .addTag("role").addText("ha")
+ .addTag("enabled").addText("true")
+ .addTag("name").addText("HaProvider")
+ .addTag("param")
+ .addTag("name").addText("WEBHDFS")
+ .addTag("value").addText("maxFailoverAttempts=3;failoverSleep=15;maxRetryAttempts=3;retrySleep=10;enabled=true").gotoParent()
+ .gotoRoot()
+ .addTag("service")
+ .addTag("role").addText(role)
+ .addTag("url").addText("http://localhost:" + masterServer.getPort() + "/webhdfs")
+ .gotoRoot();
+ return xml;
+ }
+
+ /**
+ * This utility method will return the next available port
+ * that can be used.
+ *
+ * @return Port that is available.
+ */
+ public static int getAvailablePort(final int min, final int max) {
+
+ for (int i = min; i <= max; i++) {
+
+ if (!GatewayServer.isPortInUse(i)) {
+ return i;
+ }
+ }
+ // too bad
+ return -1;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/111d8974/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayPortMappingFuncTest.java
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayPortMappingFuncTest.java b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayPortMappingFuncTest.java
new file mode 100644
index 0000000..41e583b
--- /dev/null
+++ b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayPortMappingFuncTest.java
@@ -0,0 +1,298 @@
+package org.apache.hadoop.gateway;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.mycila.xmltool.XMLDoc;
+import com.mycila.xmltool.XMLTag;
+import org.apache.hadoop.test.TestUtils;
+import org.apache.hadoop.test.category.ReleaseTest;
+import org.apache.hadoop.test.mock.MockServer;
+import org.apache.http.HttpStatus;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static com.jayway.restassured.RestAssured.given;
+import static org.apache.hadoop.test.TestUtils.LOG_ENTER;
+import static org.apache.hadoop.test.TestUtils.LOG_EXIT;
+import static org.hamcrest.CoreMatchers.is;
+
+/**
+ * Test the Gateway Topology Port Mapping functionality
+ *
+ */
+@Category(ReleaseTest.class)
+public class GatewayPortMappingFuncTest {
+
+ // Specifies if the test requests should go through the gateway or directly to the services.
+ // This is frequently used to verify the behavior of the test both with and without the gateway.
+ private static final boolean USE_GATEWAY = true;
+
+ // Specifies if the test requests should be sent to mock services or the real services.
+ // This is frequently used to verify the behavior of the test both with and without mock services.
+ private static final boolean USE_MOCK_SERVICES = true;
+
+ private static GatewayFuncTestDriver driver = new GatewayFuncTestDriver();
+
+ private static MockServer masterServer;
+
+ private int eeriePort;
+
+ public GatewayPortMappingFuncTest() {
+ super();
+ }
+
+ /**
+ * Creates a deployment of a gateway instance that all test methods will share. This method also creates a
+ * registry of sorts for all of the services that will be used by the test methods.
+ * The createTopology method is used to create the topology file that would normally be read from disk.
+ * The driver.setupGateway invocation is where the creation of GATEWAY_HOME occurs.
+ * <p/>
+ * This would normally be done once for this suite but the failure tests start affecting each other depending
+ * on the state the last 'active' url
+ *
+ * @throws Exception Thrown if any failure occurs.
+ */
+ @Before
+ public void setup() throws Exception {
+ LOG_ENTER();
+
+ eeriePort = getAvailablePort(1240, 49151);
+
+ ConcurrentHashMap<String, Integer> topologyPortMapping = new ConcurrentHashMap<String, Integer>();
+ topologyPortMapping.put("eerie", eeriePort);
+
+ masterServer = new MockServer("master", true);
+ GatewayTestConfig config = new GatewayTestConfig();
+ config.setGatewayPath("gateway");
+ config.setTopologyPortMapping(topologyPortMapping);
+
+ driver.setResourceBase(WebHdfsHaFuncTest.class);
+ driver.setupLdap(0);
+
+ driver.setupService("WEBHDFS", "http://vm.local:50070/webhdfs", "/eerie/webhdfs", USE_MOCK_SERVICES);
+
+ driver.setupGateway(config, "eerie", createTopology("WEBHDFS"), USE_GATEWAY);
+
+ LOG_EXIT();
+ }
+
+ @After
+ public void cleanup() throws Exception {
+ LOG_ENTER();
+ driver.cleanup();
+ driver.reset();
+ masterServer.reset();
+ LOG_EXIT();
+ }
+
+ /**
+ * Test the standard case:
+ * http://localhost:{gatewayPort}/gateway/eerie/webhdfs/v1
+ *
+ * @throws IOException
+ */
+ @Test(timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testBasicListOperation() throws IOException {
+ LOG_ENTER();
+ test("http://localhost:" + driver.getGatewayPort() + "/gateway/eerie" + "/webhdfs" );
+ LOG_EXIT();
+ }
+
+ /**
+ * Test the multi port scenario.
+ *
+ * http://localhost:{eeriePort}/webhdfs/v1
+ *
+ * @throws IOException
+ */
+ @Test(timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testMultiPortOperation() throws IOException {
+ LOG_ENTER();
+ test("http://localhost:" + eeriePort + "/webhdfs" );
+ LOG_EXIT();
+ }
+
+ /**
+ * Test the multi port scenario when gateway path is included.
+ *
+ * http://localhost:{eeriePort}/gateway/eerie/webhdfs/v1
+ *
+ * @throws IOException
+ */
+ @Test(timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testMultiPortWithGatewayPath() throws IOException {
+ LOG_ENTER();
+ test("http://localhost:" + eeriePort + "/gateway/eerie" + "/webhdfs" );
+ LOG_EXIT();
+ }
+
+ /**
+ * Fail when trying to use this feature on the standard port.
+ *
+ * http://localhost:{gatewayPort}/webhdfs/v1
+ *
+ * @throws IOException
+ */
+ @Test(timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testMultiPortOperationFail() throws IOException {
+ LOG_ENTER();
+ final String url = "http://localhost:" + driver.getGatewayPort() + "/webhdfs" ;
+
+ String password = "hdfs-password";
+ String username = "hdfs";
+
+ masterServer.expect()
+ .method("GET")
+ .pathInfo("/webhdfs/v1/")
+ .queryParam("op", "LISTSTATUS")
+ .queryParam("user.name", username)
+ .respond()
+ .status(HttpStatus.SC_OK)
+ .content(driver.getResourceBytes("webhdfs-liststatus-success.json"))
+ .contentType("application/json");
+
+ given()
+ .auth().preemptive().basic(username, password)
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .queryParam("op", "LISTSTATUS")
+ .expect()
+ //.log().ifError()
+ .statusCode(HttpStatus.SC_NOT_FOUND)
+ //.content("FileStatuses.FileStatus[0].pathSuffix", is("app-logs"))
+ .when().get(url + "/v1/");
+ masterServer.isEmpty();
+
+ LOG_EXIT();
+ }
+
+
+ private void test (final String url) throws IOException {
+ String password = "hdfs-password";
+ String username = "hdfs";
+
+ masterServer.expect()
+ .method("GET")
+ .pathInfo("/webhdfs/v1/")
+ .queryParam("op", "LISTSTATUS")
+ .queryParam("user.name", username)
+ .respond()
+ .status(HttpStatus.SC_OK)
+ .content(driver.getResourceBytes("webhdfs-liststatus-success.json"))
+ .contentType("application/json");
+
+ given()
+ .auth().preemptive().basic(username, password)
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .queryParam("op", "LISTSTATUS")
+ .expect()
+ .log().ifError()
+ .statusCode(HttpStatus.SC_OK)
+ .content("FileStatuses.FileStatus[0].pathSuffix", is("app-logs"))
+ .when().get(url + "/v1/");
+ masterServer.isEmpty();
+ }
+
+
+ /**
+ * Creates a topology that is deployed to the gateway instance for the test suite.
+ * Note that this topology is shared by all of the test methods in this suite.
+ *
+ * @return A populated XML structure for a topology file.
+ */
+ private static XMLTag createTopology(final String role) {
+ XMLTag xml = XMLDoc.newDocument(true)
+ .addRoot("topology")
+ .addTag("gateway")
+ .addTag("provider")
+ .addTag("role").addText("webappsec")
+ .addTag("name").addText("WebAppSec")
+ .addTag("enabled").addText("true")
+ .addTag("param")
+ .addTag("name").addText("csrf.enabled")
+ .addTag("value").addText("true").gotoParent().gotoParent()
+ .addTag("provider")
+ .addTag("role").addText("authentication")
+ .addTag("name").addText("ShiroProvider")
+ .addTag("enabled").addText("true")
+ .addTag("param")
+ .addTag("name").addText("main.ldapRealm")
+ .addTag("value").addText("org.apache.hadoop.gateway.shirorealm.KnoxLdapRealm").gotoParent()
+ .addTag("param")
+ .addTag("name").addText("main.ldapRealm.userDnTemplate")
+ .addTag("value").addText("uid={0},ou=people,dc=hadoop,dc=apache,dc=org").gotoParent()
+ .addTag("param")
+ .addTag("name").addText("main.ldapRealm.contextFactory.url")
+ .addTag("value").addText(driver.getLdapUrl()).gotoParent()
+ .addTag("param")
+ .addTag("name").addText("main.ldapRealm.contextFactory.authenticationMechanism")
+ .addTag("value").addText("simple").gotoParent()
+ .addTag("param")
+ .addTag("name").addText("urls./**")
+ .addTag("value").addText("authcBasic").gotoParent().gotoParent()
+ .addTag("provider")
+ .addTag("role").addText("identity-assertion")
+ .addTag("enabled").addText("true")
+ .addTag("name").addText("Default").gotoParent()
+ .addTag("provider")
+ .addTag("role").addText("authorization")
+ .addTag("enabled").addText("true")
+ .addTag("name").addText("AclsAuthz").gotoParent()
+ .addTag("param")
+ .addTag("name").addText("webhdfs-acl")
+ .addTag("value").addText("hdfs;*;*").gotoParent()
+ .addTag("provider")
+ .addTag("role").addText("ha")
+ .addTag("enabled").addText("true")
+ .addTag("name").addText("HaProvider")
+ .addTag("param")
+ .addTag("name").addText("WEBHDFS")
+ .addTag("value").addText("maxFailoverAttempts=3;failoverSleep=15;maxRetryAttempts=3;retrySleep=10;enabled=true").gotoParent()
+ .gotoRoot()
+ .addTag("service")
+ .addTag("role").addText(role)
+ .addTag("url").addText("http://localhost:" + masterServer.getPort() + "/webhdfs")
+ .gotoRoot();
+ return xml;
+ }
+
+ /**
+ * This utility method will return the next available port
+ * that can be used.
+ *
+ * @return Port that is available.
+ */
+ public static int getAvailablePort(final int min, final int max) {
+
+ for (int i = min; i <= max; i++) {
+
+ if (!GatewayServer.isPortInUse(i)) {
+ return i;
+ }
+ }
+ // too bad
+ return -1;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/111d8974/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
index 580d875..2ee6f1b 100644
--- a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
+++ b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
@@ -24,6 +24,8 @@ import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
public class GatewayTestConfig extends Configuration implements GatewayConfig {
@@ -56,6 +58,8 @@ public class GatewayTestConfig extends Configuration implements GatewayConfig {
private boolean sslEnabled = false;
private String truststoreType = "jks";
private String keystoreType = "jks";
+ private boolean isTopologyPortMappingEnabled = true;
+ private ConcurrentHashMap topologyPortMapping = new ConcurrentHashMap<String, Integer>();
public void setGatewayHomeDir( String gatewayHomeDir ) {
this.gatewayHomeDir = gatewayHomeDir;
@@ -385,6 +389,15 @@ public class GatewayTestConfig extends Configuration implements GatewayConfig {
return backupVersionLimit;
}
+ public void setTopologyPortMapping(ConcurrentHashMap topologyPortMapping) {
+ this.topologyPortMapping = topologyPortMapping;
+ }
+
+ public void setGatewayPortMappingEnabled(
+ boolean topologyPortMappingEnabled) {
+ isTopologyPortMappingEnabled = topologyPortMappingEnabled;
+ }
+
private long backupAgeLimit = -1;
@Override
@@ -565,4 +578,25 @@ public class GatewayTestConfig extends Configuration implements GatewayConfig {
public String getKeyLength() {
return null;
}
+
+ /**
+ * Map of Topology names and their ports.
+ *
+ * @return
+ */
+ @Override
+ public Map<String, Integer> getGatewayPortMappings() {
+ return topologyPortMapping;
+ }
+
+ /**
+ * Is the Port Mapping feature on ?
+ *
+ * @return
+ */
+ @Override
+ public boolean isGatewayPortMappingEnabled() {
+ return isTopologyPortMappingEnabled;
+ }
+
}