You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@knox.apache.org by su...@apache.org on 2015/09/14 06:35:53 UTC
knox git commit: KNOX-570 added zookeeper lookup capability for HS2 HA
Repository: knox
Updated Branches:
refs/heads/master ccfe33d5d -> 31bb1e029
KNOX-570 added zookeeper lookup capability for HS2 HA
Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/31bb1e02
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/31bb1e02
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/31bb1e02
Branch: refs/heads/master
Commit: 31bb1e029f7a4d89e0f6bb2846e2464babbaa858
Parents: ccfe33d
Author: Sumit Gupta <su...@apache.org>
Authored: Mon Sep 14 00:35:18 2015 -0400
Committer: Sumit Gupta <su...@apache.org>
Committed: Mon Sep 14 00:35:18 2015 -0400
----------------------------------------------------------------------
gateway-provider-ha/pom.xml | 16 ++-
.../gateway/ha/dispatch/DefaultHaDispatch.java | 4 +-
.../gateway/ha/provider/HaServiceConfig.java | 8 ++
.../hadoop/gateway/ha/provider/URLManager.java | 35 +++++
.../gateway/ha/provider/URLManagerLoader.java | 43 ++++++
.../ha/provider/impl/DefaultHaDescriptor.java | 2 +-
.../ha/provider/impl/DefaultHaProvider.java | 89 ++++++------
.../provider/impl/DefaultHaServiceConfig.java | 139 ++++++++++--------
.../ha/provider/impl/DefaultURLManager.java | 85 +++++++++++
.../provider/impl/HS2ZookeeperURLManager.java | 144 +++++++++++++++++++
.../ha/provider/impl/HaDescriptorConstants.java | 4 +
.../ha/provider/impl/HaDescriptorFactory.java | 11 +-
.../ha/provider/impl/HaDescriptorManager.java | 10 +-
.../provider/impl/HaServiceConfigConstants.java | 4 +
.../gateway/ha/provider/impl/URLManager.java | 70 ---------
.../ha/provider/impl/i18n/HaMessages.java | 3 +
...apache.hadoop.gateway.ha.provider.URLManager | 19 +++
.../ha/dispatch/DefaultHaDispatchTest.java | 2 +-
.../ha/provider/impl/DefaultURLManagerTest.java | 73 ++++++++++
.../impl/HS2ZookeeperURLManagerTest.java | 135 +++++++++++++++++
.../provider/impl/HaDescriptorFactoryTest.java | 2 +-
.../provider/impl/HaDescriptorManagerTest.java | 6 +-
.../ha/provider/impl/MockURLManager.java | 40 ++++++
.../ha/provider/impl/URLManagerLoaderTest.java | 48 +++++++
.../ha/provider/impl/URLManagerTest.java | 71 ---------
...apache.hadoop.gateway.ha.provider.URLManager | 19 +++
.../resources/services/hive/0.13.0/service.xml | 2 +-
gateway-service-hive/pom.xml | 4 +
.../hadoop/gateway/hive/HiveDispatch.java | 20 +--
.../hadoop/gateway/hive/HiveDispatchUtils.java | 38 +++++
.../hadoop/gateway/hive/HiveHaDispatch.java | 47 ++++++
.../hdfs/dispatch/WebHdfsHaDispatchTest.java | 2 +-
.../dispatch/DefaultHttpClientFactory.java | 4 +-
pom.xml | 18 +++
34 files changed, 943 insertions(+), 274 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/pom.xml
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/pom.xml b/gateway-provider-ha/pom.xml
index 07c74f8..7dcad76 100644
--- a/gateway-provider-ha/pom.xml
+++ b/gateway-provider-ha/pom.xml
@@ -46,6 +46,20 @@
<groupId>${gateway-group}</groupId>
<artifactId>gateway-spi</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.easymock</groupId>
@@ -88,4 +102,4 @@
</dependencies>
-</project>
\ No newline at end of file
+</project>
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/dispatch/DefaultHaDispatch.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/dispatch/DefaultHaDispatch.java b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/dispatch/DefaultHaDispatch.java
index 03aa369..82db972 100644
--- a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/dispatch/DefaultHaDispatch.java
+++ b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/dispatch/DefaultHaDispatch.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.gateway.ha.dispatch;
import org.apache.hadoop.gateway.config.Configure;
+import org.apache.hadoop.gateway.config.Optional;
import org.apache.hadoop.gateway.dispatch.DefaultDispatch;
import org.apache.hadoop.gateway.filter.AbstractGatewayFilter;
import org.apache.hadoop.gateway.ha.dispatch.i18n.HaDispatchMessages;
@@ -50,6 +51,8 @@ public class DefaultHaDispatch extends DefaultDispatch {
private HaProvider haProvider;
+ @Optional
+ @Configure
private String serviceRole;
@Override
@@ -67,7 +70,6 @@ public class DefaultHaDispatch extends DefaultDispatch {
return serviceRole;
}
- @Configure
public void setServiceRole(String serviceRole) {
this.serviceRole = serviceRole;
}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/HaServiceConfig.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/HaServiceConfig.java b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/HaServiceConfig.java
index aa116fd..8c12b8e 100644
--- a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/HaServiceConfig.java
+++ b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/HaServiceConfig.java
@@ -42,4 +42,12 @@ public interface HaServiceConfig {
public void setRetrySleep(int sleep);
public int getRetrySleep();
+
+ public String getZookeeperEnsemble();
+
+ public void setZookeeperEnsemble(String zookeeperEnsemble);
+
+ public String getZookeeperNamespace();
+
+ public void setZookeeperNamespace(String zookeeperNamespace);
}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/URLManager.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/URLManager.java b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/URLManager.java
new file mode 100644
index 0000000..27da958
--- /dev/null
+++ b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/URLManager.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.ha.provider;
+
+import java.util.List;
+
+public interface URLManager {
+
+ public boolean supportsConfig(HaServiceConfig config);
+
+ public void setConfig(HaServiceConfig config);
+
+ public String getActiveURL();
+
+ public List<String> getURLs();
+
+ public void setURLs(List<String> urls);
+
+ public void markFailed(String url);
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/URLManagerLoader.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/URLManagerLoader.java b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/URLManagerLoader.java
new file mode 100644
index 0000000..c92d0e5
--- /dev/null
+++ b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/URLManagerLoader.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.ha.provider;
+
+import org.apache.hadoop.gateway.ha.provider.impl.DefaultURLManager;
+
+import java.util.Iterator;
+import java.util.ServiceLoader;
+
+public class URLManagerLoader {
+
+ public static URLManager loadURLManager(HaServiceConfig config) {
+ if (config != null) {
+ ServiceLoader<URLManager> loader = ServiceLoader.load(URLManager.class);
+ if ( loader != null ) {
+ Iterator<URLManager> iterator = loader.iterator();
+ while ( iterator.hasNext() ) {
+ URLManager urlManager = iterator.next();
+ if ( urlManager.supportsConfig(config) ) {
+ urlManager.setConfig(config);
+ return urlManager;
+ }
+ }
+ }
+ }
+ return new DefaultURLManager();
+ }
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultHaDescriptor.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultHaDescriptor.java b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultHaDescriptor.java
index a551bfe..7969439 100644
--- a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultHaDescriptor.java
+++ b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultHaDescriptor.java
@@ -31,7 +31,7 @@ public class DefaultHaDescriptor implements HaDescriptor {
private ConcurrentHashMap<String, HaServiceConfig> serviceConfigs;
public DefaultHaDescriptor() {
- serviceConfigs = new ConcurrentHashMap<String, HaServiceConfig>();
+ serviceConfigs = new ConcurrentHashMap<>();
}
@Override
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultHaProvider.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultHaProvider.java b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultHaProvider.java
index 880707a..302275e 100644
--- a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultHaProvider.java
+++ b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultHaProvider.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.gateway.ha.provider.impl;
import org.apache.hadoop.gateway.ha.provider.HaDescriptor;
import org.apache.hadoop.gateway.ha.provider.HaProvider;
import org.apache.hadoop.gateway.ha.provider.HaServiceConfig;
+import org.apache.hadoop.gateway.ha.provider.URLManager;
+import org.apache.hadoop.gateway.ha.provider.URLManagerLoader;
import org.apache.hadoop.gateway.ha.provider.impl.i18n.HaMessages;
import org.apache.hadoop.gateway.i18n.messages.MessagesFactory;
@@ -28,54 +30,57 @@ import java.util.concurrent.ConcurrentHashMap;
public class DefaultHaProvider implements HaProvider {
- private static final HaMessages LOG = MessagesFactory.get(HaMessages.class);
+ private static final HaMessages LOG = MessagesFactory.get(HaMessages.class);
- private HaDescriptor descriptor;
+ private HaDescriptor descriptor;
- private ConcurrentHashMap<String, URLManager> haServices;
+ private ConcurrentHashMap<String, URLManager> haServices;
- public DefaultHaProvider(HaDescriptor descriptor) {
- if (descriptor == null) {
- throw new IllegalArgumentException("Descriptor can not be null");
- }
- this.descriptor = descriptor;
- haServices = new ConcurrentHashMap<String, URLManager>();
- }
+ public DefaultHaProvider(HaDescriptor descriptor) {
+ if ( descriptor == null ) {
+ throw new IllegalArgumentException("Descriptor can not be null");
+ }
+ this.descriptor = descriptor;
+ haServices = new ConcurrentHashMap<>();
+ }
- @Override
- public HaDescriptor getHaDescriptor() {
- return descriptor;
- }
+ @Override
+ public HaDescriptor getHaDescriptor() {
+ return descriptor;
+ }
- @Override
- public void addHaService(String serviceName, List<String> urls) {
- haServices.put(serviceName, new URLManager(urls));
- }
+ @Override
+ public void addHaService(String serviceName, List<String> urls) {
+ HaServiceConfig haServiceConfig = descriptor.getServiceConfig(serviceName);
+ URLManager manager = URLManagerLoader.loadURLManager(haServiceConfig);
+ manager.setURLs(urls);
+ haServices.put(serviceName, manager);
+ }
- @Override
- public boolean isHaEnabled(String serviceName) {
- HaServiceConfig config = descriptor.getServiceConfig(serviceName);
- if (config != null && config.isEnabled()) {
- return true;
- }
- return false;
- }
+ @Override
+ public boolean isHaEnabled(String serviceName) {
+ HaServiceConfig config = descriptor.getServiceConfig(serviceName);
+ if ( config != null && config.isEnabled() ) {
+ return true;
+ }
+ return false;
+ }
- @Override
- public String getActiveURL(String serviceName) {
- if (haServices.containsKey(serviceName)) {
- return haServices.get(serviceName).getActiveURL();
- }
- LOG.noActiveUrlFound(serviceName);
- return null;
- }
+ @Override
+ public String getActiveURL(String serviceName) {
+ if ( haServices.containsKey(serviceName) ) {
+ return haServices.get(serviceName).getActiveURL();
+ }
+ LOG.noActiveUrlFound(serviceName);
+ return null;
+ }
- @Override
- public void markFailedURL(String serviceName, String url) {
- if (haServices.containsKey(serviceName)) {
- haServices.get(serviceName).markFailed(url);
- } else {
- LOG.noServiceFound(serviceName);
- }
- }
+ @Override
+ public void markFailedURL(String serviceName, String url) {
+ if ( haServices.containsKey(serviceName) ) {
+ haServices.get(serviceName).markFailed(url);
+ } else {
+ LOG.noServiceFound(serviceName);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultHaServiceConfig.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultHaServiceConfig.java b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultHaServiceConfig.java
index 4bb1e62..e1d6937 100644
--- a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultHaServiceConfig.java
+++ b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultHaServiceConfig.java
@@ -21,79 +21,104 @@ import org.apache.hadoop.gateway.ha.provider.HaServiceConfig;
public class DefaultHaServiceConfig implements HaServiceConfig, HaServiceConfigConstants {
- private String name;
+ private String name;
- private boolean enabled = DEFAULT_ENABLED;
+ private boolean enabled = DEFAULT_ENABLED;
- private int maxFailoverAttempts = DEFAULT_MAX_FAILOVER_ATTEMPTS;
+ private int maxFailoverAttempts = DEFAULT_MAX_FAILOVER_ATTEMPTS;
- private int failoverSleep = DEFAULT_FAILOVER_SLEEP;
+ private int failoverSleep = DEFAULT_FAILOVER_SLEEP;
- private int maxRetryAttempts = DEFAULT_MAX_RETRY_ATTEMPTS;
+ private int maxRetryAttempts = DEFAULT_MAX_RETRY_ATTEMPTS;
- private int retrySleep = DEFAULT_RETRY_SLEEP;
+ private int retrySleep = DEFAULT_RETRY_SLEEP;
- public DefaultHaServiceConfig(String name) {
- this.name = name;
- }
- @Override
+ private String zookeeperEnsemble;
- public String getServiceName() {
- return name;
- }
+ private String zookeeperNamespace;
- @Override
- public void setServiceName(String name) {
- this.name = name;
- }
+ public DefaultHaServiceConfig(String name) {
+ this.name = name;
+ }
- @Override
- public boolean isEnabled() {
- return enabled;
- }
+ @Override
- @Override
- public void setEnabled(boolean enabled) {
- this.enabled = enabled;
- }
+ public String getServiceName() {
+ return name;
+ }
- @Override
- public int getMaxFailoverAttempts() {
- return maxFailoverAttempts;
- }
+ @Override
+ public void setServiceName(String name) {
+ this.name = name;
+ }
- @Override
- public void setMaxFailoverAttempts(int maxFailoverAttempts) {
- this.maxFailoverAttempts = maxFailoverAttempts;
- }
+ @Override
+ public boolean isEnabled() {
+ return enabled;
+ }
- @Override
- public int getFailoverSleep() {
- return failoverSleep;
- }
+ @Override
+ public void setEnabled(boolean enabled) {
+ this.enabled = enabled;
+ }
- @Override
- public void setFailoverSleep(int failoverSleep) {
- this.failoverSleep = failoverSleep;
- }
+ @Override
+ public int getMaxFailoverAttempts() {
+ return maxFailoverAttempts;
+ }
- @Override
- public int getMaxRetryAttempts() {
- return maxRetryAttempts;
- }
+ @Override
+ public void setMaxFailoverAttempts(int maxFailoverAttempts) {
+ this.maxFailoverAttempts = maxFailoverAttempts;
+ }
- @Override
- public void setMaxRetryAttempts(int maxRetryAttempts) {
- this.maxRetryAttempts = maxRetryAttempts;
- }
+ @Override
+ public int getFailoverSleep() {
+ return failoverSleep;
+ }
- @Override
- public int getRetrySleep() {
- return retrySleep;
- }
+ @Override
+ public void setFailoverSleep(int failoverSleep) {
+ this.failoverSleep = failoverSleep;
+ }
- @Override
- public void setRetrySleep(int retrySleep) {
- this.retrySleep = retrySleep;
- }
+ @Override
+ public int getMaxRetryAttempts() {
+ return maxRetryAttempts;
+ }
+
+ @Override
+ public void setMaxRetryAttempts(int maxRetryAttempts) {
+ this.maxRetryAttempts = maxRetryAttempts;
+ }
+
+ @Override
+ public int getRetrySleep() {
+ return retrySleep;
+ }
+
+ @Override
+ public void setRetrySleep(int retrySleep) {
+ this.retrySleep = retrySleep;
+ }
+
+ @Override
+ public String getZookeeperEnsemble() {
+ return zookeeperEnsemble;
+ }
+
+ @Override
+ public void setZookeeperEnsemble(String zookeeperEnsemble) {
+ this.zookeeperEnsemble = zookeeperEnsemble;
+ }
+
+ @Override
+ public String getZookeeperNamespace() {
+ return zookeeperNamespace;
+ }
+
+ @Override
+ public void setZookeeperNamespace(String zookeeperNamespace) {
+ this.zookeeperNamespace = zookeeperNamespace;
+ }
}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultURLManager.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultURLManager.java b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultURLManager.java
new file mode 100644
index 0000000..1783d1a
--- /dev/null
+++ b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultURLManager.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.ha.provider.impl;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.gateway.ha.provider.HaServiceConfig;
+import org.apache.hadoop.gateway.ha.provider.URLManager;
+import org.apache.hadoop.gateway.ha.provider.impl.i18n.HaMessages;
+import org.apache.hadoop.gateway.i18n.messages.MessagesFactory;
+
+import java.net.URI;
+import java.util.List;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+public class DefaultURLManager implements URLManager {
+
+ private static final HaMessages LOG = MessagesFactory.get(HaMessages.class);
+
+ private ConcurrentLinkedQueue<String> urls = new ConcurrentLinkedQueue<>();
+
+ public DefaultURLManager() {
+ }
+
+ @Override
+ public boolean supportsConfig(HaServiceConfig config) {
+ return true;
+ }
+
+ @Override
+ public void setConfig(HaServiceConfig config) {
+ //no-op
+ }
+
+ @Override
+ public String getActiveURL() {
+ return urls.peek();
+ }
+
+ @Override
+ public List<String> getURLs() {
+ return Lists.newArrayList(urls.iterator());
+ }
+
+ @Override
+ public synchronized void setURLs(List<String> urls) {
+ if ( urls != null && !urls.isEmpty()) {
+ this.urls.clear();
+ this.urls.addAll(urls);
+ }
+ }
+
+ @Override
+ public synchronized void markFailed(String url) {
+ String top = urls.peek();
+ boolean pushToBottom = false;
+ URI topUri = URI.create(top);
+ URI incomingUri = URI.create(url);
+ String topHostPort = topUri.getHost() + topUri.getPort();
+ String incomingHostPort = incomingUri.getHost() + incomingUri.getPort();
+ if ( topHostPort.equals(incomingHostPort) ) {
+ pushToBottom = true;
+ }
+ //put the failed url at the bottom
+ if ( pushToBottom ) {
+ String failed = urls.poll();
+ urls.offer(failed);
+ LOG.markedFailedUrl(failed, urls.peek());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HS2ZookeeperURLManager.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HS2ZookeeperURLManager.java b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HS2ZookeeperURLManager.java
new file mode 100644
index 0000000..ac2bfac
--- /dev/null
+++ b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HS2ZookeeperURLManager.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.ha.provider.impl;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.hadoop.gateway.ha.provider.HaServiceConfig;
+import org.apache.hadoop.gateway.ha.provider.impl.i18n.HaMessages;
+import org.apache.hadoop.gateway.i18n.messages.MessagesFactory;
+
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class HS2ZookeeperURLManager extends DefaultURLManager {
+
+ private static final HaMessages LOG = MessagesFactory.get(HaMessages.class);
+
+ private static final Pattern kvPattern = Pattern.compile("([^=;]*)=([^;]*)[;]?");
+
+ private String zooKeeperEnsemble;
+
+ private String zooKeeperNamespace;
+
+ private HashSet<String> failedSet;
+
+ public HS2ZookeeperURLManager() {
+ failedSet = new HashSet<>();
+ }
+
+ @Override
+ public boolean supportsConfig(HaServiceConfig config) {
+ if (!config.getServiceName().equalsIgnoreCase("HIVE")) {
+ return false;
+ }
+ String zookeeperEnsemble = config.getZookeeperEnsemble();
+ String zookeeperNamespace = config.getZookeeperNamespace();
+ if ( zookeeperEnsemble != null && zookeeperNamespace != null && zookeeperEnsemble.trim().length() > 0 && zookeeperNamespace.trim().length() > 0) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void setConfig(HaServiceConfig config) {
+ zooKeeperEnsemble = config.getZookeeperEnsemble();
+ zooKeeperNamespace = config.getZookeeperNamespace();
+ setURLs(lookupURLs());
+ }
+
+ public List<String> lookupURLs() {
+ List<String> serverHosts = new ArrayList<>();
+ CuratorFramework zooKeeperClient =
+ CuratorFrameworkFactory.builder().connectString(zooKeeperEnsemble)
+ .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build();
+ try {
+ zooKeeperClient.start();
+ List<String> serverNodes = zooKeeperClient.getChildren().forPath("/" + zooKeeperNamespace);
+ for ( String serverNode : serverNodes ) {
+ String serverInfo =
+ new String(
+ zooKeeperClient.getData().forPath("/" + zooKeeperNamespace + "/" + serverNode),
+ Charset.forName("UTF-8"));
+ String serverURL = constructURL(serverInfo);
+ serverHosts.add(serverURL);
+ }
+ } catch ( Exception e ) {
+ LOG.failedToGetZookeeperUrls(e);
+ throw new RuntimeException(e);
+ } finally {
+ // Close the client connection with ZooKeeper
+ if ( zooKeeperClient != null ) {
+ zooKeeperClient.close();
+ }
+ }
+ return serverHosts;
+ }
+
+ private String constructURL(String serverInfo) {
+ Matcher matcher = kvPattern.matcher(serverInfo);
+ String scheme = "http";
+ String host = null;
+ String port = "10001";
+ String httpPath = "cliservice";
+ while (matcher.find()) {
+ if ( (matcher.group(1) != null) && matcher.group(2) != null ) {
+ switch ( matcher.group(1) ) {
+ case "hive.server2.thrift.bind.host" :
+ host = matcher.group(2);
+ break;
+ case "hive.server2.thrift.http.port" :
+ port = matcher.group(2);
+ break;
+ case "hive.server2.thrift.http.path" :
+ httpPath = matcher.group(2);
+ break;
+ case "hive.server2.use.SSL" :
+ if (Boolean.parseBoolean(matcher.group(2))) {
+ scheme = "https";
+ }
+ }
+ }
+ }
+ StringBuffer buffer = new StringBuffer();
+ buffer.append(scheme);
+ buffer.append("://");
+ buffer.append(host);
+ buffer.append(":");
+ buffer.append(port);
+ buffer.append("/");
+ buffer.append(httpPath);
+ return buffer.toString();
+ }
+
+ @Override
+ public synchronized void markFailed(String url) {
+ failedSet.add(url);
+ //refresh the list when we have hit all urls once
+ if (failedSet.size() >= getURLs().size()) {
+ failedSet.clear();
+ setURLs(lookupURLs());
+ }
+ super.markFailed(url);
+ }
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorConstants.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorConstants.java b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorConstants.java
index 8113f1c..ce3e07d 100644
--- a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorConstants.java
+++ b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorConstants.java
@@ -42,4 +42,8 @@ public interface HaDescriptorConstants {
public static final String ENABLED_ATTRIBUTE = "enabled";
+ public static final String ZOOKEEPER_ENSEMBLE = "zookeeperEnsemble";
+
+ public static final String ZOOKEEPER_NAMESPACE = "zookeeperNamespace";
+
}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorFactory.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorFactory.java b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorFactory.java
index b5e1232..fc0b811 100644
--- a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorFactory.java
+++ b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorFactory.java
@@ -36,12 +36,17 @@ public abstract class HaDescriptorFactory implements HaServiceConfigConstants {
String failoverSleep = configMap.get(CONFIG_PARAM_FAILOVER_SLEEP);
String maxRetryAttempts = configMap.get(CONFIG_PARAM_MAX_RETRY_ATTEMPTS);
String retrySleep = configMap.get(CONFIG_PARAM_RETRY_SLEEP);
- return createServiceConfig(serviceName, enabledValue, maxFailoverAttempts, failoverSleep, maxRetryAttempts, retrySleep);
+ String zookeeperEnsemble = configMap.get(CONFIG_PARAM_ZOOKEEPER_ENSEMBLE);
+ String zookeeperNamespace = configMap.get(CONFIG_PARAM_ZOOKEEPER_NAMESPACE);
+ return createServiceConfig(serviceName, enabledValue, maxFailoverAttempts,
+ failoverSleep, maxRetryAttempts, retrySleep,
+ zookeeperEnsemble, zookeeperNamespace);
}
public static HaServiceConfig createServiceConfig(String serviceName, String enabledValue,
String maxFailoverAttemptsValue, String failoverSleepValue,
- String maxRetryAttemptsValue, String retrySleepValue) {
+ String maxRetryAttemptsValue, String retrySleepValue,
+ String zookeeperEnsemble, String zookeeperNamespace) {
boolean enabled = DEFAULT_ENABLED;
int maxFailoverAttempts = DEFAULT_MAX_FAILOVER_ATTEMPTS;
int failoverSleep = DEFAULT_FAILOVER_SLEEP;
@@ -68,6 +73,8 @@ public abstract class HaDescriptorFactory implements HaServiceConfigConstants {
serviceConfig.setFailoverSleep(failoverSleep);
serviceConfig.setMaxRetryAttempts(maxRetryAttempts);
serviceConfig.setRetrySleep(retrySleep);
+ serviceConfig.setZookeeperEnsemble(zookeeperEnsemble);
+ serviceConfig.setZookeeperNamespace(zookeeperNamespace);
return serviceConfig;
}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorManager.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorManager.java b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorManager.java
index ba538f0..cbd0c78 100644
--- a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorManager.java
+++ b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorManager.java
@@ -64,6 +64,12 @@ public class HaDescriptorManager implements HaDescriptorConstants {
serviceElement.setAttribute(MAX_RETRY_ATTEMPTS, Integer.toString(config.getMaxRetryAttempts()));
serviceElement.setAttribute(RETRY_SLEEP, Integer.toString(config.getRetrySleep()));
serviceElement.setAttribute(ENABLED_ATTRIBUTE, Boolean.toString(config.isEnabled()));
+ if (config.getZookeeperEnsemble() != null) {
+ serviceElement.setAttribute(ZOOKEEPER_ENSEMBLE, config.getZookeeperEnsemble());
+ }
+ if (config.getZookeeperNamespace() != null) {
+ serviceElement.setAttribute(ZOOKEEPER_NAMESPACE, config.getZookeeperNamespace());
+ }
root.appendChild(serviceElement);
}
}
@@ -101,7 +107,9 @@ public class HaDescriptorManager implements HaDescriptorConstants {
element.getAttribute(MAX_FAILOVER_ATTEMPTS),
element.getAttribute(FAILOVER_SLEEP),
element.getAttribute(MAX_RETRY_ATTEMPTS),
- element.getAttribute(RETRY_SLEEP));
+ element.getAttribute(RETRY_SLEEP),
+ element.getAttribute(ZOOKEEPER_ENSEMBLE),
+ element.getAttribute(ZOOKEEPER_NAMESPACE));
descriptor.addServiceConfig(config);
}
}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaServiceConfigConstants.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaServiceConfigConstants.java b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaServiceConfigConstants.java
index 7dc3bc8..1d84819 100644
--- a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaServiceConfigConstants.java
+++ b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/HaServiceConfigConstants.java
@@ -33,6 +33,10 @@ public interface HaServiceConfigConstants {
public static final String CONFIG_PARAM_ENABLED = "enabled";
+ public static final String CONFIG_PARAM_ZOOKEEPER_ENSEMBLE = "zookeeperEnsemble";
+
+ public static final String CONFIG_PARAM_ZOOKEEPER_NAMESPACE = "zookeeperNamespace";
+
public static final int DEFAULT_MAX_FAILOVER_ATTEMPTS = 3;
public static final int DEFAULT_FAILOVER_SLEEP = 1000;
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/URLManager.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/URLManager.java b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/URLManager.java
deleted file mode 100644
index fc40909..0000000
--- a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/URLManager.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.gateway.ha.provider.impl;
-
-import com.google.common.collect.Lists;
-import org.apache.hadoop.gateway.ha.provider.impl.i18n.HaMessages;
-import org.apache.hadoop.gateway.i18n.messages.MessagesFactory;
-
-import java.net.URI;
-import java.util.List;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-public class URLManager {
-
- private static final HaMessages LOG = MessagesFactory.get(HaMessages.class);
-
- private ConcurrentLinkedQueue<String> urls = new ConcurrentLinkedQueue<String>();
-
- public URLManager(List<String> urls) {
- this.urls.addAll(urls);
- }
-
- public String getActiveURL() {
- return urls.peek();
- }
-
- public List<String> getURLs() {
- return Lists.newArrayList(urls.iterator());
- }
-
- public void setURLs(List<String> urls) {
- if (urls != null) {
- urls.clear();
- urls.addAll(urls);
- }
- }
-
- public synchronized void markFailed(String url) {
- String top = urls.peek();
- boolean pushToBottom = false;
- URI topUri = URI.create(top);
- URI incomingUri = URI.create(url);
- String topHostPort = topUri.getHost() + topUri.getPort();
- String incomingHostPort = incomingUri.getHost() + incomingUri.getPort();
- if (topHostPort.equals(incomingHostPort)) {
- pushToBottom = true;
- }
- //put the failed url at the bottom
- if (pushToBottom) {
- String failed = urls.poll();
- urls.offer(failed);
- LOG.markedFailedUrl(failed, urls.peek());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/i18n/HaMessages.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/i18n/HaMessages.java b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/i18n/HaMessages.java
index 61a2b96..1bad024 100644
--- a/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/i18n/HaMessages.java
+++ b/gateway-provider-ha/src/main/java/org/apache/hadoop/gateway/ha/provider/impl/i18n/HaMessages.java
@@ -39,4 +39,7 @@ public interface HaMessages {
@Message(level = MessageLevel.DEBUG, text = "Moving failed URL to the bottom {0}, new top is {1}")
void markedFailedUrl(String failedUrl, String top);
+ @Message(level = MessageLevel.ERROR, text = "Failed to get Zookeeper URLs : {0}")
+ void failedToGetZookeeperUrls(Exception e);
+
}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/main/resources/META-INF/services/org.apache.hadoop.gateway.ha.provider.URLManager
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/main/resources/META-INF/services/org.apache.hadoop.gateway.ha.provider.URLManager b/gateway-provider-ha/src/main/resources/META-INF/services/org.apache.hadoop.gateway.ha.provider.URLManager
new file mode 100644
index 0000000..d1ec0b9
--- /dev/null
+++ b/gateway-provider-ha/src/main/resources/META-INF/services/org.apache.hadoop.gateway.ha.provider.URLManager
@@ -0,0 +1,19 @@
+##########################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##########################################################################
+
+org.apache.hadoop.gateway.ha.provider.impl.HS2ZookeeperURLManager
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/dispatch/DefaultHaDispatchTest.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/dispatch/DefaultHaDispatchTest.java b/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/dispatch/DefaultHaDispatchTest.java
index 5fd8a5f..3cf13c6 100644
--- a/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/dispatch/DefaultHaDispatchTest.java
+++ b/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/dispatch/DefaultHaDispatchTest.java
@@ -47,7 +47,7 @@ public class DefaultHaDispatchTest {
public void testConnectivityFailover() throws Exception {
String serviceName = "OOZIE";
HaDescriptor descriptor = HaDescriptorFactory.createDescriptor();
- descriptor.addServiceConfig(HaDescriptorFactory.createServiceConfig(serviceName, "true", "1", "1000", "2", "1000"));
+ descriptor.addServiceConfig(HaDescriptorFactory.createServiceConfig(serviceName, "true", "1", "1000", "2", "1000", null, null));
HaProvider provider = new DefaultHaProvider(descriptor);
URI uri1 = new URI( "http://unreachable-host" );
URI uri2 = new URI( "http://reachable-host" );
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultURLManagerTest.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultURLManagerTest.java b/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultURLManagerTest.java
new file mode 100644
index 0000000..6c07d23
--- /dev/null
+++ b/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/DefaultURLManagerTest.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.ha.provider.impl;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class DefaultURLManagerTest {
+
+ @Test
+ public void testActiveURLManagement() {
+ ArrayList<String> urls = new ArrayList<>();
+ String url1 = "http://host1";
+ urls.add(url1);
+ String url2 = "http://host2";
+ urls.add(url2);
+ DefaultURLManager manager = new DefaultURLManager();
+ manager.setURLs(urls);
+ assertTrue(manager.getURLs().containsAll(urls));
+ assertEquals(url1, manager.getActiveURL());
+ manager.markFailed(url1);
+ assertEquals(url2, manager.getActiveURL());
+ manager.markFailed(url2);
+ assertEquals(url1, manager.getActiveURL());
+ }
+
+ @Test
+ public void testMarkingFailedURL() {
+ ArrayList<String> urls = new ArrayList<>();
+ String url1 = "http://host1:4555";
+ urls.add(url1);
+ String url2 = "http://host2:1234";
+ urls.add(url2);
+ String url3 = "http://host1:1234";
+ urls.add(url3);
+ String url4 = "http://host2:4555";
+ urls.add(url4);
+ DefaultURLManager manager = new DefaultURLManager();
+ manager.setURLs(urls);
+ assertTrue(manager.getURLs().containsAll(urls));
+ assertEquals(url1, manager.getActiveURL());
+ manager.markFailed(url1);
+ assertEquals(url2, manager.getActiveURL());
+ manager.markFailed(url1);
+ assertEquals(url2, manager.getActiveURL());
+ manager.markFailed(url3);
+ assertEquals(url2, manager.getActiveURL());
+ manager.markFailed(url4);
+ assertEquals(url2, manager.getActiveURL());
+ manager.markFailed(url2);
+ assertEquals(url3, manager.getActiveURL());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/HS2ZookeeperURLManagerTest.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/HS2ZookeeperURLManagerTest.java b/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/HS2ZookeeperURLManagerTest.java
new file mode 100644
index 0000000..96cf7b4
--- /dev/null
+++ b/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/HS2ZookeeperURLManagerTest.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.ha.provider.impl;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.curator.test.TestingCluster;
+import org.apache.hadoop.gateway.ha.provider.HaServiceConfig;
+import org.apache.hadoop.gateway.ha.provider.URLManager;
+import org.apache.hadoop.gateway.ha.provider.URLManagerLoader;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class HS2ZookeeperURLManagerTest {
+
+ private TestingCluster cluster;
+ private HS2ZookeeperURLManager manager;
+
+ @Before
+ public void setup() throws Exception {
+ cluster = new TestingCluster(3);
+ cluster.start();
+
+ CuratorFramework zooKeeperClient =
+ CuratorFrameworkFactory.builder().connectString(cluster.getConnectString())
+ .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build();
+
+ String host1 = "hive.server2.authentication=NONE;hive.server2.transport.mode=http;hive.server2.thrift.http.path=cliservice;" +
+ "hive.server2.thrift.http.port=10001;hive.server2.thrift.bind.host=host1;hive.server2.use.SSL=true";
+ String host2 = "hive.server2.authentication=NONE;hive.server2.transport.mode=http;hive.server2.thrift.http.path=foobar;" +
+ "hive.server2.thrift.http.port=10002;hive.server2.thrift.bind.host=host2;hive.server2.use.SSL=false";
+ String host3 = "hive.server2.authentication=NONE;hive.server2.transport.mode=http;hive.server2.thrift.http.path=cliservice;" +
+ "hive.server2.thrift.http.port=10003;hive.server2.thrift.bind.host=host3;hive.server2.use.SSL=false";
+ String host4 = "hive.server2.authentication=NONE;hive.server2.transport.mode=http;hive.server2.thrift.http.path=cliservice;" +
+ "hive.server2.thrift.http.port=10004;hive.server2.thrift.bind.host=host4;hive.server2.use.SSL=true";
+ zooKeeperClient.start();
+ zooKeeperClient.create().forPath("/hiveServer2");
+ zooKeeperClient.create().forPath("/hiveServer2/host1", host1.getBytes());
+ zooKeeperClient.create().forPath("/hiveServer2/host2", host2.getBytes());
+ zooKeeperClient.create().forPath("/hiveServer2/host3", host3.getBytes());
+ zooKeeperClient.create().forPath("/hiveServer2/host4", host4.getBytes());
+ zooKeeperClient.close();
+ manager = new HS2ZookeeperURLManager();
+ HaServiceConfig config = new DefaultHaServiceConfig("HIVE");
+ config.setEnabled(true);
+ config.setZookeeperEnsemble(cluster.getConnectString());
+ config.setZookeeperNamespace("hiveServer2");
+ manager.setConfig(config);
+
+ }
+
+ @After
+ public void teardown() throws IOException {
+ cluster.stop();
+ }
+
+ @Test
+ public void testActiveURLManagement() throws Exception {
+ List<String> urls = manager.getURLs();
+ Assert.assertNotNull(urls);
+ String url1 = "https://host4:10004/cliservice";
+ String url2 = "http://host3:10003/cliservice";
+ String url3 = "http://host2:10002/foobar";
+ assertEquals(url1, urls.get(0));
+ assertEquals(url1, manager.getActiveURL());
+ manager.markFailed(url1);
+ assertEquals(url2, manager.getActiveURL());
+ manager.markFailed(url2);
+ assertEquals(url3, manager.getActiveURL());
+ }
+
+ @Test
+ public void testMarkingFailedURL() {
+ ArrayList<String> urls = new ArrayList<>();
+ String url1 = "https://host4:10004/cliservice";
+ urls.add(url1);
+ String url2 = "http://host3:10003/cliservice";
+ urls.add(url2);
+ String url3 = "http://host2:10002/foobar";
+ urls.add(url3);
+ String url4 = "https://host1:10001/cliservice";
+ urls.add(url4);
+ assertTrue(manager.getURLs().containsAll(urls));
+ assertEquals(url1, manager.getActiveURL());
+ manager.markFailed(url1);
+ assertEquals(url2, manager.getActiveURL());
+ manager.markFailed(url1);
+ assertEquals(url2, manager.getActiveURL());
+ manager.markFailed(url3);
+ assertEquals(url2, manager.getActiveURL());
+ manager.markFailed(url4);
+ assertEquals(url2, manager.getActiveURL());
+ manager.markFailed(url2);
+ //now the urls should get re-looked up
+ assertEquals(url1, manager.getActiveURL());
+ }
+
+ @Test
+ public void testHS2URLManagerLoading() {
+ HaServiceConfig config = new DefaultHaServiceConfig("HIVE");
+ config.setEnabled(true);
+ config.setZookeeperEnsemble(cluster.getConnectString());
+ config.setZookeeperNamespace("hiveServer2");
+ URLManager manager = URLManagerLoader.loadURLManager(config);
+ Assert.assertNotNull(manager);
+ Assert.assertTrue(manager instanceof HS2ZookeeperURLManager);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorFactoryTest.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorFactoryTest.java b/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorFactoryTest.java
index d4ff0f7..55622c1 100644
--- a/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorFactoryTest.java
+++ b/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorFactoryTest.java
@@ -40,7 +40,7 @@ public class HaDescriptorFactoryTest {
assertEquals(1, serviceConfig.getMaxRetryAttempts());
assertEquals(1000, serviceConfig.getRetrySleep());
- serviceConfig = HaDescriptorFactory.createServiceConfig("bar", "false", "3", "1000", "5", "3000");
+ serviceConfig = HaDescriptorFactory.createServiceConfig("bar", "false", "3", "1000", "5", "3000", null, null);
assertNotNull(serviceConfig);
assertFalse(serviceConfig.isEnabled());
assertEquals("bar", serviceConfig.getServiceName());
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorManagerTest.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorManagerTest.java b/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorManagerTest.java
index 7c2e580..0e5f0b6 100644
--- a/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorManagerTest.java
+++ b/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/HaDescriptorManagerTest.java
@@ -71,14 +71,14 @@ public class HaDescriptorManagerTest {
@Test
public void testDescriptorStoring() throws IOException {
HaDescriptor descriptor = HaDescriptorFactory.createDescriptor();
- descriptor.addServiceConfig(HaDescriptorFactory.createServiceConfig("foo", "false", "42", "1000", "3", "3000"));
- descriptor.addServiceConfig(HaDescriptorFactory.createServiceConfig("bar", "true", "3", "5000", "5", "8000"));
+ descriptor.addServiceConfig(HaDescriptorFactory.createServiceConfig("foo", "false", "42", "1000", "3", "3000", "foo:2181,bar:2181", "hiveserver2"));
+ descriptor.addServiceConfig(HaDescriptorFactory.createServiceConfig("bar", "true", "3", "5000", "5", "8000", null, null));
StringWriter writer = new StringWriter();
HaDescriptorManager.store(descriptor, writer);
String descriptorXml = writer.toString();
String xml = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n" +
"<ha>\n" +
- " <service enabled=\"false\" failoverSleep=\"1000\" maxFailoverAttempts=\"42\" maxRetryAttempts=\"3\" name=\"foo\" retrySleep=\"3000\"/>\n" +
+ " <service enabled=\"false\" failoverSleep=\"1000\" maxFailoverAttempts=\"42\" maxRetryAttempts=\"3\" name=\"foo\" retrySleep=\"3000\" zookeeperEnsemble=\"foo:2181,bar:2181\" zookeeperNamespace=\"hiveserver2\"/>\n" +
" <service enabled=\"true\" failoverSleep=\"5000\" maxFailoverAttempts=\"3\" maxRetryAttempts=\"5\" name=\"bar\" retrySleep=\"8000\"/>\n" +
"</ha>\n";
assertThat( the( descriptorXml ), isEquivalentTo( the( xml ) ) );
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/MockURLManager.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/MockURLManager.java b/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/MockURLManager.java
new file mode 100644
index 0000000..f11785f
--- /dev/null
+++ b/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/MockURLManager.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.ha.provider.impl;
+
+import org.apache.hadoop.gateway.ha.provider.HaServiceConfig;
+
+public class MockURLManager extends DefaultURLManager {
+
+ HaServiceConfig config;
+
+ @Override
+ public boolean supportsConfig(HaServiceConfig config) {
+ return config.getServiceName().equalsIgnoreCase("mock-test");
+ }
+
+ @Override
+ public void setConfig(HaServiceConfig config) {
+ this.config = config;
+ super.setConfig(config);
+ }
+
+ public HaServiceConfig getConfig() {
+ return config;
+ }
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/URLManagerLoaderTest.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/URLManagerLoaderTest.java b/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/URLManagerLoaderTest.java
new file mode 100644
index 0000000..6aadfec
--- /dev/null
+++ b/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/URLManagerLoaderTest.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.ha.provider.impl;
+
+import org.apache.hadoop.gateway.ha.provider.URLManager;
+import org.apache.hadoop.gateway.ha.provider.URLManagerLoader;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class URLManagerLoaderTest {
+
+ @Test
+ public void testURLManagerLoader() {
+ DefaultHaServiceConfig serviceConfig = new DefaultHaServiceConfig("mock-test");
+ URLManager manager = URLManagerLoader.loadURLManager(serviceConfig);
+ Assert.assertNotNull(manager);
+ Assert.assertTrue(manager instanceof MockURLManager);
+ Assert.assertNotNull(((MockURLManager) manager).getConfig());
+ Assert.assertEquals("mock-test", ((MockURLManager) manager).getConfig().getServiceName());
+ }
+
+ @Test
+ public void testDefaultURLManager() {
+ DefaultHaServiceConfig serviceConfig = new DefaultHaServiceConfig("nothing like this exists");
+ URLManager manager = URLManagerLoader.loadURLManager(serviceConfig);
+ Assert.assertNotNull(manager);
+ Assert.assertTrue(manager instanceof DefaultURLManager);
+ manager = URLManagerLoader.loadURLManager(null);
+ Assert.assertNotNull(manager);
+ Assert.assertTrue(manager instanceof DefaultURLManager);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/URLManagerTest.java
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/URLManagerTest.java b/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/URLManagerTest.java
deleted file mode 100644
index 1de2a7c..0000000
--- a/gateway-provider-ha/src/test/java/org/apache/hadoop/gateway/ha/provider/impl/URLManagerTest.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.gateway.ha.provider.impl;
-
-import org.junit.Test;
-
-import java.util.ArrayList;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class URLManagerTest {
-
- @Test
- public void testActiveURLManagement() {
- ArrayList<String> urls = new ArrayList<String>();
- String url1 = "http://host1";
- urls.add(url1);
- String url2 = "http://host2";
- urls.add(url2);
- URLManager manager = new URLManager(urls);
- assertTrue(manager.getURLs().containsAll(urls));
- assertEquals(url1, manager.getActiveURL());
- manager.markFailed(url1);
- assertEquals(url2, manager.getActiveURL());
- manager.markFailed(url2);
- assertEquals(url1, manager.getActiveURL());
- }
-
- @Test
- public void testMarkingFailedURL() {
- ArrayList<String> urls = new ArrayList<String>();
- String url1 = "http://host1:4555";
- urls.add(url1);
- String url2 = "http://host2:1234";
- urls.add(url2);
- String url3 = "http://host1:1234";
- urls.add(url3);
- String url4 = "http://host2:4555";
- urls.add(url4);
- URLManager manager = new URLManager(urls);
- assertTrue(manager.getURLs().containsAll(urls));
- assertEquals(url1, manager.getActiveURL());
- manager.markFailed(url1);
- assertEquals(url2, manager.getActiveURL());
- manager.markFailed(url1);
- assertEquals(url2, manager.getActiveURL());
- manager.markFailed(url3);
- assertEquals(url2, manager.getActiveURL());
- manager.markFailed(url4);
- assertEquals(url2, manager.getActiveURL());
- manager.markFailed(url2);
- assertEquals(url3, manager.getActiveURL());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-provider-ha/src/test/resources/META-INF/services/org.apache.hadoop.gateway.ha.provider.URLManager
----------------------------------------------------------------------
diff --git a/gateway-provider-ha/src/test/resources/META-INF/services/org.apache.hadoop.gateway.ha.provider.URLManager b/gateway-provider-ha/src/test/resources/META-INF/services/org.apache.hadoop.gateway.ha.provider.URLManager
new file mode 100644
index 0000000..ea210ce
--- /dev/null
+++ b/gateway-provider-ha/src/test/resources/META-INF/services/org.apache.hadoop.gateway.ha.provider.URLManager
@@ -0,0 +1,19 @@
+##########################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##########################################################################
+
+org.apache.hadoop.gateway.ha.provider.impl.MockURLManager
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-service-definitions/src/main/resources/services/hive/0.13.0/service.xml
----------------------------------------------------------------------
diff --git a/gateway-service-definitions/src/main/resources/services/hive/0.13.0/service.xml b/gateway-service-definitions/src/main/resources/services/hive/0.13.0/service.xml
index aba1d23..3ea5b20 100644
--- a/gateway-service-definitions/src/main/resources/services/hive/0.13.0/service.xml
+++ b/gateway-service-definitions/src/main/resources/services/hive/0.13.0/service.xml
@@ -18,5 +18,5 @@
<routes>
<route path="/hive"/>
</routes>
- <dispatch classname="org.apache.hadoop.gateway.hive.HiveDispatch"/>
+ <dispatch classname="org.apache.hadoop.gateway.hive.HiveDispatch" ha-classname="org.apache.hadoop.gateway.hive.HiveHaDispatch"/>
</service>
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-service-hive/pom.xml
----------------------------------------------------------------------
diff --git a/gateway-service-hive/pom.xml b/gateway-service-hive/pom.xml
index 2908ffb..ca87dae 100644
--- a/gateway-service-hive/pom.xml
+++ b/gateway-service-hive/pom.xml
@@ -46,6 +46,10 @@
<artifactId>gateway-provider-rewrite</artifactId>
</dependency>
<dependency>
+ <groupId>${gateway-group}</groupId>
+ <artifactId>gateway-provider-ha</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-service-hive/src/main/java/org/apache/hadoop/gateway/hive/HiveDispatch.java
----------------------------------------------------------------------
diff --git a/gateway-service-hive/src/main/java/org/apache/hadoop/gateway/hive/HiveDispatch.java b/gateway-service-hive/src/main/java/org/apache/hadoop/gateway/hive/HiveDispatch.java
index 2504b9f..ce1b5de 100644
--- a/gateway-service-hive/src/main/java/org/apache/hadoop/gateway/hive/HiveDispatch.java
+++ b/gateway-service-hive/src/main/java/org/apache/hadoop/gateway/hive/HiveDispatch.java
@@ -29,9 +29,7 @@ import org.apache.http.impl.auth.BasicScheme;
* default HttpClientDispatch.
*/
public class HiveDispatch extends DefaultDispatch {
- private static final String PASSWORD_PLACEHOLDER = "*";
private boolean basicAuthPreemptive = false;
- private boolean kerberos = false;
@Override
public void init() {
@@ -40,14 +38,7 @@ public class HiveDispatch extends DefaultDispatch {
protected void addCredentialsToRequest(HttpUriRequest request) {
if( isBasicAuthPreemptive() ) {
- String principal = SubjectUtils.getCurrentEffectivePrincipalName();
- if( principal != null ) {
-
- UsernamePasswordCredentials credentials =
- new UsernamePasswordCredentials( principal, PASSWORD_PLACEHOLDER );
-
- request.addHeader(BasicScheme.authenticate(credentials,"US-ASCII",false));
- }
+ HiveDispatchUtils.addCredentialsToRequest(request);
}
}
@@ -60,14 +51,5 @@ public class HiveDispatch extends DefaultDispatch {
return basicAuthPreemptive;
}
- public boolean isKerberos() {
- return kerberos;
- }
-
- @Configure
- public void setKerberos(boolean kerberos) {
- this.kerberos = kerberos;
- }
-
}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-service-hive/src/main/java/org/apache/hadoop/gateway/hive/HiveDispatchUtils.java
----------------------------------------------------------------------
diff --git a/gateway-service-hive/src/main/java/org/apache/hadoop/gateway/hive/HiveDispatchUtils.java b/gateway-service-hive/src/main/java/org/apache/hadoop/gateway/hive/HiveDispatchUtils.java
new file mode 100644
index 0000000..84f3c6b
--- /dev/null
+++ b/gateway-service-hive/src/main/java/org/apache/hadoop/gateway/hive/HiveDispatchUtils.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.hive;
+
+import org.apache.hadoop.gateway.security.SubjectUtils;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.impl.auth.BasicScheme;
+
+public class HiveDispatchUtils {
+
+ private static final String PASSWORD_PLACEHOLDER = "*";
+
+ public static void addCredentialsToRequest(HttpUriRequest request) {
+ String principal = SubjectUtils.getCurrentEffectivePrincipalName();
+ if ( principal != null ) {
+ UsernamePasswordCredentials credentials =
+ new UsernamePasswordCredentials(principal, PASSWORD_PLACEHOLDER);
+ request.addHeader(BasicScheme.authenticate(credentials, "US-ASCII", false));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-service-hive/src/main/java/org/apache/hadoop/gateway/hive/HiveHaDispatch.java
----------------------------------------------------------------------
diff --git a/gateway-service-hive/src/main/java/org/apache/hadoop/gateway/hive/HiveHaDispatch.java b/gateway-service-hive/src/main/java/org/apache/hadoop/gateway/hive/HiveHaDispatch.java
new file mode 100644
index 0000000..6915aa7
--- /dev/null
+++ b/gateway-service-hive/src/main/java/org/apache/hadoop/gateway/hive/HiveHaDispatch.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.hive;
+
+import org.apache.hadoop.gateway.config.Configure;
+import org.apache.hadoop.gateway.ha.dispatch.DefaultHaDispatch;
+import org.apache.http.client.methods.HttpUriRequest;
+
+
+public class HiveHaDispatch extends DefaultHaDispatch {
+
+ private boolean basicAuthPreemptive = false;
+
+ public HiveHaDispatch() {
+ setServiceRole("HIVE");
+ }
+
+ protected void addCredentialsToRequest(HttpUriRequest request) {
+ if ( isBasicAuthPreemptive() ) {
+ HiveDispatchUtils.addCredentialsToRequest(request);
+ }
+ }
+
+ @Configure
+ public void setBasicAuthPreemptive(boolean basicAuthPreemptive) {
+ this.basicAuthPreemptive = basicAuthPreemptive;
+ }
+
+ public boolean isBasicAuthPreemptive() {
+ return basicAuthPreemptive;
+ }
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-service-webhdfs/src/test/java/org/apache/hadoop/gateway/hdfs/dispatch/WebHdfsHaDispatchTest.java
----------------------------------------------------------------------
diff --git a/gateway-service-webhdfs/src/test/java/org/apache/hadoop/gateway/hdfs/dispatch/WebHdfsHaDispatchTest.java b/gateway-service-webhdfs/src/test/java/org/apache/hadoop/gateway/hdfs/dispatch/WebHdfsHaDispatchTest.java
index 0ac6b57..7e6dc1b 100644
--- a/gateway-service-webhdfs/src/test/java/org/apache/hadoop/gateway/hdfs/dispatch/WebHdfsHaDispatchTest.java
+++ b/gateway-service-webhdfs/src/test/java/org/apache/hadoop/gateway/hdfs/dispatch/WebHdfsHaDispatchTest.java
@@ -47,7 +47,7 @@ public class WebHdfsHaDispatchTest {
public void testConnectivityFailover() throws Exception {
String serviceName = "WEBHDFS";
HaDescriptor descriptor = HaDescriptorFactory.createDescriptor();
- descriptor.addServiceConfig(HaDescriptorFactory.createServiceConfig(serviceName, "true", "1", "1000", "2", "1000"));
+ descriptor.addServiceConfig(HaDescriptorFactory.createServiceConfig(serviceName, "true", "1", "1000", "2", "1000", null, null));
HaProvider provider = new DefaultHaProvider(descriptor);
URI uri1 = new URI( "http://unreachable-host" );
URI uri2 = new URI( "http://reachable-host" );
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java
----------------------------------------------------------------------
diff --git a/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java b/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java
index bc4b5ef..afbd00c 100644
--- a/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java
+++ b/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java
@@ -53,7 +53,7 @@ public class DefaultHttpClientFactory implements HttpClientFactory {
public HttpClient createHttpClient(FilterConfig filterConfig) {
HttpClientBuilder builder = HttpClients.custom();
- if ("true".equals( System.getProperty( GatewayConfig.HADOOP_KERBEROS_SECURED ))) {
+ if ( "true".equals(System.getProperty(GatewayConfig.HADOOP_KERBEROS_SECURED)) ) {
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UseJaasCredentials());
@@ -61,7 +61,7 @@ public class DefaultHttpClientFactory implements HttpClientFactory {
.register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true))
.build();
- builder = builder.setDefaultAuthSchemeRegistry(authSchemeRegistry)
+ builder = builder.setDefaultAuthSchemeRegistry(authSchemeRegistry)
.setDefaultCookieStore(new HadoopAuthCookieStore())
.setDefaultCredentialsProvider(credentialsProvider);
} else {
http://git-wip-us.apache.org/repos/asf/knox/blob/31bb1e02/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 50abb5f..f03fd26 100644
--- a/pom.xml
+++ b/pom.xml
@@ -965,6 +965,17 @@
<version>1.2.3</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>3.4.6</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>2.6.0</version>
+ </dependency>
+
<!-- Html pull parser. EPLv1 license -->
<dependency>
<groupId>net.htmlparser.jericho</groupId>
@@ -1073,6 +1084,13 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
+ <version>2.6.0</version>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
</dependencyManagement>