You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by el...@apache.org on 2015/01/29 18:23:26 UTC
[03/28] incubator-slider git commit: SLIDER-719: bonding jersey to
UrlConnectionOperations
SLIDER-719: bonding jersey to UrlConnectionOperations
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/61f4743f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/61f4743f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/61f4743f
Branch: refs/heads/develop
Commit: 61f4743f83d73e255eee5744be3e72bbb9c60754
Parents: 901f38c
Author: Steve Loughran <st...@apache.org>
Authored: Fri Jan 16 16:57:09 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jan 20 14:50:23 2015 +0000
----------------------------------------------------------------------
pom.xml | 5 +-
.../core/restclient/UgiJerseyBinding.java | 79 ++++++++++++++++++++
.../restclient/UrlConnectionOperations.java | 2 +-
.../slider/agent/rest/RestTestDelegates.groovy | 20 ++++-
.../slider/agent/rest/TestStandaloneREST.groovy | 7 +-
.../agent/TestAgentAMManagementWS.groovy | 4 +-
.../publisher/TestPublisherRestResources.groovy | 2 +-
.../apache/slider/test/SliderTestUtils.groovy | 61 ++++++++++++---
.../funtest/lifecycle/AgentWebPagesIT.groovy | 3 +-
9 files changed, 159 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index cf989e4..b206d66 100644
--- a/pom.xml
+++ b/pom.xml
@@ -154,6 +154,7 @@
<gson.version>2.2.2</gson.version>
<guice.version>3.0</guice.version>
<httpclient.version>3.1</httpclient.version>
+ <httpclient4.version>4.2.5</httpclient4.version>
<jackson.version>1.9.13</jackson.version>
<jcommander.version>1.30</jcommander.version>
@@ -645,13 +646,13 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
- <version>4.2.5</version>
+ <version>${httpclient4.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
- <version>4.2.5</version>
+ <version>${httpclient4.version}</version>
</dependency>
<!-- ======================================================== -->
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
new file mode 100644
index 0000000..6af8e23
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.restclient;
+
+import com.google.common.base.Preconditions;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+/**
+ * Class to bond to a Jersey client, for UGI integration and SPNEGO.
+ * <p>
+ * Usage: create an instance, then when creating a Jersey <code>Client</code>
+ * pass in to the constructor the handler provided by {@link #getHandler()}
+ *
+ * @see https://jersey.java.net/apidocs/1.17/jersey/com/sun/jersey/client/urlconnection/HttpURLConnectionFactory.html
+ */
+public class UgiJerseyBinding implements
+ HttpURLConnectionFactory {
+ private final UrlConnectionOperations operations;
+ private final URLConnectionClientHandler handler;
+
+ /**
+ * Construct an instance
+ * @param operations operations instance
+ */
+ @SuppressWarnings("ThisEscapedInObjectConstruction")
+ public UgiJerseyBinding(UrlConnectionOperations operations) {
+ Preconditions.checkArgument(operations != null, "Null operations");
+ this.operations = operations;
+ handler = new URLConnectionClientHandler(this);
+ }
+
+ /**
+ * Get a URL connection.
+ * @param url
+ * @return the connection
+ * @throws IOException any problem. {@link AuthenticationException}
+ * errors are wrapped
+ */
+ @Override
+ public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
+ try {
+ return operations.openConnection(url);
+ } catch (AuthenticationException e) {
+ throw new IOException(e);
+ }
+ }
+
+ public UrlConnectionOperations getOperations() {
+ return operations;
+ }
+
+ public URLConnectionClientHandler getHandler() {
+ return handler;
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
index fa0da5a..c51914a 100644
--- a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
@@ -40,7 +40,7 @@ import java.net.URL;
* Operations on the JDK UrlConnection class. This uses WebHDFS
* methods to set up the operations.
*/
-public class UrlConnectionOperations extends Configured {
+public class UrlConnectionOperations extends Configured {
private static final Logger log =
LoggerFactory.getLogger(UrlConnectionOperations.class);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
index 3431175..0826204 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
@@ -18,6 +18,8 @@
package org.apache.slider.agent.rest
+import com.sun.jersey.api.client.Client
+import com.sun.jersey.api.client.WebResource
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.webapp.NotFoundException
@@ -27,9 +29,11 @@ import org.apache.slider.api.types.SerializedContainerInformation
import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.conf.ConfTree
import org.apache.slider.core.conf.ConfTreeOperations
+import org.apache.slider.core.registry.docstore.PublishedConfiguration
import org.apache.slider.core.restclient.HttpOperationResponse
import org.apache.slider.core.restclient.HttpVerb
import org.apache.slider.core.restclient.UrlConnectionOperations
+import org.apache.slider.server.appmaster.web.rest.RestPaths
import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource
import org.apache.slider.server.appmaster.web.rest.application.resources.PingResource
import org.apache.slider.test.Outcome
@@ -53,11 +57,14 @@ class RestTestDelegates extends SliderTestUtils {
public static final String TEST_GLOBAL_OPTION_PRESENT = "present"
final String appmaster;
+ final String application;
RestTestDelegates(String appmaster) {
this.appmaster = appmaster
+ application = appendToURL(appmaster, RestPaths.SLIDER_PATH_APPLICATION)
}
+
public void testCodahaleOperations() throws Throwable {
describe "Codahale operations"
getWebPage(appmaster)
@@ -89,6 +96,15 @@ class RestTestDelegates extends SliderTestUtils {
assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_COMPLETED)
assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_RELEASING)
}
+
+
+ public void testRestletOperations() throws Throwable {
+ Client client = createJerseyClient()
+ String path = appendToURL(application, LIVE_RESOURCES)
+ WebResource webResource = client.resource(path)
+ webResource.type(MediaType.APPLICATION_JSON)
+ .get(ConfTree.class);
+ }
public void testLiveContainers() throws Throwable {
describe "Application REST ${LIVE_CONTAINERS}"
@@ -211,7 +227,7 @@ class RestTestDelegates extends SliderTestUtils {
HttpVerb verb,
URL pingUrl,
String payload) {
- return pingAction(connectionFactory, verb, pingUrl, payload)
+ return pingAction(connectionOperations, verb, pingUrl, payload)
}
private HttpOperationResponse pingAction(
@@ -244,7 +260,7 @@ class RestTestDelegates extends SliderTestUtils {
String target = appendToURL(appmaster, SLIDER_PATH_APPLICATION, ACTION_STOP)
describe "Stop URL $target"
URL targetUrl = new URL(target)
- def outcome = connectionFactory.execHttpOperation(
+ def outcome = connectionOperations.execHttpOperation(
HttpVerb.POST,
targetUrl,
new byte[0],
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
index a3378a8..22bc4ae 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
@@ -62,14 +62,14 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
def realappmaster = report.originalTrackingUrl
// set up url config to match
- initConnectionFactory(launcher.configuration)
+ initHttpTestSupport(launcher.configuration)
- execHttpRequest(WEB_STARTUP_TIME) {
+ execOperation(WEB_STARTUP_TIME) {
GET(realappmaster)
}
- execHttpRequest(WEB_STARTUP_TIME) {
+ execOperation(WEB_STARTUP_TIME) {
def metrics = GET(realappmaster, SYSTEM_METRICS)
log.info metrics
}
@@ -90,6 +90,7 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
RestTestDelegates proxied = new RestTestDelegates(appmaster)
RestTestDelegates direct = new RestTestDelegates(realappmaster)
+ direct.testRestletOperations();
proxied.testCodahaleOperations()
direct.testCodahaleOperations()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
index 2e2e62b..7eb1d17 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -153,7 +153,7 @@ class TestAgentAMManagementWS extends AgentTestBase {
log.info("tracking URL is $proxyAM")
// spin awaiting the agent web page coming up.
- execHttpRequest(WEB_STARTUP_TIME) {
+ execOperation(WEB_STARTUP_TIME) {
GET(proxyAM)
}
@@ -173,7 +173,7 @@ class TestAgentAMManagementWS extends AgentTestBase {
log.info "AM live, now fetching agent at $agent_url"
// spin awaiting the agent web page coming up.
- execHttpRequest(WEB_STARTUP_TIME) {
+ execOperation(WEB_STARTUP_TIME) {
GET(agent_url)
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
index e3f38eb..29f0510 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
@@ -107,7 +107,7 @@ class TestPublisherRestResources extends AgentTestBase {
webResource = client.resource(sliderConfigset + "dummy-site");
- execHttpRequest(30000) {
+ execOperation(30000) {
GET(sliderConfigset)
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 2c6b5fe..9dd2828 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -18,6 +18,10 @@
package org.apache.slider.test
+import com.sun.jersey.api.client.Client
+import com.sun.jersey.api.client.config.ClientConfig
+import com.sun.jersey.api.client.config.DefaultClientConfig
+import com.sun.jersey.api.json.JSONConfiguration
import groovy.json.JsonOutput
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
@@ -55,6 +59,7 @@ import org.apache.slider.core.main.ServiceLaunchException
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.persist.JsonSerDeser
import org.apache.slider.core.registry.docstore.PublishedConfigSet
+import org.apache.slider.core.restclient.UgiJerseyBinding
import org.apache.slider.core.restclient.UrlConnectionOperations
import org.apache.slider.server.appmaster.web.HttpCacheHeaders
import org.apache.slider.server.appmaster.web.rest.RestPaths
@@ -450,7 +455,8 @@ class SliderTestUtils extends Assert {
}
/**
- * Fetch a web page
+ * Fetch a web page using HttpClient 3.1.
+ * This <i>DOES NOT</i> work with secure connections.
* @param url URL
* @return the response body
*/
@@ -478,6 +484,9 @@ class SliderTestUtils extends Assert {
/**
* Fetches a web page asserting that the response code is between 200 and 400.
+ * This <i>DOES NOT</i> work with secure connections.
+ * <p>
+ *
* Will error on 400 and 500 series response codes and let 200 and 300 through.
* @param url URL to get as string
* @return body of response
@@ -548,7 +557,7 @@ class SliderTestUtils extends Assert {
/**
* Fetches a web page asserting that the response code is between 200 and 400.
* Will error on 400 and 500 series response codes and let 200 and 300 through.
- *
+ * <p>
* if security is enabled, this uses SPNEGO to auth
* @param page
* @return body of response
@@ -559,13 +568,14 @@ class SliderTestUtils extends Assert {
}
/**
- * Execute any of the http requests, swallowing exceptions until
- * eventually they time out
+ * Execute any operation provided as a closure which returns a string, swallowing exceptions until
+ * eventually they time out.
+ *
* @param timeout
* @param operation
* @return
*/
- public static String execHttpRequest(int timeout, Closure operation) {
+ public static String execOperation(int timeout, Closure operation) {
Duration duration = new Duration(timeout).start()
Exception ex = new IOException("limit exceeded before starting");
while (!duration.limitExceeded) {
@@ -581,12 +591,41 @@ class SliderTestUtils extends Assert {
throw ex;
}
- static UrlConnectionOperations connectionFactory
+ /**
+ * Static factory for URL connections
+ */
+ static UrlConnectionOperations connectionOperations
+ static UgiJerseyBinding jerseyBinding;
- public static def initConnectionFactory(Configuration conf) {
- connectionFactory = new UrlConnectionOperations(conf);
+
+ /**
+ * Static initializer of the connection operations
+ * @param conf config
+ */
+ public static synchronized void initHttpTestSupport(Configuration conf) {
+ connectionOperations = new UrlConnectionOperations(conf);
+ jerseyBinding = new UgiJerseyBinding(connectionOperations)
+
}
+ /**
+ * Check for the HTTP support being initialized
+ */
+ public static synchronized void assertHttpSupportInitialized() {
+ assert connectionOperations
+ assert jerseyBinding
+ }
+
+ /**
+ * Create Jersey client
+ * @return
+ */
+ public static Client createJerseyClient() {
+ assertHttpSupportInitialized()
+ ClientConfig clientConfig = new DefaultClientConfig();
+ clientConfig.features[JSONConfiguration.FEATURE_POJO_MAPPING] =Boolean.TRUE;
+ return new Client(jerseyBinding.handler, clientConfig);
+ }
/**
* Fetches a web page asserting that the response code is between 200 and 400.
@@ -594,7 +633,7 @@ class SliderTestUtils extends Assert {
*
* if security is enabled, this uses SPNEGO to auth
* <p>
- * Relies on {@link #initConnectionFactory(org.apache.hadoop.conf.Configuration)}
+ * Relies on {@link #initHttpTestSupport(org.apache.hadoop.conf.Configuration)}
* to have been called.
*
* @param path path to page
@@ -603,11 +642,11 @@ class SliderTestUtils extends Assert {
*/
public static String getWebPage(String path, Closure connectionChecks = null) {
assert path
- assert null != connectionFactory
+ assertHttpSupportInitialized()
log.info("Fetching HTTP content at " + path);
URL url = new URL(path)
- def outcome = connectionFactory.execGet(url)
+ def outcome = connectionOperations.execGet(url)
String body = new String(outcome.data)
return body;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
index ce1d955..de8dcdf 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
@@ -29,7 +29,6 @@ import org.apache.slider.common.tools.ConfigHelper
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
-import org.apache.slider.server.appmaster.web.rest.RestPaths
import org.junit.After
import org.junit.Before
import org.junit.Test
@@ -82,7 +81,7 @@ public class AgentWebPagesIT extends AgentCommandTestBase
def conf = SLIDER_CONFIG
- initConnectionFactory(conf)
+ initHttpTestSupport(conf)
def appId = ensureYarnApplicationIsUp(launchReportFile)
assert appId