You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/12/18 22:32:50 UTC
[42/50] incubator-slider git commit: SLIDER-711 verify that cache
headers propagate through minimr cluster connections
SLIDER-711 verify that cache headers propagate through minimr cluster connections
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/515ccba5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/515ccba5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/515ccba5
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 515ccba5524106e86dadb6dbab2fad71a2acbff0
Parents: ca4686d
Author: Steve Loughran <st...@apache.org>
Authored: Wed Dec 17 17:59:01 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Dec 17 17:59:01 2014 +0000
----------------------------------------------------------------------
.../server/appmaster/web/HttpCacheHeaders.java | 35 ++++++++++++
.../standalone/TestStandaloneAgentWeb.groovy | 24 ++++++---
.../org/apache/slider/test/KeysForTests.groovy | 1 +
.../apache/slider/test/SliderTestUtils.groovy | 56 +++++++++++++++-----
.../test/YarnZKMiniClusterTestBase.groovy | 1 +
5 files changed, 96 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/515ccba5/slider-core/src/main/java/org/apache/slider/server/appmaster/web/HttpCacheHeaders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/HttpCacheHeaders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/HttpCacheHeaders.java
new file mode 100644
index 0000000..be8960d
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/HttpCacheHeaders.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web;
+
+/*
+
+
+ , );
+ long now = System.currentTimeMillis();
+ httpRes.addDateHeader ( "Expires", now );
+ httpRes.addDateHeader ( "Date", now );
+ httpRes.addHeader ( "Pragma", "no-cache" );
+ */
+public interface HttpCacheHeaders {
+ String HTTP_HEADER_CACHE_CONTROL = "Cache-Control";
+ String HTTP_HEADER_CACHE_CONTROL_NONE = "no-cache";
+ String HTTP_HEADER_PRAGMA = "Pragma";
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/515ccba5/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
index d524698..7f8ddf5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
@@ -23,6 +23,7 @@ import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.agent.AgentMiniClusterTestBase
+
import static org.apache.slider.api.ResourceKeys.*
import static org.apache.slider.api.StatusKeys.*
import org.apache.slider.client.SliderClient
@@ -58,6 +59,11 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
ApplicationReport report = waitForClusterLive(client)
def realappmaster = report.originalTrackingUrl
+
+ // set up url config to match
+ initConnectionFactory(launcher.configuration)
+
+
execHttpRequest(30000) {
GET(realappmaster)
}
@@ -78,14 +84,19 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
describe "Codahale operations"
// now switch to the Hadoop URL connection, with SPNEGO escalation
- getWebPage(conf, appmaster)
- getWebPage(conf, appmaster, SYSTEM_THREADS)
- getWebPage(conf, appmaster, SYSTEM_HEALTHCHECK)
- getWebPage(conf, appmaster, SYSTEM_METRICS_JSON)
+ getWebPage(appmaster)
+ getWebPage(appmaster, SYSTEM_THREADS)
+ getWebPage(appmaster, SYSTEM_HEALTHCHECK)
+ getWebPage(appmaster, SYSTEM_METRICS_JSON)
- log.info getWebPage(conf, realappmaster, SYSTEM_METRICS_JSON)
+ log.info getWebPage(realappmaster, SYSTEM_METRICS_JSON)
+
+ // get the root page, including some checks for connectivity
+ getWebPage(appmaster, {
+ HttpURLConnection conn ->
+ assertConnectionNotCaching(conn)
+ })
-
// now some REST gets
describe "Application REST GETs"
@@ -111,7 +122,6 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
ConfTreeSerDeser serDeser = new ConfTreeSerDeser()
def json = getWebPage(
- conf,
appmaster,
SLIDER_PATH_APPLICATION + subpath)
def ctree = serDeser.fromJson(json)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/515ccba5/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy b/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
index ddd52eb..4fae2ae 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
@@ -34,4 +34,5 @@ public interface KeysForTests extends SliderKeys, SliderXMLConfKeysForTesting {
String WAIT_TIME_ARG = WAIT_TIME.toString()
String SLIDER_TEST_XML = "slider-test.xml"
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/515ccba5/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index d3cbae0..39dfe39 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -52,10 +52,12 @@ import org.apache.slider.core.exceptions.WaitTimeoutException
import org.apache.slider.core.main.ServiceLaunchException
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.registry.docstore.PublishedConfigSet
+import org.apache.slider.server.appmaster.web.HttpCacheHeaders
import org.apache.slider.server.services.workflow.ForkedProcessService
import org.junit.Assert
import org.junit.Assume
+import javax.ws.rs.core.HttpHeaders
import java.util.concurrent.TimeoutException
import static Arguments.ARG_OPTION
@@ -531,11 +533,9 @@ class SliderTestUtils extends Assert {
* @param page
* @return body of response
*/
- public static String getWebPage(Configuration conf,
- String base,
- String path) {
+ public static String getWebPage(String base, String path) {
String s = appendToURL(base, path)
- return getWebPage(conf, s)
+ return getWebPage(s)
}
/**
@@ -561,21 +561,33 @@ class SliderTestUtils extends Assert {
throw ex;
}
+ static URLConnectionFactory connectionFactory
+
+ public static def initConnectionFactory(Configuration conf) {
+ connectionFactory = URLConnectionFactory
+ .newDefaultURLConnectionFactory(conf);
+ }
+
+
/**
* Fetches a web page asserting that the response code is between 200 and 400.
* Will error on 400 and 500 series response codes and let 200 and 300 through.
*
* if security is enabled, this uses SPNEGO to auth
- * @param page
+ * <p>
+ * Relies on {@link #initConnectionFactory(org.apache.hadoop.conf.Configuration)}
+ * to have been called.
+ *
+ * @param path path to page
+ * @param connectionChecks optional closure to run against an open connection
* @return body of response
*/
- public static String getWebPage(Configuration conf, String page) {
- assert null != page
+ public static String getWebPage(String path, Closure connectionChecks = null) {
+ assert path
+ assert null != connectionFactory
- log.info("Fetching HTTP content at " + page);
- URLConnectionFactory connectionFactory = URLConnectionFactory
- .newDefaultURLConnectionFactory(conf);
- URL url = new URL(page)
+ log.info("Fetching HTTP content at " + path);
+ URL url = new URL(path)
assert url.port != 0
HttpURLConnection conn = null;
int resultCode = 0
@@ -584,8 +596,14 @@ class SliderTestUtils extends Assert {
conn = (HttpURLConnection) connectionFactory.openConnection(url);
conn.instanceFollowRedirects = true;
conn.connect()
+
resultCode = conn.responseCode
+
+ if (connectionChecks) {
+ connectionChecks(conn)
+ }
+
InputStream stream = conn.errorStream;
if (stream == null) {
stream = conn.inputStream;
@@ -597,11 +615,21 @@ class SliderTestUtils extends Assert {
} finally {
conn?.disconnect()
}
- uprateFaults(page, resultCode, body)
+ uprateFaults(path, resultCode, body)
return body;
}
-
-
+
+ /**
+ * Assert that a connection is not caching by looking at the headers
+ * @param conn connection to examine
+ */
+ public static void assertConnectionNotCaching(HttpURLConnection conn) {
+ assert conn.expiration <= conn.date
+ assert conn.getHeaderField(HttpHeaders.CACHE_CONTROL) ==
+ HttpCacheHeaders.HTTP_HEADER_CACHE_CONTROL_NONE
+ assert conn.getHeaderField(HttpCacheHeaders.HTTP_HEADER_PRAGMA) ==
+ HttpCacheHeaders.HTTP_HEADER_CACHE_CONTROL_NONE
+ }
/**
* Assert that a service operation succeeded
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/515ccba5/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
index f6b13a4..e907209 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
@@ -103,6 +103,7 @@ public abstract class YarnZKMiniClusterTestBase extends YarnMiniClusterTestBase
createMicroZKCluster("-${name?:methodName.methodName}", conf)
conf.setBoolean(RegistryConstants.KEY_REGISTRY_ENABLED, true)
conf.set(RegistryConstants.KEY_REGISTRY_ZK_QUORUM, ZKBinding)
+ //now create the cluster
name = super.createMiniCluster(name, conf, noOfNodeManagers, numLocalDirs, numLogDirs,
startHDFS)