You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/11/28 18:36:28 UTC
[1/3] incubator-slider git commit: SLIDER-689 add support for the
SPNEGO-enabled Hadoop web connection code from WebHDFS
Repository: incubator-slider
Updated Branches:
refs/heads/develop 6e55f771b -> 3fe140c20
SLIDER-689 add support for the SPNEGO-enabled Hadoop web connection code from WebHDFS
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7c73a4a4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7c73a4a4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7c73a4a4
Branch: refs/heads/develop
Commit: 7c73a4a4091d4ce753cc3fb0d3253243432cd696
Parents: 6e55f77
Author: Steve Loughran <st...@apache.org>
Authored: Fri Nov 28 10:44:21 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Nov 28 10:44:21 2014 +0000
----------------------------------------------------------------------
.../standalone/TestStandaloneAgentWeb.groovy | 18 ++++--
.../apache/slider/test/SliderTestUtils.groovy | 58 +++++++++++++++++++-
2 files changed, 71 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7c73a4a4/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
index 1553b2f..b2d29f3 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
@@ -39,10 +39,10 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
describe "create a standalone AM then perform actions on it"
//launch fake master
- def configuration = configuration
- configuration.setBoolean(METRICS_LOGGING_ENABLED, true)
- configuration.setInt(METRICS_LOGGING_LOG_INTERVAL, 1)
- String clustername = createMiniCluster("", configuration, 1, true)
+ def conf = configuration
+ conf.setBoolean(METRICS_LOGGING_ENABLED, true)
+ conf.setInt(METRICS_LOGGING_LOG_INTERVAL, 1)
+ String clustername = createMiniCluster("", conf, 1, true)
ServiceLauncher<SliderClient> launcher =
@@ -66,6 +66,16 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
log.info GET(appmaster, RestPaths.SYSTEM_HEALTHCHECK)
log.info GET(appmaster, RestPaths.SYSTEM_METRICS_JSON)
+ describe "Hadoop HTTP operations"
+ // now switch to the Hadoop URL connection, with SPNEGO escalation
+ getWebPage(conf, appmaster)
+ getWebPage(conf, appmaster, RestPaths.SYSTEM_THREADS)
+ getWebPage(conf, appmaster, RestPaths.SYSTEM_HEALTHCHECK)
+ getWebPage(conf, appmaster, RestPaths.SYSTEM_METRICS_JSON)
+
+ log.info getWebPage(conf, realappmaster, RestPaths.SYSTEM_METRICS_JSON)
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7c73a4a4/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 3688644..e612aa3 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -28,6 +28,8 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileStatus
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
+import org.apache.hadoop.hdfs.web.URLConnectionFactory
+import org.apache.hadoop.io.IOUtils
import org.apache.hadoop.service.ServiceStateException
import org.apache.hadoop.util.Shell
import org.apache.hadoop.yarn.api.records.ApplicationReport
@@ -423,7 +425,6 @@ class SliderTestUtils extends Assert {
public static String GET(String base, String path) {
String s = appendToURL(base, path)
return GET(s)
-
}
def static String GET(String s) {
@@ -493,6 +494,61 @@ class SliderTestUtils extends Assert {
}
/**
+ * Fetches a web page asserting that the response code is between 200 and 400.
+ * Will error on 400 and 500 series response codes and let 200 and 300 through.
+ *
+ * if security is enabled, this uses SPNEGO to auth
+ * @param page
+ * @return body of response
+ */
+ public static String getWebPage(Configuration conf,
+ String base,
+ String path) {
+ String s = appendToURL(base, path)
+ return getWebPage(conf, s)
+ }
+
+ /**
+ * Fetches a web page asserting that the response code is between 200 and 400.
+ * Will error on 400 and 500 series response codes and let 200 and 300 through.
+ *
+ * if security is enabled, this uses SPNEGO to auth
+ * @param page
+ * @return body of response
+ */
+ public static String getWebPage(Configuration conf, String page) {
+ assert null != page
+
+ log.info("Fetching HTTP content at " + page);
+ URLConnectionFactory connectionFactory = URLConnectionFactory
+ .newDefaultURLConnectionFactory(conf);
+ URL url = new URL(page)
+ HttpURLConnection conn =
+ (HttpURLConnection) connectionFactory.openConnection(url);
+ try {
+ conn.instanceFollowRedirects = true;
+ conn.connect()
+
+ int resultCode = conn.responseCode
+ InputStream stream = conn.errorStream;
+ if (stream == null) {
+ stream = conn.inputStream;
+ }
+
+ def body = stream ? stream.text : "(no body)"
+ if (!(resultCode >= 200 && resultCode < 400)) {
+ def message = "Request to $url failed with ${conn.responseMessage}, body length ${body?.length()}:\n$body"
+ log.error(message)
+ fail(message)
+ }
+ return body;
+ } finally {
+ conn?.disconnect()
+
+ }
+ }
+
+ /**
* Assert that a service operation succeeded
* @param service service
*/
[2/3] incubator-slider git commit: SLIDER-689 switch AgentWebPagesIT
to the WebHDFS/SPNEGO page fetching
Posted by st...@apache.org.
SLIDER-689 switch AgentWebPagesIT to the WebHDFS/SPNEGO page fetching
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/54065b5a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/54065b5a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/54065b5a
Branch: refs/heads/develop
Commit: 54065b5a92ff875e75dc9e1a1faba370a626d4d5
Parents: 7c73a4a
Author: Steve Loughran <st...@apache.org>
Authored: Fri Nov 28 10:59:48 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Nov 28 10:59:48 2014 +0000
----------------------------------------------------------------------
.../apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/54065b5a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
index 416fa11..d29fe63 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
@@ -82,11 +82,12 @@ public class AgentWebPagesIT extends AgentCommandTestBase
def report = loadAppReport(liveReportFile)
assert report.url
+ def conf = SLIDER_CONFIG
def root = report.url
- log.info GET(root, RestPaths.SYSTEM_METRICS)
- GET(root, RestPaths.SYSTEM_THREADS)
- log.info GET(root, RestPaths.SYSTEM_HEALTHCHECK)
- log.info GET(root, RestPaths.SYSTEM_PING)
+ log.info getWebPage (conf, root, RestPaths.SYSTEM_METRICS)
+ log.info getWebPage(conf, root, RestPaths.SYSTEM_THREADS)
+ log.info getWebPage(conf, root, RestPaths.SYSTEM_HEALTHCHECK)
+ log.info getWebPage(conf, root, RestPaths.SYSTEM_PING)
}
}
[3/3] incubator-slider git commit: Merge branch
'feature/SLIDER-689_AgentWebPagesIT_fails_to_run_against_a_kerberos_cluster'
into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-689_AgentWebPagesIT_fails_to_run_against_a_kerberos_cluster' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3fe140c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3fe140c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3fe140c2
Branch: refs/heads/develop
Commit: 3fe140c20f9b21214e43999197d5b65a7e610ff0
Parents: 6e55f77 54065b5
Author: Steve Loughran <st...@apache.org>
Authored: Fri Nov 28 17:20:42 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Nov 28 17:20:42 2014 +0000
----------------------------------------------------------------------
.../standalone/TestStandaloneAgentWeb.groovy | 18 ++++--
.../apache/slider/test/SliderTestUtils.groovy | 58 +++++++++++++++++++-
.../funtest/lifecycle/AgentWebPagesIT.groovy | 9 +--
3 files changed, 76 insertions(+), 9 deletions(-)
----------------------------------------------------------------------