You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by el...@apache.org on 2015/01/29 18:23:24 UTC
[01/28] incubator-slider git commit: SLIDER-759 slider tarball
doesn't always contain slider-agent.tar.gz
Repository: incubator-slider
Updated Branches:
refs/heads/develop 7f8c5b23f -> 3105ba9f3
SLIDER-759 slider tarball doesn't always contain slider-agent.tar.gz
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/901f38cd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/901f38cd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/901f38cd
Branch: refs/heads/develop
Commit: 901f38cd3840237159ddfcadb666893ed07de0ee
Parents: 7f8c5b2
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jan 19 16:26:59 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jan 19 16:27:21 2015 +0000
----------------------------------------------------------------------
pom.xml | 1 +
slider-assembly/pom.xml | 26 ++++++++++--
slider-assembly/src/assembly/slider-bin.xml | 46 ++++++++-------------
slider-assembly/src/assembly/verifications.xml | 34 +++++++++++++++
4 files changed, 74 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901f38cd/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index efe05fb..cf989e4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -219,6 +219,7 @@
<maven-surefire-plugin.version>2.17</maven-surefire-plugin.version>
<maven-surefire-report-plugin.version>${maven-surefire-plugin.version}</maven-surefire-report-plugin.version>
<maven-failsafe-plugin.version>${maven-surefire-plugin.version}</maven-failsafe-plugin.version>
+ <maven-verifier-plugin.version>1.0</maven-verifier-plugin.version>
<apache-rat-plugin.version>0.11</apache-rat-plugin.version>
<!-- build options-->
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901f38cd/slider-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml
index ce98e65..3e5d9ed 100644
--- a/slider-assembly/pom.xml
+++ b/slider-assembly/pom.xml
@@ -102,7 +102,7 @@
<executions>
<execution>
<id>copy-dependencies</id>
- <phase>package</phase>
+ <phase>prepare-package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
@@ -117,7 +117,7 @@
<execution>
<!-- copy in the agent tar file -->
<id>copy</id>
- <phase>package</phase>
+ <phase>prepare-package</phase>
<goals>
<goal>copy</goal>
</goals>
@@ -137,7 +137,7 @@
<execution>
<!-- copy in the agent tar file -->
<id>copy-to-lib</id>
- <phase>package</phase>
+ <phase>prepare-package</phase>
<goals>
<goal>copy</goal>
</goals>
@@ -222,7 +222,25 @@
</execution>
</executions>
</plugin>
-
+
+ <!-- verification phase checks for files in the target assembly -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-verifier-plugin</artifactId>
+ <version>${maven-verifier-plugin.version}</version>
+ <configuration>
+ <verificationFile>src/assembly/verifications.xml</verificationFile>
+ </configuration>
+ <executions>
+ <execution>
+ <id>main</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901f38cd/slider-assembly/src/assembly/slider-bin.xml
----------------------------------------------------------------------
diff --git a/slider-assembly/src/assembly/slider-bin.xml b/slider-assembly/src/assembly/slider-bin.xml
index c5611f8..f13f5df 100644
--- a/slider-assembly/src/assembly/slider-bin.xml
+++ b/slider-assembly/src/assembly/slider-bin.xml
@@ -18,9 +18,9 @@
<assembly
- xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
<id>all</id>
<formats>
<format>tar.gz</format>
@@ -43,35 +43,9 @@
</fileSet>
<fileSet>
- <directory>
- src/conf
- </directory>
+ <directory>src/conf</directory>
<outputDirectory>conf</outputDirectory>
</fileSet>
-
- <fileSet>
- <directory>${project.build.directory}</directory>
- <outputDirectory>lib</outputDirectory>
- <includes>
- <include>*.jar</include>
- </includes>
- </fileSet>
-
- <fileSet>
- <directory>${project.build.directory}/agent</directory>
- <outputDirectory>lib</outputDirectory>
- <includes>
- <include>slider-agent.tar.gz</include>
- </includes>
- </fileSet>
-
- <fileSet>
- <directory>${project.build.directory}/agent</directory>
- <outputDirectory>agent</outputDirectory>
- <includes>
- <include>slider-agent.tar.gz</include>
- </includes>
- </fileSet>
<fileSet>
<directory>${project.build.directory}/../../slider-agent/conf</directory>
@@ -97,6 +71,20 @@
</fileSet>
</fileSets>
+
+ <files>
+
+ <file>
+ <source>${project.build.directory}/agent/slider-agent.tar.gz</source>
+ <outputDirectory>lib</outputDirectory>
+ </file>
+
+ <file>
+ <source>${project.build.directory}/agent/slider-agent.tar.gz</source>
+ <outputDirectory>agent</outputDirectory>
+ </file>
+
+ </files>
<dependencySets>
<dependencySet>
<scope>runtime</scope>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901f38cd/slider-assembly/src/assembly/verifications.xml
----------------------------------------------------------------------
diff --git a/slider-assembly/src/assembly/verifications.xml b/slider-assembly/src/assembly/verifications.xml
new file mode 100644
index 0000000..920fedb
--- /dev/null
+++ b/slider-assembly/src/assembly/verifications.xml
@@ -0,0 +1,34 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<verifications>
+ <!-- These checks verify that the tarfile is in place.
+ However, they don't check that the tarfile has made it through to the -all tarball, as the name of
+ that file is not known & the verifier plugin doesn't do property expansion
+ -->
+ <files>
+ <file>
+ <location>target/lib/slider-agent.tar.gz</location>
+ <exists>true</exists>
+ </file>
+ <file>
+ <location>target/agent/slider-agent.tar.gz</location>
+ <exists>true</exists>
+ </file>
+ </files>
+</verifications>
\ No newline at end of file
[15/28] incubator-slider git commit: SLIDER-641. Remove
agent/slider-agent.tar.gz from slider tarball
Posted by el...@apache.org.
SLIDER-641. Remove agent/slider-agent.tar.gz from slider tarball
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ae1077db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ae1077db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ae1077db
Branch: refs/heads/develop
Commit: ae1077db9a5cc5a74402e1e37551286193c0c3bd
Parents: b74b1f5
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Jan 22 18:05:50 2015 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Jan 22 18:05:50 2015 -0800
----------------------------------------------------------------------
slider-assembly/pom.xml | 17 -----------------
slider-assembly/src/assembly/slider-bin.xml | 5 -----
slider-assembly/src/assembly/verifications.xml | 4 ----
3 files changed, 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae1077db/slider-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml
index 3e5d9ed..6cb9128 100644
--- a/slider-assembly/pom.xml
+++ b/slider-assembly/pom.xml
@@ -373,23 +373,6 @@
</sources>
</mapping>
- <!-- agent -->
- <mapping>
- <directory>${rpm.agentdir}</directory>
- <filemode>0755</filemode>
- <username>${rpm.username}</username>
- <groupname>${rpm.groupname}</groupname>
- <directoryIncluded>false</directoryIncluded>
- <sources>
- <source>
- <location>${project.build.directory}/agent</location>
- <includes>
- <include>slider-agent.tar.gz</include>
- </includes>
- </source>
- </sources>
- </mapping>
-
<!-- agent.ini -->
<mapping>
<directory>${rpm.agentdir}/conf</directory>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae1077db/slider-assembly/src/assembly/slider-bin.xml
----------------------------------------------------------------------
diff --git a/slider-assembly/src/assembly/slider-bin.xml b/slider-assembly/src/assembly/slider-bin.xml
index f13f5df..c1eb499 100644
--- a/slider-assembly/src/assembly/slider-bin.xml
+++ b/slider-assembly/src/assembly/slider-bin.xml
@@ -78,11 +78,6 @@
<source>${project.build.directory}/agent/slider-agent.tar.gz</source>
<outputDirectory>lib</outputDirectory>
</file>
-
- <file>
- <source>${project.build.directory}/agent/slider-agent.tar.gz</source>
- <outputDirectory>agent</outputDirectory>
- </file>
</files>
<dependencySets>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae1077db/slider-assembly/src/assembly/verifications.xml
----------------------------------------------------------------------
diff --git a/slider-assembly/src/assembly/verifications.xml b/slider-assembly/src/assembly/verifications.xml
index 920fedb..8826eac 100644
--- a/slider-assembly/src/assembly/verifications.xml
+++ b/slider-assembly/src/assembly/verifications.xml
@@ -26,9 +26,5 @@
<location>target/lib/slider-agent.tar.gz</location>
<exists>true</exists>
</file>
- <file>
- <location>target/agent/slider-agent.tar.gz</location>
- <exists>true</exists>
- </file>
</files>
</verifications>
\ No newline at end of file
[17/28] incubator-slider git commit: SLIDER-782 funtest for slider
application API
Posted by el...@apache.org.
SLIDER-782 funtest for slider application API
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/73e16399
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/73e16399
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/73e16399
Branch: refs/heads/develop
Commit: 73e163998596a9f3acc2ea8f740054c2a9b3434c
Parents: b0963bd
Author: Steve Loughran <st...@apache.org>
Authored: Fri Jan 23 12:45:52 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Jan 23 12:45:52 2015 +0000
----------------------------------------------------------------------
.../funtest/lifecycle/AgentWebPagesIT.groovy | 18 +++++++++++++++++-
1 file changed, 17 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/73e16399/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
index 2d1c863..fd54b55 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
@@ -24,6 +24,7 @@ import org.apache.hadoop.security.UserGroupInformation
import org.apache.hadoop.yarn.webapp.ForbiddenException
import org.apache.slider.agent.rest.JerseyTestDelegates
import org.apache.slider.agent.rest.RestTestDelegates
+import org.apache.slider.agent.rest.SliderRestClientTestDelegates
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
@@ -126,7 +127,22 @@ public class AgentWebPagesIT extends AgentCommandTestBase
new JerseyTestDelegates(directAM, createUGIJerseyClient())
directJerseyTests.testSuiteGetOperations()
directJerseyTests.testSuiteComplexVerbs()
-
+
+ describe "Proxy SliderRestClient Tests"
+ SliderRestClientTestDelegates proxySliderRestClient =
+ new SliderRestClientTestDelegates(proxyAM, createUGIJerseyClient())
+ proxySliderRestClient.testSuiteGetOperations()
+ if (!wsBackDoorRequired) {
+ proxySliderRestClient.testSuiteComplexVerbs()
+ }
+ describe "Direct SliderRestClient Tests"
+ SliderRestClientTestDelegates directSliderRestClient =
+ new SliderRestClientTestDelegates(directAM, createUGIJerseyClient())
+ directSliderRestClient.testSuiteGetOperations()
+ directSliderRestClient.testSuiteComplexVerbs()
+
+
+
if (UserGroupInformation.securityEnabled) {
describe "Insecure Proxy Tests against a secure cluster"
[12/28] incubator-slider git commit: SLIDER-762 API-level slider REST
client. "Yes, but where are the tests"
Posted by el...@apache.org.
SLIDER-762 API-level slider REST client. "Yes, but where are the tests"
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5b5dac88
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5b5dac88
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5b5dac88
Branch: refs/heads/develop
Commit: 5b5dac889698ecd5428ac1e0b89c271b97f75c4d
Parents: 1282802
Author: Steve Loughran <st...@apache.org>
Authored: Thu Jan 22 18:27:44 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Jan 22 18:27:44 2015 +0000
----------------------------------------------------------------------
.../apache/slider/client/SliderRestClient.java | 332 +++++++++++++
.../core/restclient/UgiJerseyBinding.java | 44 ++
.../restclient/UrlConnectionOperations.java | 25 +-
.../server/appmaster/state/RoleStatus.java | 5 +
.../rest/application/ApplicationResource.java | 2 +-
.../agent/rest/JerseyTestDelegates.groovy | 24 +-
.../rest/SliderRestClientTestDelegates.groovy | 476 +++++++++++++++++++
7 files changed, 885 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5b5dac88/slider-core/src/main/java/org/apache/slider/client/SliderRestClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderRestClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderRestClient.java
new file mode 100644
index 0000000..14b85e9
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderRestClient.java
@@ -0,0 +1,332 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client;
+
+import com.google.common.base.Preconditions;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.GenericType;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.representation.Form;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.slider.api.types.SerializedComponentInformation;
+import org.apache.slider.api.types.SerializedContainerInformation;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.restclient.HttpVerb;
+import org.apache.slider.core.restclient.UgiJerseyBinding;
+import org.apache.slider.server.appmaster.web.rest.application.resources.PingResource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.MediaType;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
+
+public class SliderRestClient extends AbstractService {
+ private static final Logger log =
+ LoggerFactory.getLogger(SliderRestClient.class);
+ private final Client jersey;
+ private WebResource appmaster;
+ private WebResource appResource;
+
+ public SliderRestClient(String name, Client jersey, WebResource appmaster) {
+ super(name);
+ Preconditions.checkNotNull(jersey, "null jersey");
+ this.jersey = jersey;
+ }
+
+ public SliderRestClient(Client jersey, WebResource appmaster) {
+ this("SliderRestClient", jersey, appmaster);
+ }
+
+ public Client getJersey() {
+ return jersey;
+ }
+
+ public void bindToAppmaster(WebResource appmaster) {
+ this.appmaster = appmaster;
+ this.appResource = appmaster.path(SLIDER_PATH_APPLICATION);
+ }
+
+ public WebResource getAppmaster() {
+ return appmaster;
+ }
+
+ /**
+ * Create a resource under the application path
+ * @param subpath
+ * @return an resource under the application path
+ */
+ public WebResource applicationResource(String subpath) {
+ return appResource.path(subpath);
+ }
+
+ /**
+ * Get operation against a path under the Application
+ * @param <T> type expected
+ * @param subpath path
+ * @param c class to instantiate
+ * @return instance
+ * @throws IOException on any problem
+ */
+ public <T> T getApplicationResource(String subpath, Class<T> c)
+ throws IOException {
+ return appResourceOperation(HttpVerb.GET, subpath, c);
+ }
+
+ /**
+ * Get operation against a path under the Application
+ * @param <T> type expected
+ * @param subpath path
+ * @param t type info
+ * @return instance
+ * @throws IOException on any problem
+ */
+ public <T> T getApplicationResource(String subpath, GenericType<T> t)
+ throws IOException {
+ return appResourceOperation(HttpVerb.GET, subpath, t);
+ }
+
+ /**
+ *
+ * @param method method to exec
+ * @param <T> type expected
+ * @param subpath path
+ * @param c class to instantiate
+ * @return instance
+ * @throws IOException on any problem
+ */
+ public <T> T appResourceOperation(HttpVerb method, String subpath, Class<T> c)
+ throws IOException {
+ WebResource resource = applicationResource(subpath);
+ return exec(method, resource, c);
+ }
+
+
+ /**
+ * Get operation against a path under the Application
+ * @param <T> type expected
+ * @param subpath path
+ * @param t type info
+ * @return instance
+ * @throws IOException on any problem
+ */
+ public <T> T appResourceOperation(HttpVerb method, String subpath,
+ GenericType<T> t)
+ throws IOException {
+ WebResource resource = applicationResource(subpath);
+ return exec(method, resource, t);
+ }
+
+ /**
+ * Execute the operation. Failures are raised as IOException subclasses
+ * @param method method to execute
+ * @param resource resource to work against
+ * @param c class to build
+ * @param <T> type expected
+ * @return an instance of the type T
+ * @throws IOException on any failure
+ */
+ public <T> T exec(HttpVerb method, WebResource resource, Class<T> c)
+ throws IOException {
+ try {
+ Preconditions.checkArgument(c != null);
+ resource.accept(MediaType.APPLICATION_JSON_TYPE);
+ return (T) resource.method(method.getVerb(), c);
+ } catch (UniformInterfaceException ex) {
+ throw UgiJerseyBinding.uprateFaults(method, resource.getURI().toString(),
+ ex);
+ }
+ }
+
+
+ /**
+ * Execute the operation. Failures are raised as IOException subclasses
+ * @param method method to execute
+ * @param resource resource to work against
+ * @param generic type to work with
+ * @param <T> type expected
+ * @return an instance of the type T
+ * @throws IOException on any failure
+ */
+ public <T> T exec(HttpVerb method, WebResource resource, GenericType<T> t)
+ throws IOException {
+ try {
+ Preconditions.checkArgument(t != null);
+ resource.accept(MediaType.APPLICATION_JSON_TYPE);
+ return resource.method(method.getVerb(), t);
+ } catch (UniformInterfaceException ex) {
+ throw UgiJerseyBinding.uprateFaults(method, resource.getURI().toString(),
+ ex);
+ }
+ }
+
+
+
+ /**
+ * Get the aggregate desired model
+ * @return the aggregate configuration of what was asked for
+ * —before resolution has taken place
+ * @throws IOException on any failure
+ */
+ public AggregateConf getDesiredModel() throws IOException {
+ return getApplicationResource(MODEL_DESIRED, AggregateConf.class);
+ }
+
+
+ /**
+ * Get the desired application configuration
+ * @return the application configuration asked for
+ * —before resolution has taken place
+ * @throws IOException on any failure
+ */
+ public ConfTreeOperations getDesiredAppconf() throws IOException {
+ ConfTree resource =
+ getApplicationResource(MODEL_DESIRED_APPCONF, ConfTree.class);
+ return new ConfTreeOperations(resource);
+ }
+
+ /**
+ * Get the desired YARN resources
+ * @return the resources asked for
+ * —before resolution has taken place
+ * @throws IOException on any failure
+ */
+ public ConfTreeOperations getDesiredYarnResources() throws IOException {
+ ConfTree resource =
+ getApplicationResource(MODEL_DESIRED_RESOURCES, ConfTree.class);
+ return new ConfTreeOperations(resource);
+ }
+
+ /**
+ * Get the aggregate resolved model
+ * @return the aggregate configuration of what was asked for
+ * —after resolution has taken place
+ * @throws IOException on any failure
+ */
+ public AggregateConf getResolvedModel() throws IOException {
+ return getApplicationResource(MODEL_RESOLVED, AggregateConf.class);
+ }
+
+
+ /**
+ * Get the resolved application configuration
+ * @return the application configuration asked for
+ * —after resolution has taken place
+ * @throws IOException on any failure
+ */
+ public ConfTreeOperations getResolvedAppconf() throws IOException {
+ ConfTree resource =
+ getApplicationResource(MODEL_RESOLVED_APPCONF, ConfTree.class);
+ return new ConfTreeOperations(resource);
+ }
+
+ /**
+ * Get the resolved YARN resources
+ * @return the resources asked for
+ * —after resolution has taken place
+ * @throws IOException on any failure
+ */
+ public ConfTreeOperations getResolvedYarnResources() throws IOException {
+ ConfTree resource =
+ getApplicationResource(MODEL_RESOLVED_RESOURCES, ConfTree.class);
+ return new ConfTreeOperations(resource);
+ }
+
+ /**
+ * Get the live YARN resources
+ * @return the live set of resources in the cluster
+ * @throws IOException on any failure
+ */
+ public ConfTreeOperations getLiveYarnResources() throws IOException {
+ ConfTree resource =
+ getApplicationResource(LIVE_RESOURCES, ConfTree.class);
+ return new ConfTreeOperations(resource);
+ }
+
+ /**
+ * Get a map of live containers [containerId:info]
+ * @return a possibly empty list of serialized containers
+ * @throws IOException on any failure
+ */
+ public Map<String, SerializedContainerInformation> enumContainers() throws
+ IOException {
+ return getApplicationResource(LIVE_RESOURCES,
+ new GenericType<Map<String, SerializedContainerInformation>>() {
+ });
+ }
+
+ /**
+ * Get a container from the container Id
+ * @param containerId YARN container ID
+ * @return the container information
+ * @throws IOException on any failure
+ */
+ public SerializedContainerInformation getContainer( String containerId) throws
+ IOException {
+ return getApplicationResource(LIVE_CONTAINERS + "/" + containerId,
+ SerializedContainerInformation.class);
+ }
+
+ /**
+ * List all components into a map of [name:info]
+ * @return a possibly empty map of components
+ * @throws IOException on any failure
+ */
+ public Map<String, SerializedComponentInformation> enumComponents() throws
+ IOException {
+ return getApplicationResource(LIVE_COMPONENTS,
+ new GenericType<Map<String, SerializedComponentInformation>>() {
+ });
+ }
+
+ /**
+ * Get information about a component
+ * @param componentName name of the component
+ * @return the component details
+ * @throws IOException on any failure
+ */
+ public SerializedComponentInformation getComponent(String componentName) throws
+ IOException {
+ return getApplicationResource(LIVE_COMPONENTS + "/" + componentName,
+ SerializedComponentInformation.class);
+ }
+
+ /**
+ * Ping as a post
+ * @param text text to include
+ * @return the response
+ * @throws IOException on any failure
+ */
+ public PingResource ping(String text) throws IOException {
+ WebResource pingOut = applicationResource(ACTION_PING);
+ pingOut.accept(MediaType.APPLICATION_JSON_TYPE);
+ pingOut.type(MediaType.APPLICATION_JSON_TYPE);
+ Form f = new Form();
+ f.add("text", text);
+ return pingOut.post(PingResource.class, f);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5b5dac88/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
index 6f002ab..f2991d2 100644
--- a/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
@@ -19,10 +19,19 @@
package org.apache.slider.core.restclient;
import com.google.common.base.Preconditions;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.PathAccessDeniedException;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.yarn.webapp.ForbiddenException;
+import org.apache.hadoop.yarn.webapp.NotFoundException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.HttpURLConnection;
@@ -38,6 +47,8 @@ import java.net.URL;
*/
public class UgiJerseyBinding implements
HttpURLConnectionFactory {
+ private static final Logger log =
+ LoggerFactory.getLogger(UgiJerseyBinding.class);
private final UrlConnectionOperations operations;
private final URLConnectionClientHandler handler;
@@ -93,6 +104,39 @@ public class UgiJerseyBinding implements
return operations.isUseSpnego();
}
+
+ /**
+ * Uprate error codes 400 and up into faults;
+ * 404 is converted to a {@link NotFoundException},
+ * 401 to {@link ForbiddenException}
+ *
+ * @param verb HTTP Verb used
+ * @param url URL as string
+ * @param ex exception
+ */
+ public static IOException uprateFaults(HttpVerb verb, String url,
+ UniformInterfaceException ex)
+ throws IOException {
+
+ ClientResponse response = ex.getResponse();
+ int resultCode = response.getStatus();
+ String msg = verb.toString() + " " + url;
+ if (resultCode == 404) {
+ return (IOException) new PathNotFoundException(url).initCause(ex);
+ }
+ if (resultCode == 401) {
+ return (IOException) new PathAccessDeniedException(url).initCause(ex);
+ }
+ // all other error codes
+
+
+ // get a string respnse
+ String message = msg +
+ " failed with exit code " + resultCode
+ + ", message " + ex.toString();
+ log.error(message, ex);
+ return (IOException) new PathIOException(url, message).initCause(ex);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5b5dac88/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
index 6e29c55..328684e 100644
--- a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
@@ -160,7 +160,7 @@ public class UrlConnectionOperations extends Configured {
conn.disconnect();
}
}
- uprateFaults(HttpVerb.GET, url, resultCode, body);
+ uprateFaults(HttpVerb.GET, url.toString(), resultCode, "", body);
outcome.responseCode = resultCode;
outcome.data = body;
return outcome;
@@ -174,18 +174,18 @@ public class UrlConnectionOperations extends Configured {
* @param verb HTTP Verb used
* @param url URL as string
* @param resultCode response from the request
- * @param body optional body of the request
- * @throws IOException if the result was considered a failure
+ * @param bodyAsString
+ *@param body optional body of the request @throws IOException if the result was considered a failure
*/
- public static void uprateFaults(HttpVerb verb, URL url,
- int resultCode, byte[] body)
+ public static void uprateFaults(HttpVerb verb, String url,
+ int resultCode, String bodyAsString, byte[] body)
throws IOException {
if (resultCode < 400) {
//success
return;
}
- String msg = verb.toString() +" "+ url.toString();
+ String msg = verb.toString() +" "+ url;
if (resultCode == 404) {
throw new NotFoundException(msg);
}
@@ -193,11 +193,14 @@ public class UrlConnectionOperations extends Configured {
throw new ForbiddenException(msg);
}
// all other error codes
- String bodyAsString;
- if (body != null && body.length > 0) {
- bodyAsString = new String(body);
- } else {
- bodyAsString = "";
+
+ // get a string respnse
+ if (bodyAsString == null) {
+ if (body != null && body.length > 0) {
+ bodyAsString = new String(body);
+ } else {
+ bodyAsString = "";
+ }
}
String message = msg +
" failed with exit code " + resultCode
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5b5dac88/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 0ffc309..3edc5f1 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -285,6 +285,10 @@ public final class RoleStatus implements Cloneable {
return stats;
}
+ /**
+ * Produced a serialized form which can be served up as JSON
+ * @return a summary of the current role status.
+ */
public synchronized SerializedComponentInformation serialize() {
SerializedComponentInformation info = new SerializedComponentInformation();
info.name = name;
@@ -298,6 +302,7 @@ public final class RoleStatus implements Cloneable {
info.requested = requested;
info.placementPolicy = getPlacementPolicy();
info.failureMessage = failureMessage;
+ info.totalRequested = totalRequested;
return info;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5b5dac88/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
index c2c83a0..735fd48 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
@@ -215,7 +215,7 @@ public class ApplicationResource extends AbstractSliderResource {
@GET
@Path(LIVE_RESOURCES)
@Produces({APPLICATION_JSON})
- public Object getLiveResources() {
+ public ConfTree getLiveResources() {
markGet(SLIDER_SUBPATH_APPLICATION, LIVE_RESOURCES);
return lookupConfTree(LIVE_RESOURCES);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5b5dac88/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
index 09788e1..40f86a8 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
@@ -18,6 +18,7 @@
package org.apache.slider.agent.rest
+import com.google.common.base.Preconditions
import com.sun.jersey.api.client.Client
import com.sun.jersey.api.client.ClientResponse
import com.sun.jersey.api.client.UniformInterfaceException
@@ -29,6 +30,7 @@ import org.apache.hadoop.yarn.webapp.NotFoundException
import org.apache.slider.api.StateValues
import org.apache.slider.api.types.SerializedComponentInformation
import org.apache.slider.api.types.SerializedContainerInformation
+import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.conf.ConfTree
import org.apache.slider.core.conf.ConfTreeOperations
@@ -80,7 +82,7 @@ class JerseyTestDelegates extends SliderTestUtils {
* @return
*/
public <T> T jGetApplicationResource(String subpath, Class<T> c) {
- return (T)jExec(HttpVerb.GET, subpath, c)
+ return (T)jExec(HttpVerb.GET, subpath, c);
}
/**
@@ -90,17 +92,17 @@ class JerseyTestDelegates extends SliderTestUtils {
* @return
*/
public <T> T jExec(HttpVerb method, String subpath, Class<T> c) {
- WebResource resource = applicationResource(subpath)
- jExec(method, resource, c)
+ WebResource resource = applicationResource(subpath);
+ jExec(method, resource, c);
}
public <T> T jExec(HttpVerb method, WebResource resource, Class<T> c) {
try {
- assert c
- resource.accept(MediaType.APPLICATION_JSON_TYPE)
- (T) resource.method(method.verb, c)
+ Preconditions.checkArgument(c != null);
+ resource.accept(MediaType.APPLICATION_JSON_TYPE);
+ (T) resource.method(method.verb, c);
} catch (UniformInterfaceException ex) {
- uprateFaults(method, resource, ex)
+ uprateFaults(method, resource, ex);
}
}
@@ -158,11 +160,11 @@ class JerseyTestDelegates extends SliderTestUtils {
*/
public WebResource buildResource(String path) {
assert path
- String fullpath = appendToURL(appmaster, path)
- WebResource webResource = jersey.resource(fullpath)
- webResource.type(MediaType.APPLICATION_JSON)
+ String fullpath = SliderUtils.appendToURL(appmaster, path);
+ WebResource webResource = jersey.resource(fullpath);
+ webResource.type(MediaType.APPLICATION_JSON);
log.info("HTTP operation against $fullpath");
- return webResource
+ return webResource;
}
public void testJerseyGetConftree() throws Throwable {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5b5dac88/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy
new file mode 100644
index 0000000..a97d136
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy
@@ -0,0 +1,476 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.rest
+
+import com.google.common.base.Preconditions
+import com.sun.jersey.api.client.Client
+import com.sun.jersey.api.client.ClientResponse
+import com.sun.jersey.api.client.UniformInterfaceException
+import com.sun.jersey.api.client.WebResource
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.webapp.NotFoundException
+import org.apache.slider.api.StateValues
+import org.apache.slider.api.types.SerializedComponentInformation
+import org.apache.slider.api.types.SerializedContainerInformation
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.core.conf.AggregateConf
+import org.apache.slider.core.conf.ConfTree
+import org.apache.slider.core.conf.ConfTreeOperations
+import org.apache.slider.core.restclient.HttpVerb
+import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource
+import org.apache.slider.server.appmaster.web.rest.application.resources.PingResource
+import org.apache.slider.test.SliderTestUtils
+
+import javax.ws.rs.core.MediaType
+
+import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES
+import static org.apache.slider.api.StatusKeys.*
+import static org.apache.slider.common.SliderKeys.COMPONENT_AM
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.*
+
+/**
+ * This class contains parts of tests that can be run
+ * against a deployed AM: local or remote.
+ * It uses Jersey ... and must be passed a client that is either secure
+ * or not
+ *
+ */
+@CompileStatic
+@Slf4j
+class SliderRestClientTestDelegates extends SliderTestUtils {
+ public static final String TEST_GLOBAL_OPTION = "test.global.option"
+ public static final String TEST_GLOBAL_OPTION_PRESENT = "present"
+
+ final String appmaster;
+ final String application;
+ final Client jersey;
+ final WebResource amResource
+ final WebResource appResource
+
+
+ SliderRestClientTestDelegates(String appmaster, Client jersey) {
+ this.jersey = jersey
+ this.appmaster = appmaster
+ application = appendToURL(appmaster, SLIDER_PATH_APPLICATION)
+ amResource = jersey.resource(appmaster)
+ amResource.type(MediaType.APPLICATION_JSON)
+ appResource = amResource.path(SLIDER_PATH_APPLICATION)
+ }
+
+ /**
+ * <T> T get(Class<T> c)
+ * Get operation against a path under the Application
+ * @param subpath path
+ * @return
+ */
+ public <T> T jGetApplicationResource(String subpath, Class<T> c) {
+ return (T)jExec(HttpVerb.GET, subpath, c);
+ }
+
+ /**
+ * <T> T get(Class<T> c)
+ * Get operation against a path under the Application
+ * @param subpath path
+ * @return
+ */
+ public <T> T jExec(HttpVerb method, String subpath, Class<T> c) {
+ WebResource resource = applicationResource(subpath);
+ jExec(method, resource, c);
+ }
+
+ public <T> T jExec(HttpVerb method, WebResource resource, Class<T> c) {
+ try {
+ Preconditions.checkArgument(c != null);
+ resource.accept(MediaType.APPLICATION_JSON_TYPE);
+ (T) resource.method(method.verb, c);
+ } catch (UniformInterfaceException ex) {
+ uprateFaults(method, resource, ex);
+ }
+ }
+
+ /**
+ * Create a resource under the application path
+ * @param subpath
+ * @return
+ */
+ public WebResource applicationResource(String subpath) {
+ return appResource.path(subpath)
+ }
+
+ /**
+ * Convert faults to exceptions; pass through 200 responses
+ * @param method
+ * @param webResource
+ * @param ex
+ * @return
+ */
+ public uprateFaults(
+ HttpVerb method,
+ WebResource webResource,
+ UniformInterfaceException ex) {
+ uprateFaults(method.verb,
+ webResource.URI.toString(),
+ ex.response.status,
+ ex.response.toString())
+ }
+
+ /**
+ * <T> T get(Class<T> c)
+ * Get operation against a path under the AM
+ * @param path path
+ * @return
+ */
+ public <T> T jGetAMResource(String path, Class<T> c) {
+ assert c
+ WebResource webResource = buildResource(path)
+ (T)webResource.get(c)
+ }
+
+ /**
+ * Get operation against a path under the AM
+ * @param path path
+ * @return the string value
+ */
+ public String jerseyGet(String path) {
+ return jGetAMResource(path, String.class)
+ }
+
+ /**
+ * Build a resource against a path under the AM API
+ * @param path path
+ * @return a resource for use
+ */
+ public WebResource buildResource(String path) {
+ assert path
+ String fullpath = SliderUtils.appendToURL(appmaster, path);
+ WebResource webResource = jersey.resource(fullpath);
+ webResource.type(MediaType.APPLICATION_JSON);
+ log.info("HTTP operation against $fullpath");
+ return webResource;
+ }
+
+ public void testJerseyGetConftree() throws Throwable {
+ jGetApplicationResource(LIVE_RESOURCES, ConfTree.class);
+ }
+ public void testCodahaleOperations() throws Throwable {
+ describe "Codahale operations"
+
+ jerseyGet("/")
+ jerseyGet(SYSTEM_THREADS)
+ jerseyGet(SYSTEM_HEALTHCHECK)
+ jerseyGet(SYSTEM_PING)
+ jerseyGet(SYSTEM_METRICS_JSON)
+ }
+
+ public void logCodahaleMetrics() {
+ // query Coda Hale metrics
+ log.info jerseyGet(SYSTEM_HEALTHCHECK)
+ log.info jerseyGet(SYSTEM_METRICS)
+ }
+
+ /**
+ * Fetch a typed entry <i>under the application path</i>
+ * @param subpath
+ * @param clazz
+ * @return
+ */
+ public <T> T jFetchType(
+ String subpath, Class<T> clazz) {
+ (T)jGetApplicationResource(subpath, clazz)
+ }
+
+ public ConfTreeOperations jGetConfigTree(
+ String path) {
+ ConfTree ctree = jGetApplicationResource(path, ConfTree)
+ ConfTreeOperations tree = new ConfTreeOperations(ctree)
+ return tree
+ }
+
+
+ public void testMimeTypes() throws Throwable {
+ describe "Mime Types"
+
+ WebResource resource = applicationResource(LIVE_RESOURCES)
+ def response = resource.get(ClientResponse)
+ response.headers.each {key, val -> log.info("$key: $val")}
+ log.info response.toString()
+ assert response.type.equals(MediaType.APPLICATION_JSON_TYPE)
+ }
+
+
+ public void testLiveResources() throws Throwable {
+ describe "Live Resources"
+
+ ConfTreeOperations tree = jGetConfigTree(LIVE_RESOURCES)
+
+ log.info tree.toString()
+ def liveAM = tree.getComponent(COMPONENT_AM)
+ def desiredInstances = liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES);
+ assert desiredInstances ==
+ liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_ACTUAL)
+
+ assert 1 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_STARTED)
+ assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_REQUESTING)
+ assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_FAILED)
+ assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_COMPLETED)
+ assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_RELEASING)
+ }
+
+ public void testLiveContainers() throws Throwable {
+ describe "Application REST ${LIVE_CONTAINERS}"
+
+ Map<String, SerializedContainerInformation> containers =
+ jGetApplicationResource(LIVE_CONTAINERS, HashMap)
+ assert containers.size() == 1
+ log.info "${containers}"
+ SerializedContainerInformation amContainerInfo =
+ (SerializedContainerInformation) containers.values()[0]
+ assert amContainerInfo.containerId
+
+ def amContainerId = amContainerInfo.containerId
+ assert containers[amContainerId]
+
+ assert amContainerInfo.component == COMPONENT_AM
+ assert amContainerInfo.createTime > 0
+ assert amContainerInfo.exitCode == null
+ assert amContainerInfo.output == null
+ assert amContainerInfo.released == null
+ assert amContainerInfo.state == StateValues.STATE_LIVE
+
+ describe "containers"
+
+ SerializedContainerInformation retrievedContainerInfo =
+ jFetchType(
+ LIVE_CONTAINERS + "/${amContainerId}",
+ SerializedContainerInformation
+ )
+ assert retrievedContainerInfo.containerId == amContainerId
+
+ // fetch missing
+ try {
+ def result = jFetchType(
+ LIVE_CONTAINERS + "/unknown",
+ SerializedContainerInformation
+ )
+ fail("expected an error, got $result")
+ } catch (NotFoundException e) {
+ // expected
+ }
+
+
+ describe "components"
+
+ Map<String, SerializedComponentInformation> components =
+ jFetchType(LIVE_COMPONENTS, HashMap)
+ // two components
+ assert components.size() >= 1
+ log.info "${components}"
+
+ SerializedComponentInformation amComponentInfo =
+ (SerializedComponentInformation) components[COMPONENT_AM]
+
+ SerializedComponentInformation amFullInfo = jFetchType(
+ LIVE_COMPONENTS + "/${COMPONENT_AM}",
+ SerializedComponentInformation
+ )
+
+ assert amFullInfo.containers.size() == 1
+ assert amFullInfo.containers[0] == amContainerId
+
+ }
+
+ /**
+ * Assert that a path resolves to an array list that contains
+ * those entries (and only those entries) expected
+ * @param appmaster AM ref
+ * @param path path under AM
+ * @param entries entries to assert the presence of
+ */
+ public void assertPathServesList(
+ String appmaster,
+ String path,
+ List<String> entries) {
+ def list = jFetchType(path, ArrayList)
+ assert list.size() == entries.size()
+ assert entries.containsAll(list)
+ }
+
+ /**
+ * Fetch a list of URLs, all of which must be of the same type
+ * @param clazz class of resolved values
+ * @param appmaster URL to app master
+ * @param subpaths list of subpaths
+ * @return a map of paths to values
+ */
+ public <T> Map<String, T> fetchTypeList(
+ Class<T> clazz, String appmaster, List<String> subpaths
+ ) {
+ Map<String, T> results = [:]
+ subpaths.each { String it ->
+ results[it] = (jFetchType(it, clazz))
+ }
+ return results;
+ }
+
+ /**
+ * Test the rest model. For this to work the cluster has to be configured
+ * with the global option
+ * @param appmaster
+ */
+ public void testRESTModel() {
+ describe "model"
+
+ assertPathServesList(appmaster,
+ MODEL,
+ ApplicationResource.MODEL_ENTRIES)
+
+ def unresolvedConf = jFetchType(MODEL_DESIRED, AggregateConf)
+// log.info "Unresolved \n$unresolvedConf"
+ def unresolvedAppConf = unresolvedConf.appConfOperations
+
+ def sam = "slider-appmaster"
+ assert unresolvedAppConf.getComponentOpt(sam,
+ TEST_GLOBAL_OPTION, "") == ""
+ def resolvedConf = jFetchType(MODEL_RESOLVED, AggregateConf)
+// log.info "Resolved \n$resolvedConf"
+ assert resolvedConf.appConfOperations.getComponentOpt(
+ sam, TEST_GLOBAL_OPTION, "") == TEST_GLOBAL_OPTION_PRESENT
+
+ def unresolved = fetchTypeList(ConfTree, appmaster,
+ [MODEL_DESIRED_APPCONF, MODEL_DESIRED_RESOURCES])
+ assert unresolved[MODEL_DESIRED_APPCONF].components[sam]
+ [TEST_GLOBAL_OPTION] == null
+
+
+ def resolved = fetchTypeList(ConfTree, appmaster,
+ [MODEL_RESOLVED_APPCONF, MODEL_RESOLVED_RESOURCES])
+ assert resolved[MODEL_RESOLVED_APPCONF].
+ components[sam][TEST_GLOBAL_OPTION] == TEST_GLOBAL_OPTION_PRESENT
+ }
+
+ public void testPing() {
+ // GET
+ describe "pinging"
+
+ def pinged = jExec(HttpVerb.GET, ACTION_PING, PingResource)
+ log.info "Ping GET: $pinged"
+ // HEAD
+// jExec(HttpVerb.HEAD, ACTION_PING, PingResource)
+ jExec(HttpVerb.PUT, ACTION_PING, PingResource)
+ jExec(HttpVerb.DELETE, ACTION_PING, PingResource)
+ jExec(HttpVerb.POST, ACTION_PING, PingResource)
+ ping(HttpVerb.PUT, ACTION_PING, "ping-text")
+ ping(HttpVerb.POST, ACTION_PING, "ping-text")
+ ping(HttpVerb.DELETE, ACTION_PING, "ping-text")
+ }
+
+ /**
+ * Execute a ping; assert that a response came back with the relevant
+ * verb if the verb has a response body
+ * @param method method to invoke
+ * @param subpath ping path
+ * @param payload payload
+ * @return the resource if the verb has a response
+ */
+ private PingResource ping(HttpVerb method, String subpath, Object payload) {
+ def actionPing = applicationResource(ACTION_PING)
+ def upload = method.hasUploadBody() ? payload : null
+ if (method.hasResponseBody()) {
+ def pinged = actionPing.method(method.verb, PingResource, upload)
+ assert method.verb == pinged.verb
+ return pinged
+ } else {
+ actionPing.method(method.verb, upload)
+ return null
+ }
+ }
+
+ /**
+ * Test the stop command.
+ * Important: once executed, the AM is no longer there.
+ * This must be the last test in the sequence.
+ */
+/*
+
+ public void testStop() {
+ String target = appendToURL(appmaster, SLIDER_PATH_APPLICATION, ACTION_STOP)
+ describe "Stop URL $target"
+ URL targetUrl = new URL(target)
+ def outcome = connectionOperations.execHttpOperation(
+ HttpVerb.POST,
+ targetUrl,
+ new byte[0],
+ MediaType.TEXT_PLAIN)
+ log.info "Stopped: $outcome"
+
+ // await the shutdown
+ sleep(1000)
+
+ // now a ping is expected to fail
+ String ping = appendToURL(appmaster, SLIDER_PATH_APPLICATION, ACTION_PING)
+ URL pingUrl = new URL(ping)
+
+ repeatUntilSuccess("probe for missing registry entry",
+ this.&probePingFailing, 30000, 500,
+ [url: ping],
+ true,
+ "AM failed to shut down") {
+ def pinged = jFetchType(ACTION_PING + "?body=hello",
+ PingResource
+ )
+ fail("AM didn't shut down; Ping GET= $pinged")
+ }
+
+ }
+*/
+
+ /**
+ * Probe that spins until the url specified by "url") refuses
+ * connections
+ * @param args argument map
+ * @return the outcome
+ */
+/*
+ Outcome probePingFailing(Map args) {
+ String ping = args["url"]
+ URL pingUrl = new URL(ping)
+ try {
+ def response = pingAction(HttpVerb.HEAD, pingUrl, "should not be running")
+ return Outcome.Retry
+ } catch (IOException e) {
+ // expected
+ return Outcome.Success
+ }
+ }
+*/
+
+ public void testSuiteGetOperations() {
+
+ testCodahaleOperations()
+ testMimeTypes()
+ testLiveResources()
+ testLiveContainers();
+ testRESTModel()
+ }
+
+ public void testSuiteComplexVerbs() {
+ testPing();
+ }
+}
[28/28] incubator-slider git commit: SLIDER-743 add the counters and
resetting feature to the Nodes,
but no use of the outcome in choosing placements
Posted by el...@apache.org.
SLIDER-743 add the counters and resetting feature to the Nodes, but no use of the outcome in choosing placements
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3105ba9f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3105ba9f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3105ba9f
Branch: refs/heads/develop
Commit: 3105ba9f315059c5e957645625666975014ad4e5
Parents: 618cdc4
Author: Steve Loughran <st...@apache.org>
Authored: Wed Jan 28 21:38:51 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Jan 28 21:38:51 2015 +0000
----------------------------------------------------------------------
.../slider/server/appmaster/state/AppState.java | 4 ++-
.../server/appmaster/state/NodeEntry.java | 26 +++++++++++++++++---
.../server/appmaster/state/NodeInstance.java | 10 ++++++++
.../slider/server/appmaster/state/NodeMap.java | 11 +++++++++
.../server/appmaster/state/RoleHistory.java | 21 +++++++++++++---
5 files changed, 63 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3105ba9f/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 4b37e14..95a7ca5 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1711,9 +1711,11 @@ public class AppState {
public void resetFailureCounts() {
for (RoleStatus roleStatus : getRoleStatusMap().values()) {
int failed = roleStatus.resetFailed();
- log.debug("Resetting failure count of {}; was {}", roleStatus.getName(),
+ log.info("Resetting failure count of {}; was {}",
+ roleStatus.getName(),
failed);
}
+ roleHistory.resetFailedRecently();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3105ba9f/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
index ebddaf9..c2e203a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
@@ -53,6 +53,11 @@ public class NodeEntry {
private int startFailed;
private int failed;
/**
+ * Counter of "failed recently" events. These are all failures
+ * which have happened since it was last reset.
+ */
+ private int failedRecently;
+ /**
* Number of live nodes.
*/
private int live;
@@ -104,7 +109,7 @@ public class NodeEntry {
live = v;
}
- private void incLive() {
+ private synchronized void incLive() {
++live;
}
@@ -132,6 +137,7 @@ public class NodeEntry {
public synchronized boolean onStartFailed() {
decStarting();
++startFailed;
+ ++failedRecently;
return containerCompleted(false);
}
@@ -183,6 +189,7 @@ public class NodeEntry {
releasing = RoleHistoryUtils.decToFloor(releasing);
} else {
++failed;
+ ++failedRecently;
}
decLive();
return isAvailable();
@@ -199,7 +206,7 @@ public class NodeEntry {
this.lastUsed = lastUsed;
}
- public int getStartFailed() {
+ public synchronized int getStartFailed() {
return startFailed;
}
@@ -207,6 +214,17 @@ public class NodeEntry {
return failed;
}
+ public synchronized int getFailedRecently() {
+ return failedRecently;
+ }
+
+ /**
+ * Reset the failed recently count.
+ */
+ public void resetFailedRecently() {
+ failedRecently = 0;
+ }
+
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("NodeEntry{");
@@ -214,10 +232,10 @@ public class NodeEntry {
sb.append(", requested=").append(requested);
sb.append(", starting=").append(starting);
sb.append(", live=").append(live);
- sb.append(", failed=").append(failed);
- sb.append(", startFailed=").append(startFailed);
sb.append(", releasing=").append(releasing);
sb.append(", lastUsed=").append(lastUsed);
+ sb.append(", failedRecently=").append(failedRecently);
+ sb.append(", startFailed=").append(startFailed);
sb.append('}');
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3105ba9f/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
index bc79b71..231865e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
@@ -132,6 +132,16 @@ public class NodeInstance {
return active;
}
+
+ /**
+ * run through each entry resetting the failure count
+ */
+ public synchronized void resetFailedRecently() {
+ for (NodeEntry entry : nodeEntries) {
+ entry.resetFailedRecently();
+ }
+ }
+
@Override
public String toString() {
return hostname;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3105ba9f/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
index 570c194..fe40086 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
@@ -102,6 +102,17 @@ public class NodeMap extends HashMap<String, NodeInstance> {
}
return purged;
}
+
+
+ /**
+ * reset the failed recently counters
+ */
+ public void resetFailedRecently() {
+ for (Map.Entry<String, NodeInstance> entry : entrySet()) {
+ NodeInstance ni = entry.getValue();
+ ni.resetFailedRecently();
+ }
+ }
/**
* Clone point
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3105ba9f/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index ce2ab0a..605a4f8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -310,6 +310,14 @@ public class RoleHistory {
}
/**
+ * reset the failed recently counters
+ */
+ public synchronized void resetFailedRecently() {
+ log.info("Resetting failure history");
+ nodemap.resetFailedRecently();
+ }
+
+ /**
* Get the path used for history files
* @return the directory used for history files
*/
@@ -480,18 +488,23 @@ public class RoleHistory {
}
int roleKey = role.getKey();
NodeInstance nodeInstance = null;
-
+ // get the list of possible targets
List<NodeInstance> targets = getNodesForRoleId(roleKey);
- int cnt = targets == null ? 0 : targets.size();
+ if (targets == null) {
+ // add an empty list here for ease downstream
+ targets = new ArrayList<NodeInstance>(0);
+ }
+ int cnt = targets.size();
log.debug("There are {} node(s) to consider for {}", cnt, role.getName());
- while (targets != null && !targets.isEmpty() && nodeInstance == null) {
+ // spin until there's a candidate
+ while (!targets.isEmpty() && nodeInstance == null) {
NodeInstance head = targets.remove(0);
if (head.getActiveRoleInstances(roleKey) == 0) {
nodeInstance = head;
}
}
if (nodeInstance == null) {
- log.debug("No historical node found for {}", role.getName());
+ log.info("No historical node found for {}", role.getName());
}
return nodeInstance;
}
[19/28] incubator-slider git commit: SLIDER-740. Support
configurating HBASE_OPTS (Yang Hao via smohanty)
Posted by el...@apache.org.
SLIDER-740. Support configurating HBASE_OPTS (Yang Hao via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ea6eca12
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ea6eca12
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ea6eca12
Branch: refs/heads/develop
Commit: ea6eca122b8e81703b6ec0a7a739aaf0b8736448
Parents: ae1077d
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Jan 23 09:19:20 2015 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Fri Jan 23 09:19:20 2015 -0800
----------------------------------------------------------------------
app-packages/hbase/configuration/hbase-env.xml | 1 +
app-packages/hbase/package/scripts/params.py | 2 ++
2 files changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ea6eca12/app-packages/hbase/configuration/hbase-env.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase/configuration/hbase-env.xml b/app-packages/hbase/configuration/hbase-env.xml
index 554c3c5..79a8276 100644
--- a/app-packages/hbase/configuration/hbase-env.xml
+++ b/app-packages/hbase/configuration/hbase-env.xml
@@ -105,6 +105,7 @@ export HBASE_PID_DIR={{pid_dir}}
# Tell HBase whether it should manage it's own instance of Zookeeper or not.
export HBASE_MANAGES_ZK=false
+export HBASE_OPTS="$HBASE_OPTS {{hbase_opts}}"
{% if security_enabled %}
export HBASE_OPTS="$HBASE_OPTS -Djava.security.auth.login.config={{client_jaas_config_file}}"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ea6eca12/app-packages/hbase/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/params.py b/app-packages/hbase/package/scripts/params.py
index fc2b49f..027630b 100644
--- a/app-packages/hbase/package/scripts/params.py
+++ b/app-packages/hbase/package/scripts/params.py
@@ -41,6 +41,8 @@ metric_prop_file_name = "hadoop-metrics2-hbase.properties"
java64_home = config['hostLevelParams']['java_home']
log_dir = config['configurations']['global']['app_log_dir']
+#configuration for HBASE_OPTS
+hbase_opts = default('/configurations/hbase-env/hbase_opts', '')
master_heapsize = config['configurations']['hbase-env']['hbase_master_heapsize']
regionserver_heapsize = config['configurations']['hbase-env']['hbase_regionserver_heapsize']
[04/28] incubator-slider git commit: SLIDER-719 factoring out jersey
tests into own class
Posted by el...@apache.org.
SLIDER-719 factoring out jersey tests into own class
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/54f1f125
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/54f1f125
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/54f1f125
Branch: refs/heads/develop
Commit: 54f1f12510cd5720fa59f2acb083f3b2193a9757
Parents: c0a39d9
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jan 20 14:48:17 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jan 20 14:50:24 2015 +0000
----------------------------------------------------------------------
.../slider/agent/rest/JerseyTestDelegates.groovy | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/54f1f125/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
index 2f20bf5..32ca2cc 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
@@ -101,17 +101,17 @@ class JerseyTestDelegates extends SliderTestUtils {
public void testCodahaleOperations() throws Throwable {
describe "Codahale operations"
- getWebPage(appmaster)
- getWebPage(appmaster, SYSTEM_THREADS)
- getWebPage(appmaster, SYSTEM_HEALTHCHECK)
- getWebPage(appmaster, SYSTEM_PING)
- getWebPage(appmaster, SYSTEM_METRICS_JSON)
+ jerseyGet(appmaster)
+ jerseyGet(appmaster, SYSTEM_THREADS)
+ jerseyGet(appmaster, SYSTEM_HEALTHCHECK)
+ jerseyGet(appmaster, SYSTEM_PING)
+ jerseyGet(appmaster, SYSTEM_METRICS_JSON)
}
public void logCodahaleMetrics() {
// query Coda Hale metrics
- log.info getWebPage(appmaster, SYSTEM_HEALTHCHECK)
- log.info getWebPage(appmaster, SYSTEM_METRICS)
+ log.info jerseyGet(appmaster, SYSTEM_HEALTHCHECK)
+ log.info jerseyGet(appmaster, SYSTEM_METRICS)
}
public void testLiveResources() throws Throwable {
[24/28] incubator-slider git commit: SLIDER-755. AgentProvider
doesn't raise and exception when the agent tar isn't there SLIDER-723.
Memcached component launch fail does not propagate
Posted by el...@apache.org.
SLIDER-755. AgentProvider doesn't raise and exception when the agent tar isn't there
SLIDER-723. Memcached component launch fail does not propagate
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/657f734c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/657f734c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/657f734c
Branch: refs/heads/develop
Commit: 657f734c1104be650318241c5c0bdbcd0d61533e
Parents: 98d676b
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Mon Jan 26 17:18:02 2015 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Mon Jan 26 17:18:10 2015 -0800
----------------------------------------------------------------------
.../slider/providers/agent/AgentKeys.java | 16 +--
.../providers/agent/AgentProviderService.java | 21 ++-
.../slider/providers/agent/TestAgentEcho.groovy | 4 +-
.../appmaster/TestDelayInContainerLaunch.groovy | 1 +
.../agent/TestAgentProviderService.java | 132 ++++++++++++++++++-
5 files changed, 155 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/657f734c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index e682b13..7977181 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -25,20 +25,7 @@ public interface AgentKeys {
String PROVIDER_AGENT = "agent";
String ROLE_NODE = "echo";
- /**
- * {@value}
- */
- String REGION_SERVER = "regionserver";
- /**
- * What is the command for hbase to print a version: {@value}
- */
- String COMMAND_VERSION = "version";
- String ACTION_START = "start";
- String ACTION_STOP = "stop";
- /**
- * Config directory : {@value}
- */
- String ARG_CONFIG = "--config";
+
/**
* Template stored in the slider classpath -to use if there is
* no site-specific template
@@ -100,6 +87,7 @@ public interface AgentKeys {
String KEY_AGENT_TWO_WAY_SSL_ENABLED = "ssl.server.client.auth";
String CERT_FILE_LOCALIZATION_PATH = "certs/ca.crt";
String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
+ String TEST_RELAX_VERIFICATION = "test.relax.validation";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/657f734c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index e4680e2..439b5b6 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -353,13 +353,21 @@ public class AgentProviderService extends AbstractProviderService implements
agentImagePath = new Path(agentImage);
}
- // TODO: throw exception when agent tarball is not available
-
if (fileSystem.getFileSystem().exists(agentImagePath)) {
LocalResource agentImageRes = fileSystem.createAmResource(agentImagePath, LocalResourceType.ARCHIVE);
launcher.addLocalResource(AgentKeys.AGENT_INSTALL_DIR, agentImageRes);
} else {
- log.error("Required agent image slider-agent.tar.gz is unavailable.");
+ String msg =
+ String.format("Required agent image slider-agent.tar.gz is unavailable at %s", agentImagePath.toString());
+ MapOperations compOps = instanceDefinition.
+ getAppConfOperations().getComponent(role);
+ boolean relaxVerificationForTest = compOps != null ? Boolean.valueOf(compOps.
+ getOptionBool(AgentKeys.TEST_RELAX_VERIFICATION, false)) : false;
+ log.error(msg);
+
+ if (!relaxVerificationForTest) {
+ throw new SliderException(SliderExitCodes.EXIT_DEPLOYMENT_FAILED, msg);
+ }
}
log.info("Using {} for agent.", scriptPath);
@@ -750,6 +758,13 @@ public class AgentProviderService extends AbstractProviderService implements
&& command == Command.NOP && isMarkedAutoRestart(roleName)) {
response.setRestartEnabled(true);
}
+
+ //If INSTALL_FAILED and no INSTALL is scheduled let the agent fail
+ if(componentStatus.getState() == State.INSTALL_FAILED && command == command.NOP) {
+ log.warn("Sending terminate signal to container that failed installation: {}", label);
+ response.setTerminateAgent(true);
+ }
+
} catch (SliderException e) {
log.warn("Component instance failed operation.", e);
componentStatus.applyCommandResult(CommandResult.FAILED, command);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/657f734c/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
index 2eb39e3..973114d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
@@ -99,7 +99,9 @@ class TestAgentEcho extends AgentTestBase {
ARG_COMP_OPT, role, SCRIPT_PATH, echo_py,
ARG_COMP_OPT, role, SERVICE_NAME, "Agent",
ARG_DEFINE,
- SliderXmlConfKeys.KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED + "=false"
+ SliderXmlConfKeys.KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED + "=false",
+ ARG_COMP_OPT, role, TEST_RELAX_VERIFICATION, "true",
+
],
true, true,
true)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/657f734c/slider-core/src/test/groovy/org/apache/slider/server/appmaster/TestDelayInContainerLaunch.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/TestDelayInContainerLaunch.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/TestDelayInContainerLaunch.groovy
index 26a2c95..51e01d5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/TestDelayInContainerLaunch.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/TestDelayInContainerLaunch.groovy
@@ -107,6 +107,7 @@ class TestDelayInContainerLaunch extends AgentTestBase {
ARG_RES_COMP_OPT, role, ResourceKeys.COMPONENT_PRIORITY, "1",
ARG_COMP_OPT, role, SCRIPT_PATH, echo_py,
ARG_COMP_OPT, role, SERVICE_NAME, "Agent",
+ ARG_COMP_OPT, role, TEST_RELAX_VERIFICATION, "true",
],
true, true,
true)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/657f734c/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index dbc1b0a..3b78fb8 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -496,6 +496,136 @@ public class TestAgentProviderService {
}
@Test
+ public void testThreeInstallFailures() throws IOException, SliderException {
+ InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
+ Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
+ ConfTree tree = new ConfTree();
+ tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
+
+ Configuration conf = new Configuration();
+ AgentProviderService aps = createAgentProviderService(conf);
+ YarnRegistryViewForProviders registryViewForProviders = aps.getYarnRegistry();
+ assertNotNull(registryViewForProviders);
+
+ ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class);
+ AggregateConf instanceDefinition = new AggregateConf();
+
+ instanceDefinition.setInternal(tree);
+ instanceDefinition.setAppConf(tree);
+ instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.APP_DEF, ".");
+ instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_CONF, ".");
+ instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_VERSION, ".");
+
+ Container container = createNiceMock(Container.class);
+ String role_hm = "HBASE_MASTER";
+ SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class);
+ ContainerLauncher launcher = createNiceMock(ContainerLauncher.class);
+ Path generatedConfPath = new Path(".", "test");
+ MapOperations resourceComponent = new MapOperations();
+ MapOperations appComponent = new MapOperations();
+ Path containerTmpDirPath = new Path(".", "test");
+ FilterFileSystem mockFs = createNiceMock(FilterFileSystem.class);
+ expect(sliderFileSystem.getFileSystem())
+ .andReturn(mockFs).anyTimes();
+ expect(mockFs.exists(anyObject(Path.class))).andReturn(true).anyTimes();
+ expect(sliderFileSystem.createAmResource(anyObject(Path.class),
+ anyObject(LocalResourceType.class)))
+ .andReturn(createNiceMock(LocalResource.class)).anyTimes();
+ expect(container.getId()).andReturn(new MockContainerId(1)).anyTimes();
+ expect(container.getNodeId()).andReturn(new MockNodeId("localhost")).anyTimes();
+ StateAccessForProviders access = createNiceMock(StateAccessForProviders.class);
+
+ AgentProviderService mockAps = Mockito.spy(aps);
+ doReturn(access).when(mockAps).getAmState();
+ doReturn(metainfo).when(mockAps).getApplicationMetainfo(any(SliderFileSystem.class), anyString());
+ doReturn(new HashMap<String, DefaultConfig>()).when(mockAps).
+ initializeDefaultConfigs(any(SliderFileSystem.class), anyString(), any(Metainfo.class));
+
+
+ try {
+ doReturn(true).when(mockAps).isMaster(anyString());
+ doNothing().when(mockAps).addInstallCommand(
+ anyString(),
+ anyString(),
+ any(HeartBeatResponse.class),
+ anyString(),
+ Mockito.anyLong());
+ doReturn(conf).when(mockAps).getConfig();
+ } catch (SliderException e) {
+ }
+
+ expect(access.isApplicationLive()).andReturn(true).anyTimes();
+ ClusterDescription desc = new ClusterDescription();
+ desc.setOption(OptionKeys.ZOOKEEPER_QUORUM, "host1:2181");
+ desc.setInfo(OptionKeys.APPLICATION_NAME, "HBASE");
+ expect(access.getClusterStatus()).andReturn(desc).anyTimes();
+
+ AggregateConf aggConf = new AggregateConf();
+ ConfTreeOperations treeOps = aggConf.getAppConfOperations();
+ treeOps.getOrAddComponent("HBASE_MASTER").put(AgentKeys.WAIT_HEARTBEAT, "0");
+ treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE");
+ expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf).anyTimes();
+ expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes();
+ replay(access, ctx, container, sliderFileSystem, mockFs);
+
+ try {
+ mockAps.buildContainerLaunchContext(launcher,
+ instanceDefinition,
+ container,
+ role_hm,
+ sliderFileSystem,
+ generatedConfPath,
+ resourceComponent,
+ appComponent,
+ containerTmpDirPath);
+
+ Register reg = new Register();
+ reg.setResponseId(0);
+ reg.setLabel("mockcontainer_1___HBASE_MASTER");
+ RegistrationResponse resp = mockAps.handleRegistration(reg);
+ Assert.assertEquals(0, resp.getResponseId());
+ Assert.assertEquals(RegistrationStatus.OK, resp.getResponseStatus());
+
+ HeartBeat hb = new HeartBeat();
+ hb.setResponseId(1);
+ hb.setHostname("mockcontainer_1___HBASE_MASTER");
+ HeartBeatResponse hbr = mockAps.handleHeartBeat(hb);
+ Assert.assertEquals(2, hbr.getResponseId());
+ Assert.assertFalse(hbr.isTerminateAgent());
+
+ hb.setResponseId(2);
+ CommandReport cr = new CommandReport();
+ cr.setRole("HBASE_MASTER");
+ cr.setRoleCommand("INSTALL");
+ cr.setStatus("FAILED");
+ hb.setReports(Arrays.asList(cr));
+ hbr = mockAps.handleHeartBeat(hb);
+ Assert.assertEquals(3, hbr.getResponseId());
+ Assert.assertFalse(hbr.isTerminateAgent());
+
+ hb.setResponseId(3);
+ hbr = mockAps.handleHeartBeat(hb);
+ Assert.assertEquals(4, hbr.getResponseId());
+ Assert.assertFalse(hbr.isTerminateAgent());
+
+ //Third failure triggers a stop to the agent
+ hb.setResponseId(4);
+ hbr = mockAps.handleHeartBeat(hb);
+ Assert.assertEquals(5, hbr.getResponseId());
+ Assert.assertTrue(hbr.isTerminateAgent());
+ Mockito.verify(mockAps, Mockito.times(3)).addInstallCommand(anyString(),
+ anyString(),
+ any(HeartBeatResponse.class),
+ anyString(),
+ Mockito.anyLong());
+ } catch (SliderException he) {
+ log.warn(he.getMessage());
+ } catch (IOException he) {
+ log.warn(he.getMessage());
+ }
+ }
+
+ @Test
public void testAgentStateStarted() throws IOException, SliderException {
AggregateConf instanceDefinition = prepareConfForAgentStateTests();
AgentProviderService mockAps = prepareProviderServiceForAgentStateTests();
@@ -1056,7 +1186,7 @@ public class TestAgentProviderService {
FilterFileSystem mockFs = createNiceMock(FilterFileSystem.class);
expect(sliderFileSystem.getFileSystem())
.andReturn(mockFs).anyTimes();
- expect(mockFs.exists(anyObject(Path.class))).andReturn(true);
+ expect(mockFs.exists(anyObject(Path.class))).andReturn(true).anyTimes();
expect(sliderFileSystem.createAmResource(anyObject(Path.class),
anyObject(LocalResourceType.class)))
.andReturn(createNiceMock(LocalResource.class)).anyTimes();
[10/28] incubator-slider git commit: SLIDER-719 REST client tested on
secure clusters. The jersey + httpclient combination does not work,
but the others do
Posted by el...@apache.org.
SLIDER-719 REST client tested on secure clusters. The jersey + httpclient combination does not work, but the others do
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0097ce92
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0097ce92
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0097ce92
Branch: refs/heads/develop
Commit: 0097ce923a217db00095fba6273ce7f339064cb7
Parents: 4418179
Author: Steve Loughran <st...@apache.org>
Authored: Wed Jan 21 17:54:39 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Jan 21 17:54:39 2015 +0000
----------------------------------------------------------------------
pom.xml | 6 ++
slider-core/pom.xml | 6 ++
.../core/restclient/HttpOperationResponse.java | 5 ++
.../restclient/UrlConnectionOperations.java | 2 +
.../rest/application/ApplicationResource.java | 1 -
.../agent/rest/JerseyTestDelegates.groovy | 25 ++++--
.../slider/agent/rest/RestTestDelegates.groovy | 20 ++++-
.../slider/agent/rest/TestStandaloneREST.groovy | 91 +++++++++++++++++---
.../apache/slider/test/SliderTestUtils.groovy | 44 ++++++++--
.../funtest/lifecycle/AgentWebPagesIT.groovy | 62 +++++++++----
10 files changed, 212 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0097ce92/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b206d66..5c18f77 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1204,6 +1204,12 @@
</dependency>
<dependency>
+ <groupId>com.sun.jersey.contribs</groupId>
+ <artifactId>jersey-apache-client</artifactId>
+ <version>${jersey.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
<version>${jersey.version}</version>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0097ce92/slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index be4d95c..43a1193 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -407,6 +407,12 @@
</dependency>
<dependency>
+ <groupId>com.sun.jersey.contribs</groupId>
+ <artifactId>jersey-apache-client</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0097ce92/slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java b/slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java
index a5357a2..0266223 100644
--- a/slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java
@@ -18,12 +18,17 @@
package org.apache.slider.core.restclient;
+import java.util.List;
+import java.util.Map;
+
/**
* A response for use as a return value from operations
*/
public class HttpOperationResponse {
public int responseCode;
+ public long lastModified;
public String contentType;
public byte[] data;
+ public Map<String, List<String>> headers;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0097ce92/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
index 65a4117..6e29c55 100644
--- a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
@@ -133,7 +133,9 @@ public class UrlConnectionOperations extends Configured {
}
resultCode = conn.getResponseCode();
+ outcome.lastModified = conn.getLastModified();
outcome.contentType = conn.getContentType();
+ outcome.headers = conn.getHeaderFields();
InputStream stream = conn.getErrorStream();
if (stream == null) {
stream = conn.getInputStream();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0097ce92/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
index d72a486..c2c83a0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
@@ -366,7 +366,6 @@ public class ApplicationResource extends AbstractSliderResource {
@HEAD
@Path(ACTION_PING)
- @Produces({APPLICATION_JSON})
public Object actionPingHead(@Context HttpServletRequest request,
@Context UriInfo uriInfo) {
mark("HEAD", SLIDER_SUBPATH_APPLICATION, ACTION_PING);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0097ce92/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
index 0699800..09788e1 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
@@ -19,8 +19,10 @@
package org.apache.slider.agent.rest
import com.sun.jersey.api.client.Client
-import com.sun.jersey.api.client.WebResource
+import com.sun.jersey.api.client.ClientResponse
import com.sun.jersey.api.client.UniformInterfaceException
+import com.sun.jersey.api.client.WebResource
+import com.sun.jersey.client.impl.ClientRequestImpl
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.webapp.NotFoundException
@@ -30,14 +32,10 @@ import org.apache.slider.api.types.SerializedContainerInformation
import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.conf.ConfTree
import org.apache.slider.core.conf.ConfTreeOperations
-import org.apache.slider.core.restclient.HttpOperationResponse
import org.apache.slider.core.restclient.HttpVerb
-import org.apache.slider.server.appmaster.web.rest.RestPaths
import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource
import org.apache.slider.server.appmaster.web.rest.application.resources.PingResource
-import org.apache.slider.test.Outcome
import org.apache.slider.test.SliderTestUtils
-import org.glassfish.grizzly.servlet.ver25.WebResourceCollectionType
import javax.ws.rs.core.MediaType
@@ -99,6 +97,7 @@ class JerseyTestDelegates extends SliderTestUtils {
public <T> T jExec(HttpVerb method, WebResource resource, Class<T> c) {
try {
assert c
+ resource.accept(MediaType.APPLICATION_JSON_TYPE)
(T) resource.method(method.verb, c)
} catch (UniformInterfaceException ex) {
uprateFaults(method, resource, ex)
@@ -203,8 +202,21 @@ class JerseyTestDelegates extends SliderTestUtils {
return tree
}
+
+ public void testMimeTypes() throws Throwable {
+ describe "Mime Types"
+
+ WebResource resource = applicationResource(LIVE_RESOURCES)
+ def response = resource.get(ClientResponse)
+ response.headers.each {key, val -> log.info("$key: $val")}
+ log.info response.toString()
+ assert response.type.equals(MediaType.APPLICATION_JSON_TYPE)
+ }
+
+
public void testLiveResources() throws Throwable {
describe "Live Resources"
+
ConfTreeOperations tree = jGetConfigTree(LIVE_RESOURCES)
log.info tree.toString()
@@ -267,7 +279,7 @@ class JerseyTestDelegates extends SliderTestUtils {
Map<String, SerializedComponentInformation> components =
jFetchType(LIVE_COMPONENTS, HashMap)
// two components
- assert components.size() == 1
+ assert components.size() >= 1
log.info "${components}"
SerializedComponentInformation amComponentInfo =
@@ -451,6 +463,7 @@ class JerseyTestDelegates extends SliderTestUtils {
public void testSuiteGetOperations() {
testCodahaleOperations()
+ testMimeTypes()
testLiveResources()
testLiveContainers();
testRESTModel()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0097ce92/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
index dd23926..f90e1ad 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
@@ -18,18 +18,18 @@
package org.apache.slider.agent.rest
-import com.sun.jersey.api.client.Client
+import com.sun.jersey.api.client.ClientResponse
import com.sun.jersey.api.client.WebResource
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.webapp.NotFoundException
+import org.apache.http.entity.ContentType
import org.apache.slider.api.StateValues
import org.apache.slider.api.types.SerializedComponentInformation
import org.apache.slider.api.types.SerializedContainerInformation
import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.conf.ConfTree
import org.apache.slider.core.conf.ConfTreeOperations
-import org.apache.slider.core.registry.docstore.PublishedConfiguration
import org.apache.slider.core.restclient.HttpOperationResponse
import org.apache.slider.core.restclient.HttpVerb
import org.apache.slider.core.restclient.UrlConnectionOperations
@@ -40,6 +40,7 @@ import org.apache.slider.test.Outcome
import org.apache.slider.test.SliderTestUtils
import javax.ws.rs.core.MediaType
+import java.nio.charset.Charset
import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES
import static org.apache.slider.api.StatusKeys.*
@@ -80,6 +81,18 @@ class RestTestDelegates extends SliderTestUtils {
log.info getWebPage(appmaster, SYSTEM_METRICS)
}
+
+ public void testMimeTypes() throws Throwable {
+ describe "Mime Types"
+ HttpOperationResponse response= executeGet(
+ appendToURL(appmaster,
+ SLIDER_PATH_APPLICATION, LIVE_RESOURCES))
+ response.headers.each { key, val -> log.info("$key $val")}
+ log.info "Content type: ${response.contentType}"
+ assert response.contentType.contains(MediaType.APPLICATION_JSON_TYPE.toString())
+ }
+
+
public void testLiveResources() throws Throwable {
describe "Live Resources"
ConfTreeOperations tree = fetchConfigTree(appmaster, LIVE_RESOURCES)
@@ -140,7 +153,7 @@ class RestTestDelegates extends SliderTestUtils {
Map<String, SerializedComponentInformation> components =
fetchType(HashMap, appmaster, LIVE_COMPONENTS)
// two components
- assert components.size() == 1
+ assert components.size() >= 1
log.info "${components}"
SerializedComponentInformation amComponentInfo =
@@ -301,6 +314,7 @@ class RestTestDelegates extends SliderTestUtils {
public void testSuiteGetOperations() {
testCodahaleOperations()
+ testMimeTypes()
testLiveResources()
testLiveContainers();
testRESTModel()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0097ce92/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
index 9cea46c..0abf264 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
@@ -18,20 +18,28 @@
package org.apache.slider.agent.rest
+import com.sun.jersey.api.client.Client
+import com.sun.jersey.api.client.config.ClientConfig
+import com.sun.jersey.api.json.JSONConfiguration
+import com.sun.jersey.client.apache.ApacheHttpClient
+import com.sun.jersey.client.apache.ApacheHttpClientHandler
+import com.sun.jersey.client.apache.config.DefaultApacheHttpClientConfig
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.commons.httpclient.HttpClient
+import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
-import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource
-import org.apache.slider.client.SliderClient
import org.apache.slider.core.main.ServiceLauncher
-
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
+import org.apache.slider.core.restclient.HttpOperationResponse
import org.junit.Test
-import static org.apache.slider.server.appmaster.management.MetricsKeys.*
+import static org.apache.slider.server.appmaster.management.MetricsKeys.METRICS_LOGGING_ENABLED
+import static org.apache.slider.server.appmaster.management.MetricsKeys.METRICS_LOGGING_LOG_INTERVAL
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.*
@CompileStatic
@Slf4j
@@ -59,8 +67,9 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
addToTeardown(client);
ApplicationReport report = waitForClusterLive(client)
+ def proxyAM = report.trackingUrl
def directAM = report.originalTrackingUrl
-
+
// set up url config to match
initHttpTestSupport(launcher.configuration)
@@ -73,9 +82,8 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
def metrics = GET(directAM, SYSTEM_METRICS)
log.info metrics
}
-
- def proxyAM = report.trackingUrl
+
GET(proxyAM)
log.info GET(proxyAM, SYSTEM_PING)
@@ -87,33 +95,88 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
SliderXmlConfKeys.X_DEV_INSECURE_WS,
true)
+ describe "Direct response headers from AM Web resources"
+ def liveResUrl = appendToURL(directAM,
+ SLIDER_PATH_APPLICATION, LIVE_RESOURCES);
+ HttpOperationResponse response = executeGet(liveResUrl)
+ response.headers.each { key, val -> log.info("$key $val") }
+ log.info "Content type: ${response.contentType}"
+
+ describe "proxied response headers from AM Web resources"
+ response = executeGet(appendToURL(proxyAM,
+ SLIDER_PATH_APPLICATION, LIVE_RESOURCES))
+ response.headers.each { key, val -> log.info("$key $val") }
+ log.info "Content type: ${response.contentType}"
+
+
+ describe "Proxy Jersey Tests"
JerseyTestDelegates proxyJerseyTests =
- new JerseyTestDelegates(proxyAM, createJerseyClient())
+ new JerseyTestDelegates(proxyAM, createUGIJerseyClient())
proxyJerseyTests.testSuiteGetOperations()
+ describe "Direct Jersey Tests"
+
JerseyTestDelegates directJerseyTests =
- new JerseyTestDelegates(directAM, createJerseyClient())
+ new JerseyTestDelegates(directAM, createUGIJerseyClient())
directJerseyTests.testSuiteGetOperations()
directJerseyTests.testSuiteComplexVerbs()
-
- RestTestDelegates proxied = new RestTestDelegates(proxyAM)
+
+ describe "Direct Tests"
+
RestTestDelegates direct = new RestTestDelegates(directAM)
-
- proxied.testSuiteGetOperations()
direct.testSuiteGetOperations()
direct.testSuiteComplexVerbs()
+
+ describe "Proxy Tests"
+
+ RestTestDelegates proxied = new RestTestDelegates(proxyAM)
+ proxied.testSuiteGetOperations()
if (!wsBackDoorRequired) {
// and via the proxy
proxied.testSuiteComplexVerbs()
}
+
+/* DISABLED: this client does not pass the tests.
+
+ // http client direct
+ describe "Proxied Jersey Apache HttpClient"
+ JerseyTestDelegates proxiedHttpClientJersey =
+ new JerseyTestDelegates(proxyAM, createJerseyClientHttpClient())
+ proxiedHttpClientJersey.testSuiteGetOperations()
+
+ describe "Direct Jersey Apache HttpClient"
+ JerseyTestDelegates directHttpClientJersey =
+ new JerseyTestDelegates(directAM, createJerseyClientHttpClient())
+ directHttpClientJersey.testSuiteGetOperations()
+ directHttpClientJersey.testSuiteComplexVerbs()
+ */
+ createJerseyClientHttpClient()
+ // log the metrics to show what's up
direct.logCodahaleMetrics();
// this MUST be the final test
direct.testStop();
}
+ /**
+ * Create Jersey client with URL handling by way
+ * of the Apache HttpClient classes.
+ * @return a Jersey client
+ */
+ public static Client createJerseyClientHttpClient() {
+ def httpclient = new HttpClient(new MultiThreadedHttpConnectionManager());
+ httpclient.httpConnectionManager.params.connectionTimeout = 10000;
+ ClientConfig clientConfig = new DefaultApacheHttpClientConfig();
+ clientConfig.features[JSONConfiguration.FEATURE_POJO_MAPPING] = Boolean.TRUE;
+
+ def handler = new ApacheHttpClientHandler(httpclient, clientConfig);
+
+ def client = new ApacheHttpClient(handler)
+ client.followRedirects = true
+ return client;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0097ce92/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 897d7e3..1ed340f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -22,6 +22,7 @@ import com.sun.jersey.api.client.Client
import com.sun.jersey.api.client.config.ClientConfig
import com.sun.jersey.api.client.config.DefaultClientConfig
import com.sun.jersey.api.json.JSONConfiguration
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler
import groovy.json.JsonOutput
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
@@ -34,11 +35,11 @@ import org.apache.hadoop.fs.FileStatus
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
import org.apache.hadoop.net.NetUtils
+import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.hadoop.service.ServiceStateException
import org.apache.hadoop.util.Shell
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.hadoop.yarn.webapp.ForbiddenException
import org.apache.hadoop.yarn.webapp.NotFoundException
import org.apache.slider.api.ClusterDescription
@@ -59,6 +60,7 @@ import org.apache.slider.core.main.ServiceLaunchException
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.persist.JsonSerDeser
import org.apache.slider.core.registry.docstore.PublishedConfigSet
+import org.apache.slider.core.restclient.HttpOperationResponse
import org.apache.slider.core.restclient.UgiJerseyBinding
import org.apache.slider.core.restclient.UrlConnectionOperations
import org.apache.slider.server.appmaster.web.HttpCacheHeaders
@@ -616,17 +618,41 @@ class SliderTestUtils extends Assert {
}
/**
- * Create Jersey client
+ * Create Jersey client with UGI integration
* @return
*/
- public static Client createJerseyClient() {
+ public static Client createUGIJerseyClient() {
assertHttpSupportInitialized()
- ClientConfig clientConfig = new DefaultClientConfig();
- clientConfig.features[JSONConfiguration.FEATURE_POJO_MAPPING] =Boolean.TRUE;
+ ClientConfig clientConfig = createJerseyClientConfig()
return new Client(jerseyBinding.handler, clientConfig);
}
/**
+ * Create Jersey client with URL handling by way
+ * of the java.net classes. This DOES NOT have any SPNEGO
+ * integration. If used to query a secure cluster via the
+ * RM Proxy, it MUST fail.
+ * @return a basic Jersey client
+ */
+ public static Client createBasicJerseyClient() {
+ ClientConfig clientConfig = createJerseyClientConfig()
+ return new Client(new URLConnectionClientHandler(),
+ clientConfig);
+ }
+
+
+ /**
+ * Create a jersey client config with the settings needed for tests
+ * (e.g. POJO mappings)
+ * @return a client config
+ */
+ public static ClientConfig createJerseyClientConfig() {
+ ClientConfig clientConfig = new DefaultClientConfig();
+ clientConfig.features[JSONConfiguration.FEATURE_POJO_MAPPING] = Boolean.TRUE;
+ return clientConfig
+ }
+
+ /**
* Fetches a web page asserting that the response code is between 200 and 400.
* Will error on 400 and 500 series response codes and let 200 and 300 through.
*
@@ -640,14 +666,18 @@ class SliderTestUtils extends Assert {
* @return body of response
*/
public static String getWebPage(String path, Closure connectionChecks = null) {
+ HttpOperationResponse outcome = executeGet(path)
+ return new String(outcome.data);
+ }
+
+ public static HttpOperationResponse executeGet(String path) {
assert path
assertHttpSupportInitialized()
log.info("Fetching HTTP content at $path");
URL url = new URL(path)
def outcome = connectionOperations.execGet(url)
- String body = new String(outcome.data)
- return body;
+ return outcome
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0097ce92/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
index 95352d0..2d1c863 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
@@ -20,6 +20,9 @@ package org.apache.slider.funtest.lifecycle
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.yarn.webapp.ForbiddenException
+import org.apache.slider.agent.rest.JerseyTestDelegates
import org.apache.slider.agent.rest.RestTestDelegates
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.SliderXmlConfKeys
@@ -72,7 +75,7 @@ public class AgentWebPagesIT extends AgentCommandTestBase
SliderShell shell = createTemplatedSliderApplication(CLUSTER,
APP_TEMPLATE,
APP_RESOURCE2,
- [Arguments.ARG_OPTION,
+ [ARG_OPTION,
RestTestDelegates.TEST_GLOBAL_OPTION,
RestTestDelegates.TEST_GLOBAL_OPTION_PRESENT],
launchReportFile)
@@ -92,34 +95,55 @@ public class AgentWebPagesIT extends AgentCommandTestBase
def report = loadAppReport(liveReportFile)
assert report.url
- def appmaster = report.url
+ def proxyAM = report.url
// get the root page,
- getWebPage(appmaster)
+ getWebPage(proxyAM)
- def realappmaster = report.origTrackingUrl;
+ def directAM = report.origTrackingUrl;
// now attempt direct-to-AM pings
- RestTestDelegates proxied = new RestTestDelegates(appmaster)
- RestTestDelegates direct = new RestTestDelegates(realappmaster)
+ RestTestDelegates direct = new RestTestDelegates(directAM)
- proxied.testCodahaleOperations()
- direct.testCodahaleOperations()
- proxied.testLiveResources()
+ direct.testSuiteGetOperations()
+ direct.testSuiteComplexVerbs()
- proxied.testRESTModel()
+ // and via the proxy
+ RestTestDelegates proxied = new RestTestDelegates(proxyAM)
+ proxied.testSuiteGetOperations()
+ if (!wsBackDoorRequired) {
+ proxied.testSuiteComplexVerbs()
+ }
+ proxied.logCodahaleMetrics();
- direct.testRestletGetOperations()
- proxied.testRestletGetOperations()
+ describe "Proxy Jersey Tests"
- // PUT & POST &c direct
- direct.testPing()
- if (!wsBackDoorRequired) {
- // and via the proxy
- proxied.testRESTModel()
+ JerseyTestDelegates proxyJerseyTests =
+ new JerseyTestDelegates(proxyAM, createUGIJerseyClient())
+ proxyJerseyTests.testSuiteGetOperations()
+
+ describe "Direct Jersey Tests"
+ JerseyTestDelegates directJerseyTests =
+ new JerseyTestDelegates(directAM, createUGIJerseyClient())
+ directJerseyTests.testSuiteGetOperations()
+ directJerseyTests.testSuiteComplexVerbs()
+
+ if (UserGroupInformation.securityEnabled) {
+ describe "Insecure Proxy Tests against a secure cluster"
+
+ try {
+ String rootpage = fetchWebPageRaisedErrorCodes(proxyAM);
+ fail(" expected a 401, got $rootpage")
+ } catch (ForbiddenException expected) {
+ // expected
+ }
+
+ // these tests use the Jersey client without the Hadoop-specific
+ // SPNEGO
+ JerseyTestDelegates baseicJerseyClientTests =
+ new JerseyTestDelegates(proxyAM, createBasicJerseyClient())
+ baseicJerseyClientTests.testSuiteGetOperations()
}
- direct.logCodahaleMetrics();
-
// finally, stop the AM
direct.testStop();
}
[26/28] incubator-slider git commit: SLIDER-756 create
accumulo-slider client script
Posted by el...@apache.org.
SLIDER-756 create accumulo-slider client script
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/96f890b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/96f890b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/96f890b1
Branch: refs/heads/develop
Commit: 96f890b1b13985567505c10492326062f3e6dffd
Parents: f5b0940
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Tue Jan 27 09:38:50 2015 -0800
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Tue Jan 27 09:38:59 2015 -0800
----------------------------------------------------------------------
app-packages/accumulo/README.md | 53 ++-
app-packages/accumulo/accumulo-slider | 53 +++
app-packages/accumulo/accumulo-slider.py | 378 +++++++++++++++++++
app-packages/accumulo/pom.xml | 2 +
app-packages/accumulo/src/assembly/accumulo.xml | 8 +
.../funtest/accumulo/AccumuloProxyIT.groovy | 2 +-
.../funtest/accumulo/AccumuloScriptIT.groovy | 88 +++++
.../funtest/accumulo/AccumuloSliderShell.groovy | 52 +++
.../slider/funtest/framework/ShellBase.groovy | 323 ++++++++++++++++
.../slider/funtest/framework/SliderShell.groovy | 305 +--------------
10 files changed, 950 insertions(+), 314 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96f890b1/app-packages/accumulo/README.md
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/README.md b/app-packages/accumulo/README.md
index e44870e..9395c5a 100644
--- a/app-packages/accumulo/README.md
+++ b/app-packages/accumulo/README.md
@@ -15,20 +15,18 @@
limitations under the License.
-->
-# How to create a Slider package for Accumulo?
+# Creating a Slider app package for Accumulo
mvn clean package -DskipTests -Paccumulo-app-package-maven
-
+
OR
mvn clean package -DskipTests -Paccumulo-app-package -Dpkg.version=1.6.1 \
-Dpkg.name=accumulo-1.6.1-bin.tar.gz -Dpkg.src=/local/path/to/tarball
-App package can be found in
+The app package can be found in
app-packages/accumulo/target/slider-accumulo-app-package-*.zip
-
-
In the first case, the version number of the app package will match the
slider version, and in the second case it will match the `pkg.version`
@@ -38,26 +36,49 @@ Verify the content using
zip -Tv slider-accumulo-app-package*.zip
-`appConfig-default.json` and `resources-default.json` are not required to be packaged.
-These files are included as reference configuration for Slider apps and are suitable
-for a one-node cluster.
+`appConfig-default.json` and `resources-default.json` are not required to be
+packaged. These files are included as reference configuration for Slider apps
+and are suitable for a one-node cluster.
In the maven packaging case, the version of Accumulo used for the app package
can be adjusted by adding a flag such as
-Daccumulo.version=1.5.1
-**Note that the LICENSE.txt and NOTICE.txt that are bundled with the app
+**Note** that the LICENSE.txt and NOTICE.txt that are bundled with the app
package are designed for Accumulo 1.6.0 only and may need to be modified to be
applicable for other versions of the app package.
-Note also that the sample `appConfig-default.json` provided only works with Accumulo 1.6.
-For Accumulo 1.5 the instance.volumes property must be replaced with
-`instance.dfs.dir` (and it cannot use the provided variable `${DEFAULT_DATA_DIR}`
-which is an HDFS URI).
+Note also that the sample `appConfig-default.json` provided only works with
+Accumulo 1.6 or greater. For Accumulo 1.5 the instance.volumes property must be
+replaced with `instance.dfs.dir` (and it cannot use the provided variable
+`${DEFAULT_DATA_DIR}` which is an HDFS URI).
A less descriptive file name can be specified with
-`-Dapp.package.name=accumulo_160` which would create a file `accumulo_160.zip`
+`-Dapp.package.name=accumulo_160` which would create a file `accumulo_160.zip`.
+
+# Installing an Accumulo Client
+
+The Accumulo app package provides scripts to assist in client interactions with
+an Accumulo instance running on Slider. These can be extracted as follows.
+
+ unzip slider-accumulo-app-package*zip accumulo-slider
+ unzip slider-accumulo-app-package*zip accumulo-slider.py
+
+To install an Accumulo client, use the following command:
+
+ SLIDER_HOME=</path/to/slider> ./accumulo-slider --app <clusterName> install <dir>
+
+If the SLIDER_CONF_DIR is not at $SLIDER_HOME/conf, it should also be set.
+The dir specified will then contain an Accumulo installation that can be used
+to connect to the cluster.
+
+Examples of other commands that may be issued are:
+
+ SLIDER_HOME=</path/to/slider> ACCUMULO_HOME=</path/to/accumulo> ./accumulo-slider --app <clusterName> shell
+ SLIDER_HOME=</path/to/slider> ACCUMULO_HOME=</path/to/accumulo> ./accumulo-slider --app <clusterName> --appconf <accumulo_conf_dir> getconf
+ SLIDER_HOME=</path/to/slider> ACCUMULO_HOME=</path/to/accumulo> ./accumulo-slider --app <clusterName> quicklinks
+ SLIDER_HOME=</path/to/slider> ACCUMULO_HOME=</path/to/accumulo> ./accumulo-slider --app <clusterName> proxies
# Building Native Libraries
@@ -88,8 +109,8 @@ currently reside may have restrictions on the import, possession, use, and/or
re-export to another country, of encryption software. BEFORE using any
encryption software, please check your country's laws, regulations and
policies concerning the import, possession, or use, and re-export of encryption
-software, to see if this is permitted. See [http://www.wassenaar.org/](http://www.wassenaar.org/) for more
-information.
+software, to see if this is permitted. See
+[http://www.wassenaar.org/](http://www.wassenaar.org/) for more information.
The U.S. Government Department of Commerce, Bureau of Industry and Security
(BIS), has classified this software as Export Commodity Control Number (ECCN)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96f890b1/app-packages/accumulo/accumulo-slider
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/accumulo-slider b/app-packages/accumulo/accumulo-slider
new file mode 100644
index 0000000..8f9cb06
--- /dev/null
+++ b/app-packages/accumulo/accumulo-slider
@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Resolve links - $0 may be a softlink
+PRG="${0}"
+
+while [ -h "${PRG}" ]; do
+ ls=`ls -ld "${PRG}"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "${PRG}"`/"$link"
+ fi
+done
+
+# find python >= 2.6
+if [ -a /usr/bin/python2.6 ]; then
+ PYTHON=/usr/bin/python2.6
+fi
+
+if [ -z "$PYTHON" ]; then
+ PYTHON=/usr/bin/python
+fi
+
+# check for version
+majversion=`$PYTHON -V 2>&1 | awk '{print $2}' | cut -d'.' -f1`
+minversion=`$PYTHON -V 2>&1 | awk '{print $2}' | cut -d'.' -f2`
+numversion=$(( 10 * $majversion + $minversion))
+if (( $numversion < 26 )); then
+ echo "Need python version > 2.6"
+ exit 1
+fi
+
+ACCUMULO_SLIDER_BIN_DIR=`dirname ${PRG}`
+
+$PYTHON ${ACCUMULO_SLIDER_BIN_DIR}/accumulo-slider.py $@
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96f890b1/app-packages/accumulo/accumulo-slider.py
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/accumulo-slider.py b/app-packages/accumulo/accumulo-slider.py
new file mode 100644
index 0000000..1eccf59
--- /dev/null
+++ b/app-packages/accumulo/accumulo-slider.py
@@ -0,0 +1,378 @@
+#!/usr/bin/python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import json
+import glob
+import tempfile
+import subprocess
+import shutil
+
+SLIDER_DIR = os.getenv('SLIDER_HOME', None)
+if SLIDER_DIR == None or (not os.path.exists(SLIDER_DIR)):
+ print "Unable to find SLIDER_HOME. Please configure SLIDER_HOME before running accumulo-slider"
+ sys.exit(1)
+SLIDER_CMD = os.path.join(SLIDER_DIR, 'bin', 'slider.py')
+
+TMP_DIR = os.path.join(tempfile.gettempdir(), "accumulo-slider-tmp."+str(os.getpid()))
+
+CMD_OPTS = {}
+
+def call(cmd):
+ print "Running: " + " ".join(cmd)
+ retcode = subprocess.call(cmd)
+ if retcode != 0:
+ raise Exception("return code from running %s was %d" % (cmd[0], retcode))
+
+def exec_accumulo_command(command, args=[]):
+ ACCUMULO_DIR = os.getenv('ACCUMULO_HOME', None)
+ if ACCUMULO_DIR == None or (not os.path.exists(ACCUMULO_DIR)):
+ print "Unable to find ACCUMULO_HOME. Please configure ACCUMULO_HOME before running accumulo-slider " + command
+ sys.exit(1)
+ ACCUMULO_CMD = os.path.join(ACCUMULO_DIR, 'bin', 'accumulo')
+
+ confdir = get_all_conf()
+ if command == 'shell' or command == 'admin':
+ cmd = [ACCUMULO_CMD, command, '--config-file', os.path.join(confdir, 'client.conf')] + list(args)
+ else:
+ cmd = [ACCUMULO_CMD, command] + list(args)
+ print "Setting ACCUMULO_CONF_DIR=" + confdir
+ os.putenv("ACCUMULO_CONF_DIR", confdir)
+ call(cmd)
+
+def exec_tool_command(jarfile, mainclass, args=[]):
+ ACCUMULO_DIR = os.getenv('ACCUMULO_HOME', None)
+ if ACCUMULO_DIR == None or (not os.path.exists(ACCUMULO_DIR)):
+ print "Unable to find ACCUMULO_HOME. Please configure ACCUMULO_HOME before running accumulo-slider tool"
+ sys.exit(1)
+ TOOL_CMD = os.path.join(ACCUMULO_DIR, 'bin', 'tool.sh')
+
+ confdir = get_all_conf()
+ cmd = [TOOL_CMD, jarfile, mainclass] + list(args)
+ print "Setting ACCUMULO_CONF_DIR=" + confdir
+ os.putenv("ACCUMULO_CONF_DIR", confdir)
+ call(cmd)
+
+def jar(jarfile, *args):
+ """Syntax: [accumulo-slider --app appname[ --appconf confdir] jar jarfile [mainclass]]
+ OR: [accumulo-slider --appconf confdir jar jarfile [mainclass]]
+ Runs a class from a specified jar
+ """
+ exec_accumulo_command("jar", jarfile, args=args)
+
+def classname(mainclass, *args):
+ """Syntax: [accumulo-slider --app appname[ --appconf confdir] classname mainclass]
+ OR: [accumulo-slider --appconf confdir classname mainclass]
+ Runs a specified class on the existing accumulo classpath
+ """
+ exec_accumulo_command(mainclass, args=args)
+
+def shell(*args):
+ """Syntax: [accumulo-slider --app appname[ --appconf confdir] shell]
+ OR: [accumulo-slider --appconf confdir shell]
+ Runs an accumulo shell
+ """
+ exec_accumulo_command("shell", args=args)
+
+def admin(*args):
+ """Syntax: [accumulo-slider --app appname[ --appconf confdir] admin cmd]
+ OR: [accumulo-slider --appconf confdir admin cmd]
+ Executes an admin command (run without cmd argument for a list of commands)
+ """
+ exec_accumulo_command("admin", args=args)
+
+def classpath(*args):
+ """Syntax: [accumulo-slider --app appname[ --appconf confdir] classpath]
+ OR: [accumulo-slider --appconf confdir classpath]
+ Prints the classpath of the accumulo client install
+ """
+ exec_accumulo_command("classpath", args=args)
+
+def info(*args):
+ """Syntax: [accumulo-slider --app appname[ --appconf confdir] info]
+ OR: [accumulo-slider --appconf confdir info]
+ Prints information about an accumulo instance (monitor, masters, and zookeepers)
+ """
+ exec_accumulo_command("info", args=args)
+
+def rfileinfo(*args):
+ """Syntax: [accumulo-slider --app appname[ --appconf confdir] rfile-info rfilename]
+ OR: [accumulo-slider --appconf confdir rfile-info rfilename]
+ Prints information about a specified RFile
+ """
+ exec_accumulo_command("rfile-info", args=args)
+
+def logininfo(*args):
+ """Syntax: [accumulo-slider --app appname[ --appconf confdir] login-info]
+ OR: [accumulo-slider --appconf confdir login-info]
+ Prints the supported authentication token types for the accumulo instance
+ """
+ exec_accumulo_command("login-info", args=args)
+
+def createtoken(*args):
+ """Syntax: [accumulo-slider --app appname[ --appconf confdir] create-token]
+ OR: [accumulo-slider --appconf confdir create-token]
+ Saves a given accumulo authentication token to a file
+ """
+ exec_accumulo_command("create-token", args=args)
+
+def version(*args):
+ """Syntax: [accumulo-slider --app appname[ --appconf confdir] version]
+ OR: [accumulo-slider --appconf confdir version]
+ Prints the version of the accumulo client install
+ """
+ exec_accumulo_command("version", args=args)
+
+def tool(jarfile, mainclass, *args):
+ """Syntax: [accumulo-slider --app appname[ --appconf confdir] tool jarfile classname]
+ OR: [accumulo-slider --appconf confdir tool jarfile classname]
+ Runs a mapreduce job using accumulo's tool.sh
+ """
+ exec_tool_command(jarfile, mainclass, args=args)
+
+def quicklinks():
+ """Syntax: [accumulo-slider --app appname quicklinks]
+ Prints the quicklinks information of accumulo-slider registry
+ """
+ global CMD_OPTS
+ if not 'app_name' in CMD_OPTS.keys():
+ print_usage()
+ sys.exit(1)
+
+ cmd = [SLIDER_CMD, "registry", "--getconf", "quicklinks", "--format", "json",
+ "--name", CMD_OPTS['app_name']]
+
+ if 'user' in CMD_OPTS.keys():
+ cmd.append( "--user "+CMD_OPTS['user'])
+
+ call(cmd)
+
+def proxies():
+ """Syntax: [accumulo-slider --app appname proxies]
+ Prints the componentinstancedata information of accumulo-slider registry
+ """
+ global CMD_OPTS
+ if not 'app_name' in CMD_OPTS.keys():
+ print_usage()
+ sys.exit(1)
+
+ cmd = [SLIDER_CMD, "registry", "--getconf", "componentinstancedata",
+ "--format", "json", "--name", CMD_OPTS['app_name']]
+
+ if 'user' in CMD_OPTS.keys():
+ cmd.append( "--user "+CMD_OPTS['user'])
+
+ call(cmd)
+
+def install(dir):
+ """Syntax: [accumulo-slider --app appname install dir]
+ Installs a fully configured accumulo client in the specified dir
+ The resulting client may be used on its own without accumulo-slider
+ """
+ global CMD_OPTS
+ if not 'app_name' in CMD_OPTS.keys():
+ print_usage()
+ sys.exit(1)
+ if os.path.exists(dir):
+ raise Exception("Install dir must not exist: " + dir)
+
+ global TMP_DIR
+ workdir = os.path.join(TMP_DIR, 'install-work-dir')
+
+ statusfile = os.path.join(workdir, 'status.json')
+ cmd = [SLIDER_CMD, "status", CMD_OPTS['app_name'], "--out", statusfile]
+ call(cmd)
+
+ infile = open(statusfile)
+ try:
+ content = json.load(infile)
+ finally:
+ infile.close()
+
+ appdef = content['options']['application.def']
+ appdeffile = appdef[appdef.rfind('/')+1:]
+ cmd = ["hadoop", "fs", "-copyToLocal", appdef, workdir]
+ call(cmd)
+
+ cmd = ["unzip", os.path.join(workdir, appdeffile), "-d", workdir]
+ call(cmd)
+
+ gzfile = glob.glob(os.path.join(workdir, 'package', 'files', 'accumulo*gz'))
+ if len(gzfile) != 1:
+ raise Exception("got " + gzfile + " from glob")
+ cmd = ["tar", "xvzf", gzfile[0], '-C', workdir]
+ call(cmd)
+
+ tmp_accumulo = glob.glob(os.path.join(workdir, 'accumulo-[0-9]*'))
+ if len(tmp_accumulo) != 1:
+ raise Exception("got " + tmp_accumulo + " from glob")
+ tmp_accumulo = tmp_accumulo[0]
+
+ confdir = os.path.join(tmp_accumulo, 'conf')
+ tmpconf = os.path.join(workdir, 'conf-tmp')
+ shutil.move(confdir, tmpconf)
+ make_conf(os.path.join(tmpconf, 'templates'), confdir)
+
+ libdir = os.path.join(tmp_accumulo, 'lib')
+ for jar in glob.glob(os.path.join(workdir, 'package', 'files', '*jar')):
+ shutil.move(jar, libdir)
+ shutil.move(tmp_accumulo, dir)
+
+def get_all_conf():
+ """Syntax: [accumulo-slider --app appname [--appconf confdir] getconf]
+ Downloads configuration for an accumulo instance
+ If --appconf is specified, creates the specified conf dir and populates it
+ """
+ ACCUMULO_CONF_DIR = os.getenv('ACCUMULO_CONF_DIR', None)
+ if ACCUMULO_CONF_DIR == None or (not os.path.exists(ACCUMULO_CONF_DIR)):
+ ACCUMULO_DIR = os.getenv('ACCUMULO_HOME', None)
+ if ACCUMULO_DIR == None or (not os.path.exists(ACCUMULO_DIR)):
+ print "Unable to find ACCUMULO_HOME. Please configure ACCUMULO_HOME before running this command"
+ sys.exit(1)
+ ACCUMULO_CONF_DIR = os.path.join(ACCUMULO_DIR, 'conf')
+
+ global CMD_OPTS
+ global TMP_DIR
+ confdir = os.path.join(TMP_DIR, 'conf')
+ if 'app_conf' in CMD_OPTS.keys():
+ confdir = CMD_OPTS['app_conf']
+ if os.path.exists(confdir):
+ print "Using existing app conf instead of downloading it again: " + confdir
+ return confdir
+
+ if not 'app_name' in CMD_OPTS.keys():
+ print_usage()
+ sys.exit(1)
+
+ make_conf(ACCUMULO_CONF_DIR, confdir)
+ return confdir
+
+def make_conf(oldconf, newconf):
+ client_file = os.path.join(newconf, 'client.conf')
+ site_file = os.path.join(newconf, 'accumulo-site.xml')
+ env_file = os.path.join(newconf, 'accumulo-env.sh')
+ env_json = os.path.join(newconf, 'accumulo-env.json')
+
+ print "Copying base conf from " + oldconf + " to " + newconf
+ shutil.copytree(oldconf, newconf)
+
+ try_remove(client_file)
+ try_remove(site_file)
+ try_remove(env_file)
+ try_remove(env_json)
+
+ get_conf("client", "properties", client_file)
+ get_conf("accumulo-site", "xml", site_file)
+ get_conf("accumulo-env", "json", env_json)
+
+ infile = open(env_json)
+ outfile = open(env_file, 'w')
+ try:
+ content = json.load(infile)
+ outfile.write(content['content'])
+ finally:
+ outfile.close()
+ infile.close()
+
+def try_remove(path):
+ try:
+ os.remove(path)
+ except:
+ if os.path.exists(path):
+ raise
+
+def get_conf(confname, fileformat, destfile):
+ if os.path.exists(destfile):
+ print "Conf file " + destfile + " already exists, remove it to re-download"
+ return
+
+ cmd = [SLIDER_CMD, "registry", "--getconf", confname, "--format",
+ fileformat, "--dest", destfile, "--name", CMD_OPTS['app_name']]
+ if 'user' in CMD_OPTS.keys():
+ cmd.append("--user " + CMD_OPTS['user'])
+
+ call(cmd)
+ if not os.path.exists(destfile):
+ raise Exception("Failed to read slider deployed accumulo config " + confname)
+
+def print_commands():
+ """Print all client commands and link to documentation"""
+ print "Commands:\n\t", "\n\t".join(sorted(COMMANDS.keys()))
+ print "\nHelp:", "\n\thelp", "\n\thelp <command>"
+
+def print_usage(command=None):
+ """Print one help message or list of available commands"""
+ if command != None:
+ if COMMANDS.has_key(command):
+ print (COMMANDS[command].__doc__ or
+ "No documentation provided for <%s>" % command)
+ else:
+ print "<%s> is not a valid command" % command
+ else:
+ print "Usage:"
+ print "accumulo-slider --app <name>[ --appconf <confdir> --user <username>] <command>"
+ print " The option --appconf creates a conf dir that can be reused;"
+ print " on subsequent calls to accumulo-slider, --app can be left off if"
+ print " --appconf is specified. If --appconf is not specified, a"
+ print " temporary conf dir is created each time accumulo-slider is run."
+ print_commands()
+
+def unknown_command(*args):
+ print "Unknown command: [accumulo-slider %s]" % ' '.join(sys.argv[1:])
+ print_usage()
+
+COMMANDS = {"shell": shell, "tool": tool, "admin": admin, "classpath": classpath,
+ "info": info, "version": version, "jar": jar, "classname": classname,
+ "quicklinks" : quicklinks, "proxies": proxies, "getconf": get_all_conf,
+ "rfile-info": rfileinfo, "login-info": logininfo, "create-token": createtoken,
+ "install": install, "help": print_usage}
+
+def parse_config_opts(args):
+ curr = args[:]
+ curr.reverse()
+ global CMD_OPTS
+ args_list = []
+ while len(curr) > 0:
+ token = curr.pop()
+ if token == "--app":
+ CMD_OPTS['app_name'] = curr.pop() if (len(curr) != 0) else None
+ elif token == "--user":
+ CMD_OPTS['user'] = curr.pop() if (len(curr) != 0) else None
+ elif token == "--appconf":
+ CMD_OPTS['app_conf'] = curr.pop() if (len(curr) != 0) else None
+ else:
+ args_list.append(token)
+ return args_list
+
+def main():
+ args = parse_config_opts(sys.argv[1:])
+ if len(args) < 1:
+ print_usage()
+ sys.exit(-1)
+ COMMAND = args[0]
+ ARGS = args[1:]
+ try:
+ (COMMANDS.get(COMMAND, unknown_command))(*ARGS)
+ finally:
+ global CMD_OPTS
+ if os.path.exists(TMP_DIR):
+ print "Cleaning up tmp dir " + TMP_DIR
+ shutil.rmtree(TMP_DIR)
+
+if __name__ == "__main__":
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96f890b1/app-packages/accumulo/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index 23ea3d6..4467294 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -38,6 +38,7 @@
<slider.bin.dir>../../slider-assembly/target/slider-${project.version}-all/slider-${project.version}</slider.bin.dir>
<test.app.pkg.dir>${project.build.directory}</test.app.pkg.dir>
<test.app.resources.dir>${project.build.directory}/test-config</test.app.resources.dir>
+ <test.app.scripts.dir>${project.build.directory}/${app.package.name}</test.app.scripts.dir>
<!-- these properties are used in the default and the test appConfigs -->
<hadoop.dir>/usr/lib/hadoop</hadoop.dir>
<hdfs.dir>/usr/lib/hadoop-hdfs</hdfs.dir>
@@ -214,6 +215,7 @@
<test.app.pkg.file>${app.package.name}.zip</test.app.pkg.file>
<test.app.pkg.name>ACCUMULO</test.app.pkg.name>
<test.app.resources.dir>${test.app.resources.dir}</test.app.resources.dir>
+ <test.app.scripts.dir>${test.app.scripts.dir}</test.app.scripts.dir>
</systemPropertyVariables>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96f890b1/app-packages/accumulo/src/assembly/accumulo.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/assembly/accumulo.xml b/app-packages/accumulo/src/assembly/accumulo.xml
index 7be1942..3bc0e1f 100644
--- a/app-packages/accumulo/src/assembly/accumulo.xml
+++ b/app-packages/accumulo/src/assembly/accumulo.xml
@@ -24,6 +24,7 @@
<id>accumulo_v${accumulo.version}</id>
<formats>
<format>zip</format>
+ <format>dir</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
@@ -47,6 +48,12 @@
<fileMode>0755</fileMode>
</file>
<file>
+ <source>accumulo-slider</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>false</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
<source>${project.build.directory}/${work.dir}/${pkg.name}</source>
<outputDirectory>package/files</outputDirectory>
<filtered>false</filtered>
@@ -72,6 +79,7 @@
<exclude>appConfig-default.json</exclude>
<exclude>appConfig-secured-default.json</exclude>
<exclude>metainfo.xml</exclude>
+ <exclude>accumulo-slider</exclude>
</excludes>
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96f890b1/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloProxyIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloProxyIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloProxyIT.groovy
index 730355f..d25811f 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloProxyIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloProxyIT.groovy
@@ -71,7 +71,7 @@ class AccumuloProxyIT extends AccumuloBasicIT {
return proxies
} catch (Exception e) {
caught = e;
- log.info("Got exception trying to read quicklinks")
+ log.info("Got exception trying to read proxies")
if (tries-- == 0) {
break
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96f890b1/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptIT.groovy
new file mode 100644
index 0000000..2ba06af
--- /dev/null
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptIT.groovy
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.funtest.accumulo
+
+import groovy.util.logging.Slf4j
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.client.SliderClient
+import org.apache.slider.funtest.framework.AccumuloSliderShell
+import org.junit.BeforeClass
+
+import java.nio.ByteBuffer
+
+@Slf4j
+class AccumuloScriptIT extends AccumuloBasicIT {
+ public static final String RESOURCES_DIR = sysprop("test.app.resources.dir")
+ public static final String ACCUMULO_HOME = RESOURCES_DIR + "/install"
+ public static final String ACCUMULO_CONF = sysprop("test.app.resources.dir") + "/conf"
+
+ @Override
+ public String getClusterName() {
+ return "test_script";
+ }
+
+ @Override
+ public String getDescription() {
+ return "Test accumulo-slider client script $clusterName"
+ }
+
+ @BeforeClass
+ public static void setShell() {
+ AccumuloSliderShell.scriptFile = new File(sysprop("test.app.scripts.dir"),
+ "accumulo-slider").canonicalFile
+ AccumuloSliderShell.setEnv("SLIDER_HOME", SLIDER_TAR_DIR)
+ AccumuloSliderShell.setEnv("SLIDER_CONF_DIR", SLIDER_CONF_DIR)
+ AccumuloSliderShell.setEnv("ACCUMULO_HOME", ACCUMULO_HOME)
+ }
+
+ @Override
+ public void clusterLoadOperations(ClusterDescription cd, SliderClient sliderClient) {
+ String clusterName = getClusterName()
+ AccumuloSliderShell.run(0, "--app $clusterName quicklinks")
+ AccumuloSliderShell.run(0, "--app $clusterName install $ACCUMULO_HOME")
+ AccumuloSliderShell.run(0, "--app $clusterName --appconf $ACCUMULO_CONF getconf")
+ runBoth("shell -u $USER -p $PASSWORD -e tables")
+ runBoth("login-info")
+
+ AccumuloSliderShell info = runOne("info")
+ String monitor = getMonitorUrl(sliderClient, getClusterName())
+ assert info.outputContains(monitor.substring(monitor.indexOf("://")+3)),
+ "accumulo info output did not contain monitor"
+
+ runOne("version")
+ runOne("classpath")
+ runOne("create-token -u $USER -p $PASSWORD -f $RESOURCES_DIR/token")
+ runOne("admin checkTablets")
+ runOne("admin listInstances")
+ runOne("admin ping")
+ runOne("admin dumpConfig -a -d $RESOURCES_DIR")
+ runOne("admin volumes")
+
+ // TODO: test tool, jar, classname, rfile-info
+ // runOne("shell -u $USER -p $PASSWORD -e \"createtable testtable\"")
+ }
+
+ public AccumuloSliderShell runOne(String cmd) {
+ return AccumuloSliderShell.run(0, "--appconf $ACCUMULO_CONF $cmd")
+ }
+
+ public void runBoth(String cmd) {
+ String clusterName = getClusterName()
+ AccumuloSliderShell.run(0, "--app $clusterName $cmd")
+ AccumuloSliderShell.run(0, "--appconf $ACCUMULO_CONF $cmd")
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96f890b1/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSliderShell.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSliderShell.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSliderShell.groovy
new file mode 100644
index 0000000..09b8a39
--- /dev/null
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSliderShell.groovy
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.framework
+
+import org.apache.bigtop.itest.shell.Shell
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.common.tools.SliderUtils
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+
+class AccumuloSliderShell extends ShellBase {
+
+ public static File scriptFile;
+
+ /**
+ * Build the command
+ * @param commands
+ */
+ AccumuloSliderShell(Collection<String> commands) {
+ super(scriptFile.absolutePath + " " + commands.join(" "))
+ }
+
+ /**
+ * Exec any slider command
+ * @param conf
+ * @param commands
+ * @return the shell
+ */
+ public static AccumuloSliderShell run(int exitCode, String command) {
+ AccumuloSliderShell shell = new AccumuloSliderShell([command])
+ shell.execute(exitCode);
+ return shell
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96f890b1/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/ShellBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/ShellBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/ShellBase.groovy
new file mode 100644
index 0000000..112486f
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/ShellBase.groovy
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.framework
+
+import org.apache.bigtop.itest.shell.Shell
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.common.tools.SliderUtils
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+
+class ShellBase extends Shell {
+ private static final Logger log = LoggerFactory.getLogger(ShellBase.class);
+ private static final Logger LOG = log;
+
+ public static final String BASH = '/bin/bash -s'
+ public static final String CMD = 'cmd'
+
+ /**
+ * Environment variables
+ */
+ private static final Map<String, String> environment = [:]
+
+ private final String command
+
+ /**
+ * Build the command
+ * @param commands
+ */
+ ShellBase(String cmd) {
+ super(org.apache.hadoop.util.Shell.WINDOWS ? CMD : BASH)
+ command = cmd
+ }
+
+ /**
+ * Exec the command
+ * @return the script exit code
+ */
+ int execute() {
+ log.info(command)
+ List<String> commandLine = buildEnvCommands()
+
+ commandLine << command
+ if (org.apache.hadoop.util.Shell.WINDOWS) {
+ // Ensure the errorlevel returned by last call is set for the invoking shell
+ commandLine << "@echo ERRORLEVEL=%ERRORLEVEL%"
+ commandLine << "@exit %ERRORLEVEL%"
+ }
+ String script = commandLine.join("\n")
+ log.debug(script)
+ exec(script);
+ signCorrectReturnCode()
+ return ret;
+ }
+
+ public String getPathElementSeparator() {
+ File.pathSeparator
+ }
+
+ public static boolean isWindows() {
+ return org.apache.hadoop.util.Shell.WINDOWS
+ }
+
+ /**
+ * Set an environment variable
+ * @param var variable name
+ * @param val value
+ */
+ public static void setEnv(String var, Object val) {
+ environment[var] = val.toString()
+ }
+
+ /**
+ * Get an environment variable
+ * @param var variable name
+ * @return the value or null
+ */
+ public static String getEnv(String var) {
+ return environment[var]
+ }
+
+ /**
+ * Build up a list of environment variable setters from the
+ * env variables
+ * @return a list of commands to set up the env on the target system.
+ */
+ public static List<String> buildEnvCommands() {
+ List<String> commands = []
+ environment.each { String var, String val ->
+ commands << env(var, val)
+ }
+ return commands
+ }
+
+ /**
+ * Add an environment variable
+ * @param var variable
+ * @param val value (which will be stringified)
+ * @return an env variable command
+ */
+ static String env(String var, Object val) {
+ if (isWindows()) {
+ return "set " + var + "=${val.toString()}"
+ } else {
+ return "export " + var + "=${val.toString()};"
+ }
+ }
+
+ /**
+ * Fix up the return code so that a value of 255 is mapped back to -1
+ * @return twos complement return code from an unsigned byte
+ */
+ int signCorrectReturnCode() {
+ ret = signCorrect(ret)
+ }
+
+ /**
+ * Execute expecting a specific exit code
+ * @param expectedExitCode the expected exit code
+ */
+ void execute(int expectedExitCode) {
+ execute()
+ assertExitCode(expectedExitCode)
+ }
+
+ /**
+ * Sign-correct a process exit code
+ * @param exitCode the incoming exit code
+ * @return the sign-corrected version
+ */
+ public static int signCorrect(int exitCode) {
+ return (exitCode << 24) >> 24;
+ }
+
+ @Override
+ public String toString() {
+ return ret + " =>" + command
+ }
+
+ /**
+ * Dump the command, return code and outputs to the log.
+ * stdout is logged at info; stderr at error.
+ */
+ public void dumpOutput() {
+ log.error(toString())
+ log.error("return code = ${signCorrectReturnCode()}")
+ if (out.size() != 0) {
+ log.info("\n<stdout>\n${stdoutHistory}\n</stdout>");
+ }
+ if (err.size() != 0) {
+ log.error("\n<stderr>\n${stdErrHistory}\n</stderr>");
+ }
+ }
+
+ /**
+ * Get the stderr history
+ * @return the history
+ */
+ public String getStdErrHistory() {
+ return err.join('\n')
+ }
+
+ /**
+ * Get the stdout history
+ * @return the history
+ */
+ public String getStdoutHistory() {
+ return out.join('\n')
+ }
+
+ /**
+ * Assert the shell exited with a given error code
+ * if not the output is printed and an assertion is raised
+ * @param errorCode expected error code
+ */
+ public void assertExitCode(int errorCode, String extra="") {
+ if (this.ret != errorCode) {
+ dumpOutput()
+ throw new SliderException(ret,
+ "Expected exit code of command ${command} : ${errorCode} - actual=${ret} $extra")
+ }
+ }
+
+ /**
+ * Execute shell script consisting of as many Strings as we have arguments,
+ * NOTE: individual strings are concatenated into a single script as though
+ * they were delimited with new line character. All quoting rules are exactly
+ * what one would expect in standalone shell script.
+ *
+ * After executing the script its return code can be accessed as getRet(),
+ * stdout as getOut() and stderr as getErr(). The script itself can be accessed
+ * as getScript()
+ * WARNING: it isn't thread safe
+ * @param args shell script split into multiple Strings
+ * @return Shell object for chaining
+ */
+ Shell exec(Object... args) {
+ Process proc = "$shell".execute()
+ script = args.join("\n")
+ ByteArrayOutputStream baosErr = new ByteArrayOutputStream(4096);
+ ByteArrayOutputStream baosOut = new ByteArrayOutputStream(4096);
+ proc.consumeProcessOutput(baosOut, baosErr)
+
+ Thread.start {
+ def writer = new PrintWriter(new BufferedOutputStream(proc.out))
+ writer.println(script)
+ writer.flush()
+ writer.close()
+ }
+
+ proc.waitFor()
+ ret = proc.exitValue()
+
+ out = streamToLines(baosOut)
+ err = streamToLines(baosErr)
+
+ if (LOG.isTraceEnabled()) {
+ if (ret != 0) {
+ LOG.trace("return: $ret");
+ }
+ if (out.size() != 0) {
+ LOG.trace("\n<stdout>\n${out.join('\n')}\n</stdout>");
+ }
+
+ if (err.size() != 0) {
+ LOG.trace("\n<stderr>\n${err.join('\n')}\n</stderr>");
+ }
+ }
+ return this
+ }
+
+ /**
+ * Convert a stream to lines in an array
+ * @param out output stream
+ * @return the list of entries
+ */
+ protected List<String> streamToLines(ByteArrayOutputStream out) {
+ if (out.size() != 0) {
+ return out.toString().split('\n');
+ } else {
+ return [];
+ }
+ }
+
+ public String findLineEntry(String[] locaters) {
+ int index = 0;
+ def output = out +"\n"+ err
+ for (String str in output) {
+ if (str.contains("\"" + locaters[index] + "\"")) {
+ if (locaters.size() == index + 1) {
+ return str;
+ } else {
+ index++;
+ }
+ }
+ }
+
+ return null;
+ }
+
+ public boolean outputContains(
+ String lookThisUp,
+ int n = 1) {
+ int count = 0
+ def output = out + "\n" + err
+ for (String str in output) {
+ int subCount = countString(str, lookThisUp)
+ count = count + subCount
+ if (count == n) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public static int countString(String str, String search) {
+ int count = 0
+ if (SliderUtils.isUnset(str) || SliderUtils.isUnset(search)) {
+ return count
+ }
+
+ int index = str.indexOf(search, 0)
+ while (index >= 0) {
+ ++count
+ index = str.indexOf(search, index + 1)
+ }
+ return count
+ }
+
+ public findLineEntryValue(String[] locaters) {
+ String line = findLineEntry(locaters);
+
+ if (line != null) {
+ log.info("Parsing {} for value.", line)
+ int dividerIndex = line.indexOf(":");
+ if (dividerIndex > 0) {
+ String value = line.substring(dividerIndex + 1).trim()
+ if (value.endsWith(",")) {
+ value = value.subSequence(0, value.length() - 1)
+ }
+ return value;
+ }
+ }
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96f890b1/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
index ae40d6a..9270e8c 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -21,147 +21,40 @@ package org.apache.slider.funtest.framework
import org.apache.bigtop.itest.shell.Shell
import org.apache.slider.core.exceptions.SliderException
import org.apache.slider.common.tools.SliderUtils
+import org.junit.BeforeClass
import org.slf4j.Logger
import org.slf4j.LoggerFactory
-class SliderShell extends Shell {
- private static final Logger log = LoggerFactory.getLogger(SliderShell.class);
- private static final Logger LOG = log;
-
- public static final String BASH = '/bin/bash -s'
- public static final String CMD = 'cmd'
-
+class SliderShell extends ShellBase {
/**
* Configuration directory, shared across all instances. Not marked as volatile,
* assumed set up during @BeforeClass
*/
public static File confDir;
-
- public static File scriptFile;
-
- public File shellScript;
-
- public static final List<String> slider_classpath_extra = []
- /**
- * Environment varaibles
- */
- protected static final Map<String, String> environment = [:]
+ public static File scriptFile;
- final String command
+ public static final List<String> slider_classpath_extra = []
/**
* Build the command
* @param commands
*/
SliderShell(Collection<String> commands) {
- super(org.apache.hadoop.util.Shell.WINDOWS ? CMD : BASH)
+ super(scriptFile.absolutePath + " " + commands.join(" "))
assert confDir != null;
- assert scriptFile != null;
- shellScript = scriptFile;
- command = scriptFile.absolutePath + " " + commands.join(" ")
- }
- /**
- * Exec the command
- * @return the script exit code
- */
- int execute() {
- log.info(command)
setEnv(FuntestProperties.ENV_SLIDER_CONF_DIR, confDir)
if (!slider_classpath_extra.empty) {
setEnv(FuntestProperties.ENV_SLIDER_CLASSPATH_EXTRA,
- SliderUtils.join(slider_classpath_extra,
- pathElementSeparator,
- false))
- }
- List<String> commandLine = buildEnvCommands()
-
- commandLine << command
- if (org.apache.hadoop.util.Shell.WINDOWS) {
- // Ensure the errorlevel returned by last call is set for the invoking shell
- commandLine << "@echo ERRORLEVEL=%ERRORLEVEL%"
- commandLine << "@exit %ERRORLEVEL%"
- }
- String script = commandLine.join("\n")
- log.debug(script)
- exec(script);
- signCorrectReturnCode()
- return ret;
- }
-
- public String getPathElementSeparator() {
- File.pathSeparator
- }
-
- public static boolean isWindows() {
- return org.apache.hadoop.util.Shell.WINDOWS
- }
-
- /**
- * Set an environment variable
- * @param var variable name
- * @param val value
- */
- public static void setEnv(String var, Object val) {
- environment[var] = val.toString()
- }
-
- /**
- * Get an environment variable
- * @param var variable name
- * @return the value or null
- */
- public static String getEnv(String var) {
- return environment[var]
- }
-
- /**
- * Build up a list of environment variable setters from the
- * env variables
- * @return a list of commands to set up the env on the target system.
- */
- public static List<String> buildEnvCommands() {
- List<String> commands = []
- environment.each { String var, String val ->
- commands << env(var, val)
- }
- return commands
- }
-
- /**
- * Add an environment variable
- * @param var variable
- * @param val value (which will be stringified)
- * @return an env variable command
- */
- static String env(String var, Object val) {
- if (isWindows()) {
- return "set " + var + "=${val.toString()}"
- } else {
- return "export " + var + "=${val.toString()};"
+ SliderUtils.join(slider_classpath_extra,
+ pathElementSeparator,
+ false))
}
}
/**
- * Fix up the return code so that a value of 255 is mapped back to -1
- * @return twos complement return code from an unsigned byte
- */
- int signCorrectReturnCode() {
- ret = signCorrect(ret)
- }
-
- /**
- * Execute expecting a specific exit code
- * @param expectedExitCode the expected exit code
- */
- void execute(int expectedExitCode) {
- execute()
- assertExitCode(expectedExitCode)
- }
-
- /**
* Exec any slider command
* @param conf
* @param commands
@@ -172,186 +65,4 @@ class SliderShell extends Shell {
shell.execute(exitCode);
return shell
}
-
- /**
- * Sign-correct a process exit code
- * @param exitCode the incoming exit code
- * @return the sign-corrected version
- */
- public static int signCorrect(int exitCode) {
- return (exitCode << 24) >> 24;
- }
-
- @Override
- public String toString() {
- return ret + " =>" + command
- }
-
- /**
- * Dump the command, return code and outputs to the log.
- * stdout is logged at info; stderr at error.
- */
- public void dumpOutput() {
- log.error(toString())
- log.error("return code = ${signCorrectReturnCode()}")
- if (out.size() != 0) {
- log.info("\n<stdout>\n${stdoutHistory}\n</stdout>");
- }
- if (err.size() != 0) {
- log.error("\n<stderr>\n${stdErrHistory}\n</stderr>");
- }
- }
-
- /**
- * Get the stderr history
- * @return the history
- */
- public String getStdErrHistory() {
- return err.join('\n')
- }
-
- /**
- * Get the stdout history
- * @return the history
- */
- public String getStdoutHistory() {
- return out.join('\n')
- }
-
- /**
- * Assert the shell exited with a given error code
- * if not the output is printed and an assertion is raised
- * @param errorCode expected error code
- */
- public void assertExitCode(int errorCode, String extra="") {
- if (this.ret != errorCode) {
- dumpOutput()
- throw new SliderException(ret,
- "Expected exit code of command ${command} : ${errorCode} - actual=${ret} $extra")
- }
- }
-
- /**
- * Execute shell script consisting of as many Strings as we have arguments,
- * NOTE: individual strings are concatenated into a single script as though
- * they were delimited with new line character. All quoting rules are exactly
- * what one would expect in standalone shell script.
- *
- * After executing the script its return code can be accessed as getRet(),
- * stdout as getOut() and stderr as getErr(). The script itself can be accessed
- * as getScript()
- * WARNING: it isn't thread safe
- * @param args shell script split into multiple Strings
- * @return Shell object for chaining
- */
- Shell exec(Object... args) {
- Process proc = "$shell".execute()
- script = args.join("\n")
- ByteArrayOutputStream baosErr = new ByteArrayOutputStream(4096);
- ByteArrayOutputStream baosOut = new ByteArrayOutputStream(4096);
- proc.consumeProcessOutput(baosOut, baosErr)
-
- Thread.start {
- def writer = new PrintWriter(new BufferedOutputStream(proc.out))
- writer.println(script)
- writer.flush()
- writer.close()
- }
-
- proc.waitFor()
- ret = proc.exitValue()
-
- out = streamToLines(baosOut)
- err = streamToLines(baosErr)
-
- if (LOG.isTraceEnabled()) {
- if (ret != 0) {
- LOG.trace("return: $ret");
- }
- if (out.size() != 0) {
- LOG.trace("\n<stdout>\n${out.join('\n')}\n</stdout>");
- }
-
- if (err.size() != 0) {
- LOG.trace("\n<stderr>\n${err.join('\n')}\n</stderr>");
- }
- }
- return this
- }
-
- /**
- * Convert a stream to lines in an array
- * @param out output stream
- * @return the list of entries
- */
- protected List<String> streamToLines(ByteArrayOutputStream out) {
- if (out.size() != 0) {
- return out.toString().split('\n');
- } else {
- return [];
- }
- }
-
- public String findLineEntry(String[] locaters) {
- int index = 0;
- def output = out +"\n"+ err
- for (String str in output) {
- if (str.contains("\"" + locaters[index] + "\"")) {
- if (locaters.size() == index + 1) {
- return str;
- } else {
- index++;
- }
- }
- }
-
- return null;
- }
-
- public boolean outputContains(
- String lookThisUp,
- int n = 1) {
- int count = 0
- def output = out + "\n" + err
- for (String str in output) {
- int subCount = countString(str, lookThisUp)
- count = count + subCount
- if (count == n) {
- return true;
- }
- }
- return false;
- }
-
- public static int countString(String str, String search) {
- int count = 0
- if (SliderUtils.isUnset(str) || SliderUtils.isUnset(search)) {
- return count
- }
-
- int index = str.indexOf(search, 0)
- while (index >= 0) {
- ++count
- index = str.indexOf(search, index + 1)
- }
- return count
- }
-
- public findLineEntryValue(String[] locaters) {
- String line = findLineEntry(locaters);
-
- if (line != null) {
- log.info("Parsing {} for value.", line)
- int dividerIndex = line.indexOf(":");
- if (dividerIndex > 0) {
- String value = line.substring(dividerIndex + 1).trim()
- if (value.endsWith(",")) {
- value = value.subSequence(0, value.length() - 1)
- }
- return value;
- }
- }
- return null;
- }
-
}
[11/28] incubator-slider git commit: Merge branch
'feature/SLIDER-719_Create_slider_REST_client_library' into develop
Posted by el...@apache.org.
Merge branch 'feature/SLIDER-719_Create_slider_REST_client_library' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1282802f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1282802f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1282802f
Branch: refs/heads/develop
Commit: 1282802fd66924effb0e4c22b93295734fe203a0
Parents: 901f38c 0097ce9
Author: Steve Loughran <st...@apache.org>
Authored: Wed Jan 21 17:54:48 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Jan 21 17:54:48 2015 +0000
----------------------------------------------------------------------
pom.xml | 11 +-
slider-core/pom.xml | 6 +
.../core/restclient/HttpOperationResponse.java | 5 +
.../core/restclient/UgiJerseyBinding.java | 98 ++++
.../restclient/UrlConnectionOperations.java | 18 +-
.../rest/application/ApplicationResource.java | 1 -
.../agent/rest/JerseyTestDelegates.groovy | 475 +++++++++++++++++++
.../slider/agent/rest/RestTestDelegates.groovy | 41 +-
.../slider/agent/rest/TestStandaloneREST.groovy | 131 +++--
.../agent/TestAgentAMManagementWS.groovy | 4 +-
.../publisher/TestPublisherRestResources.groovy | 2 +-
.../apache/slider/test/SliderTestUtils.groovy | 101 +++-
.../funtest/lifecycle/AgentWebPagesIT.groovy | 62 ++-
13 files changed, 872 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
[02/28] incubator-slider git commit: SLIDER-719 we MAY now have
Jersey and SPNEGO hooked up, but tests need to verify this properly
Posted by el...@apache.org.
SLIDER-719 we MAY now have Jersey and SPNEGO hooked up, but tests need to verify this properly
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c97f7603
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c97f7603
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c97f7603
Branch: refs/heads/develop
Commit: c97f760310b1eccd8f92268f9fa33ad1925f4aec
Parents: 61f4743
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jan 19 14:32:48 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jan 20 14:50:23 2015 +0000
----------------------------------------------------------------------
.../core/restclient/UgiJerseyBinding.java | 21 +++++++++++++++++++-
.../restclient/UrlConnectionOperations.java | 14 +++++++++++--
.../slider/agent/rest/RestTestDelegates.groovy | 18 ++++++++---------
.../slider/agent/rest/TestStandaloneREST.groovy | 2 +-
.../apache/slider/test/SliderTestUtils.groovy | 1 -
.../funtest/lifecycle/AgentWebPagesIT.groovy | 5 ++++-
6 files changed, 46 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c97f7603/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
index 6af8e23..6f002ab 100644
--- a/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
@@ -21,6 +21,7 @@ package org.apache.slider.core.restclient;
import com.google.common.base.Preconditions;
import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import java.io.IOException;
@@ -33,7 +34,7 @@ import java.net.URL;
* Usage: create an instance, then when creating a Jersey <code>Client</code>
* pass in to the constructor the handler provided by {@link #getHandler()}
*
- * @see https://jersey.java.net/apidocs/1.17/jersey/com/sun/jersey/client/urlconnection/HttpURLConnectionFactory.html
+ * see <a href="https://jersey.java.net/apidocs/1.17/jersey/com/sun/jersey/client/urlconnection/HttpURLConnectionFactory.html">Jersey docs</a>
*/
public class UgiJerseyBinding implements
HttpURLConnectionFactory {
@@ -52,6 +53,15 @@ public class UgiJerseyBinding implements
}
/**
+ * Create an instance off the configuration. The SPNEGO policy
+ * is derived from the current UGI settings.
+ * @param conf config
+ */
+ public UgiJerseyBinding(Configuration conf) {
+ this(new UrlConnectionOperations(conf));
+ }
+
+ /**
* Get a URL connection.
* @param url
* @return the connection
@@ -74,6 +84,15 @@ public class UgiJerseyBinding implements
public URLConnectionClientHandler getHandler() {
return handler;
}
+
+ /**
+ * Get the SPNEGO flag (as found in the operations instance
+ * @return the spnego policy
+ */
+ public boolean isUseSpnego() {
+ return operations.isUseSpnego();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c97f7603/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
index c51914a..65a4117 100644
--- a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
@@ -48,10 +49,19 @@ public class UrlConnectionOperations extends Configured {
private boolean useSpnego = false;
+ /**
+ * Create an instance off the configuration. The SPNEGO policy
+ * is derived from the current UGI settings.
+ * @param conf config
+ */
public UrlConnectionOperations(Configuration conf) {
super(conf);
connectionFactory = URLConnectionFactory
.newDefaultURLConnectionFactory(conf);
+ if (UserGroupInformation.isSecurityEnabled()) {
+ log.debug("SPNEGO is enabled");
+ setUseSpnego(true);
+ }
}
@@ -66,10 +76,10 @@ public class UrlConnectionOperations extends Configured {
/**
* Opens a url with read and connect timeouts
*
- * @param url
- * to open
+ * @param url to open
* @return URLConnection
* @throws IOException
+ * @throws AuthenticationException authentication failure
*/
public HttpURLConnection openConnection(URL url) throws
IOException,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c97f7603/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
index 0826204..bf009c2 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
@@ -97,15 +97,6 @@ class RestTestDelegates extends SliderTestUtils {
assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_RELEASING)
}
-
- public void testRestletOperations() throws Throwable {
- Client client = createJerseyClient()
- String path = appendToURL(application, LIVE_RESOURCES)
- WebResource webResource = client.resource(path)
- webResource.type(MediaType.APPLICATION_JSON)
- .get(ConfTree.class);
- }
-
public void testLiveContainers() throws Throwable {
describe "Application REST ${LIVE_CONTAINERS}"
@@ -202,6 +193,15 @@ class RestTestDelegates extends SliderTestUtils {
TEST_GLOBAL_OPTION_PRESENT
}
+
+ public void testRestletGetOperations() throws Throwable {
+ Client client = createJerseyClient()
+ String path = appendToURL(application, LIVE_RESOURCES)
+ WebResource webResource = client.resource(path)
+ webResource.type(MediaType.APPLICATION_JSON)
+ .get(ConfTree.class);
+ }
+
public void testPing() {
// GET
String ping = appendToURL(appmaster, SLIDER_PATH_APPLICATION, ACTION_PING)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c97f7603/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
index 22bc4ae..fd676df 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
@@ -90,7 +90,7 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
RestTestDelegates proxied = new RestTestDelegates(appmaster)
RestTestDelegates direct = new RestTestDelegates(realappmaster)
- direct.testRestletOperations();
+ direct.testRestletGetOperations();
proxied.testCodahaleOperations()
direct.testCodahaleOperations()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c97f7603/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 9dd2828..df62057 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -605,7 +605,6 @@ class SliderTestUtils extends Assert {
public static synchronized void initHttpTestSupport(Configuration conf) {
connectionOperations = new UrlConnectionOperations(conf);
jerseyBinding = new UgiJerseyBinding(connectionOperations)
-
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c97f7603/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
index de8dcdf..95352d0 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
@@ -108,6 +108,9 @@ public class AgentWebPagesIT extends AgentCommandTestBase
proxied.testRESTModel()
+ direct.testRestletGetOperations()
+ proxied.testRestletGetOperations()
+
// PUT & POST &c direct
direct.testPing()
if (!wsBackDoorRequired) {
@@ -116,7 +119,7 @@ public class AgentWebPagesIT extends AgentCommandTestBase
}
direct.logCodahaleMetrics();
-
+
// finally, stop the AM
direct.testStop();
}
[22/28] incubator-slider git commit: SLIDER-765
TestStandaloneYarnRegistryAM failing
Posted by el...@apache.org.
SLIDER-765 TestStandaloneYarnRegistryAM failing
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/192fb49f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/192fb49f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/192fb49f
Branch: refs/heads/develop
Commit: 192fb49f878422d48a81534617681034dd65221e
Parents: 6bd7165
Author: Steve Loughran <st...@apache.org>
Authored: Sun Jan 25 17:18:53 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Sun Jan 25 17:18:53 2015 +0000
----------------------------------------------------------------------
.../registry/retrieve/RegistryRetriever.java | 21 +++++++++++++-------
.../TestStandaloneYarnRegistryAM.groovy | 1 +
2 files changed, 15 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/192fb49f/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
index 3ed7cc6..364c661 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
@@ -67,9 +67,9 @@ public class RegistryRetriever extends AMWebClient {
* not match that expected (i.e. not a list of URLs), missing endpoint...
*/
public RegistryRetriever(ServiceRecord record) throws RegistryIOException {
- internalConfigurationURL = lookupRestAPI(record,
- PUBLISHER_CONFIGURATIONS_API, true);
externalConfigurationURL = lookupRestAPI(record,
+ PUBLISHER_CONFIGURATIONS_API, true);
+ internalConfigurationURL = lookupRestAPI(record,
PUBLISHER_CONFIGURATIONS_API, false);
externalExportsURL = lookupRestAPI(record,
PUBLISHER_EXPORTS_API, true);
@@ -196,10 +196,17 @@ public class RegistryRetriever extends AMWebClient {
@Override
public String toString() {
- return super.toString()
- + ": internal URL: \"" + internalConfigurationURL
- + "\"; external \"" + externalConfigurationURL +"\"";
+ final StringBuilder sb =
+ new StringBuilder("RegistryRetriever{");
+ sb.append("externalConfigurationURL='")
+ .append(externalConfigurationURL)
+ .append('\'');
+ sb.append(", internalConfigurationURL='")
+ .append(internalConfigurationURL)
+ .append('\'');
+ sb.append(", externalExportsURL='").append(externalExportsURL).append('\'');
+ sb.append(", internalExportsURL='").append(internalExportsURL).append('\'');
+ sb.append('}');
+ return sb.toString();
}
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/192fb49f/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index 705430f..62a8c40 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -158,6 +158,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
assert serviceRecords.size() == 1
def serviceRecord = serviceRecords[0]
+ describe "service record"
log.info(serviceRecord.toString())
assert serviceRecord[YarnRegistryAttributes.YARN_ID] != null
[23/28] incubator-slider git commit: SLIDER-766 binary releases don't
include license/3rd party metadata: work on core
Posted by el...@apache.org.
SLIDER-766 binary releases don't include license/3rd party metadata: work on core
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/98d676b9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/98d676b9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/98d676b9
Branch: refs/heads/develop
Commit: 98d676b90565dd3a6472845e1018c648c57cbe8e
Parents: 192fb49
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jan 26 15:38:44 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jan 26 17:08:07 2015 +0000
----------------------------------------------------------------------
.../accumulo/src/license/THIRD-PARTY.properties | 35 ++++++
.../src/packages/tarball/all.xml | 19 +++
.../slider-pkg/src/assembly/command-logger.xml | 17 +++
pom.xml | 33 ++++++
slider-agent/DEPENDENCIES | 116 +++++++++++++++++++
slider-agent/pom.xml | 1 +
slider-agent/src/packages/tarball/all.xml | 22 ++++
slider-assembly/pom.xml | 4 +-
slider-assembly/src/assembly/slider-bin.xml | 28 +++--
slider-core/pom.xml | 1 +
slider-core/src/license/THIRD-PARTY.properties | 33 ++++++
slider-funtest/pom.xml | 1 +
.../src/license/THIRD-PARTY.properties | 29 +++++
13 files changed, 327 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/98d676b9/app-packages/accumulo/src/license/THIRD-PARTY.properties
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/license/THIRD-PARTY.properties b/app-packages/accumulo/src/license/THIRD-PARTY.properties
new file mode 100644
index 0000000..9322b8f
--- /dev/null
+++ b/app-packages/accumulo/src/license/THIRD-PARTY.properties
@@ -0,0 +1,35 @@
+# Generated by org.codehaus.mojo.license.AddThirdPartyMojo
+#-------------------------------------------------------------------------------
+# Already used licenses in project :
+# - Apache License
+# - Apache License 2.0
+# - Apache License Version 2
+# - Apache License, Version 2.0
+# - Apache Software License - Version 2.0
+# - BSD
+# - CDDL + GPLv2 with classpath exception
+# - CDDL 1.1
+# - CDDL License
+# - Common Public License Version 1.0
+# - Eclipse Public License - Version 1.0
+# - GNU Lesser General Public License (LGPL), Version 2.1
+# - GPL2 w/ CPE
+# - MIT License
+# - New BSD License
+# - New BSD license
+# - Public Domain
+# - Revised BSD
+# - The Apache Software License, Version 2.0
+# - The BSD 3-Clause License
+# - The BSD License
+#-------------------------------------------------------------------------------
+# Please fill the missing licenses for dependencies :
+#
+#
+#Mon Jan 26 16:00:07 GMT 2015
+asm--asm--3.1=The BSD 3-Clause License
+commons-beanutils--commons-beanutils--1.7.0=The Apache Software License, Version 2.0
+javax.servlet--servlet-api--2.5=CDDL License
+javax.servlet.jsp--jsp-api--2.1=CDDL License
+org.apache.zookeeper--zookeeper--3.4.6=The Apache Software License, Version 2.0
+org.codehaus.jettison--jettison--1.1=The Apache Software License, Version 2.0
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/98d676b9/app-packages/command-logger/application-pkg/src/packages/tarball/all.xml
----------------------------------------------------------------------
diff --git a/app-packages/command-logger/application-pkg/src/packages/tarball/all.xml b/app-packages/command-logger/application-pkg/src/packages/tarball/all.xml
index 5c6080b..4538d57 100644
--- a/app-packages/command-logger/application-pkg/src/packages/tarball/all.xml
+++ b/app-packages/command-logger/application-pkg/src/packages/tarball/all.xml
@@ -32,4 +32,23 @@
<outputDirectory>command-logger-app</outputDirectory>
</fileSet>
</fileSets>
+
+ <files>
+ <file>
+ <source>${project.build.directory}/maven-shared-archive-resources/META-INF/DEPENDENCIES</source>
+ <outputDirectory>/</outputDirectory>
+ </file>
+ <file>
+ <source>${project.build.directory}/maven-shared-archive-resources/META-INF/NOTICE</source>
+ <outputDirectory>/</outputDirectory>
+ </file>
+ <file>
+ <source>${project.build.directory}/maven-shared-archive-resources/META-INF/LICENSE</source>
+ <outputDirectory>/</outputDirectory>
+ </file>
+ <file>
+ <source>${project.build.directory}/maven-shared-archive-resources/META-INF/NOTICE</source>
+ <outputDirectory>/</outputDirectory>
+ </file>
+ </files>
</assembly>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/98d676b9/app-packages/command-logger/slider-pkg/src/assembly/command-logger.xml
----------------------------------------------------------------------
diff --git a/app-packages/command-logger/slider-pkg/src/assembly/command-logger.xml b/app-packages/command-logger/slider-pkg/src/assembly/command-logger.xml
index cad468a..e283404 100644
--- a/app-packages/command-logger/slider-pkg/src/assembly/command-logger.xml
+++ b/app-packages/command-logger/slider-pkg/src/assembly/command-logger.xml
@@ -46,6 +46,21 @@
<filtered>true</filtered>
<fileMode>0755</fileMode>
</file>
+
+ <file>
+ <source>../../../LICENSE</source>
+ <outputDirectory>/</outputDirectory>
+ </file>
+
+ <file>
+ <source>../../../NOTICE</source>
+ <outputDirectory>/</outputDirectory>
+ </file>
+
+ <file>
+ <source>${project.build.directory}/maven-shared-archive-resources/META-INF/NOTICE</source>
+ <outputDirectory>/</outputDirectory>
+ </file>
</files>
<fileSets>
@@ -75,4 +90,6 @@
</fileSet>
</fileSets>
+
+
</assembly>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/98d676b9/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 5c18f77..d051f35 100644
--- a/pom.xml
+++ b/pom.xml
@@ -208,6 +208,7 @@
<maven-groovydoc-plugin.version>1.3</maven-groovydoc-plugin.version>
<maven-jar-plugin.version>2.5</maven-jar-plugin.version>
<maven.javadoc.version>2.10.1</maven.javadoc.version>
+ <maven.license.version>1.8</maven.license.version>
<maven.project.version>2.4</maven.project.version>
<maven.properties.version>1.0-alpha-2</maven.properties.version>
<maven-project-info-reports-plugin.version>2.7</maven-project-info-reports-plugin.version>
@@ -377,6 +378,37 @@
</executions>
</plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>license-maven-plugin</artifactId>
+ <version>${maven.license.version}</version>
+ <executions>
+<!--
+ <execution>
+ <id>download-licenses</id>
+ <goals>
+ <goal>download-licenses</goal>
+ </goals>
+ </execution>
+-->
+ <execution>
+ <id>default-cli</id>
+ <configuration>
+ <licenseMerges>
+ <licenseMerge>The Apache Software License, Version 2.0|Apache License, Version 2.0</licenseMerge>
+ <licenseMerge>The Apache Software License, Version 2.0|Apache License 2.0</licenseMerge>
+ <licenseMerge>The Apache Software License, Version 2.0|Apache Public License 2.0</licenseMerge>
+ <licenseMerge>The Apache Software License, Version 2.0|Apache 2</licenseMerge>
+ <licenseMerge>The Apache Software License, Version 2.0|ASF 2.0</licenseMerge>
+ <licenseMerge>The Apache Software License, Version 2.0|Apache License Version 2</licenseMerge>
+ <licenseMerge>The Apache Software License, Version 2.0|Apache Software License - Version 2.0</licenseMerge>
+ <licenseMerge>New BSD License|New BSD license</licenseMerge>
+ <licenseMerge>GNU Lesser General Public License (LGPL), Version 2.1|LGPL 2.1</licenseMerge>
+ </licenseMerges>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
@@ -1382,6 +1414,7 @@
<excludes>
<exclude>**/*.json</exclude>
<exclude>**/*.tar</exclude>
+ <exclude>**/THIRD-PARTY.properties</exclude>
<exclude>**/build.properties</exclude>
<exclude>**/regionservers</exclude>
<exclude>**/slaves</exclude>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/98d676b9/slider-agent/DEPENDENCIES
----------------------------------------------------------------------
diff --git a/slider-agent/DEPENDENCIES b/slider-agent/DEPENDENCIES
new file mode 100644
index 0000000..c4e65da
--- /dev/null
+++ b/slider-agent/DEPENDENCIES
@@ -0,0 +1,116 @@
+// ------------------------------------------------------------------
+// Transitive dependencies of this project determined from the
+// maven pom organized by organization.
+// ------------------------------------------------------------------
+
+Slider Agent
+
+APACHE SLIDER SUBCOMPONENTS:
+
+The Apache Slider project contains subcomponents with separate copyright
+notices and license terms. Your use of the source code for the these
+subcomponents is subject to the terms and conditions of the following
+licenses.
+
+For Python Mock unit test framework,
+ at slider-agent/src/test/python/mock, the BSD license:
+
+ Copyright (c) 2003-2012, Michael Foord
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+For Kokki, resource management library,
+ which slider-agent/src/main/python/resource_management is derived from,
+ the BSD license:
+
+Copyright (c) 2009 Samuel Stauffer <sa...@descolada.com>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ 3. Neither the name of Samuel Stauffer nor the names of its contributors may be used
+ to endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+For jinja2, used by resource management library as a templating engine,
+ at slider-agent/src/main/python/jinja2, the BSD license:
+
+Copyright (c) 2009 by the Jinja Team, see AUTHORS for more details.
+
+Some rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+ * The names of the contributors may not be used to endorse or
+ promote products derived from this software without specific
+ prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/98d676b9/slider-agent/pom.xml
----------------------------------------------------------------------
diff --git a/slider-agent/pom.xml b/slider-agent/pom.xml
index 327d09b..398ea5d 100644
--- a/slider-agent/pom.xml
+++ b/slider-agent/pom.xml
@@ -134,6 +134,7 @@
</executions>
<configuration>
<excludes>
+ <exclude>DEPENDENCIES</exclude>
<exclude>src/test/python/agent/dummy_output_error.txt</exclude>
<exclude>src/test/python/agent/dummy_output_good.txt</exclude>
<!-- jinja2 files (BSD license) -->
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/98d676b9/slider-agent/src/packages/tarball/all.xml
----------------------------------------------------------------------
diff --git a/slider-agent/src/packages/tarball/all.xml b/slider-agent/src/packages/tarball/all.xml
index 72afb75..91bd0a8 100644
--- a/slider-agent/src/packages/tarball/all.xml
+++ b/slider-agent/src/packages/tarball/all.xml
@@ -37,4 +37,26 @@
</excludes>
</fileSet>
</fileSets>
+
+ <files>
+ <file>
+ <source>../README.md</source>
+ <outputDirectory>/</outputDirectory>
+ </file>
+ <file>
+ <source>../LICENSE</source>
+ <outputDirectory>/</outputDirectory>
+ </file>
+
+ <file>
+ <source>DEPENDENCIES</source>
+ <outputDirectory>/</outputDirectory>
+ </file>
+
+ <file>
+ <source>${project.build.directory}/maven-shared-archive-resources/META-INF/NOTICE</source>
+ <outputDirectory>/</outputDirectory>
+ </file>
+
+ </files>
</assembly>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/98d676b9/slider-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml
index 6cb9128..b6d15c1 100644
--- a/slider-assembly/pom.xml
+++ b/slider-assembly/pom.xml
@@ -17,9 +17,9 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>slider-assembly</artifactId>
- <name>Slider Assembly</name>
+ <name>Apache Slider Binary Distributione</name>
<packaging>pom</packaging>
- <description>Apache Slider is a tool for dynamically deploying distributed applications to an Apache YARN cluster</description>
+ <description>Apache Slider is a tool for dynamically deploying distributed applications to an Apache YARN cluster. This module provides the binary redistributable.</description>
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/98d676b9/slider-assembly/src/assembly/slider-bin.xml
----------------------------------------------------------------------
diff --git a/slider-assembly/src/assembly/slider-bin.xml b/slider-assembly/src/assembly/slider-bin.xml
index c1eb499..8e4e162 100644
--- a/slider-assembly/src/assembly/slider-bin.xml
+++ b/slider-assembly/src/assembly/slider-bin.xml
@@ -56,16 +56,6 @@
</fileSet>
<fileSet>
- <directory>..</directory>
- <outputDirectory>/</outputDirectory>
- <includes>
- <include>README*</include>
- <include>LICENSE*</include>
- </includes>
- </fileSet>
-
-
- <fileSet>
<directory>../target/staging</directory>
<outputDirectory>/doc</outputDirectory>
</fileSet>
@@ -73,13 +63,31 @@
</fileSets>
<files>
+ <file>
+ <source>../README.md</source>
+ <outputDirectory>/</outputDirectory>
+ </file>
+ <file>
+ <source>../LICENSE</source>
+ <outputDirectory>/</outputDirectory>
+ </file>
<file>
<source>${project.build.directory}/agent/slider-agent.tar.gz</source>
<outputDirectory>lib</outputDirectory>
</file>
+ <file>
+ <source>${project.build.directory}/maven-shared-archive-resources/META-INF/DEPENDENCIES</source>
+ <outputDirectory>/</outputDirectory>
+ </file>
+ <file>
+ <source>${project.build.directory}/maven-shared-archive-resources/META-INF/NOTICE</source>
+ <outputDirectory>/</outputDirectory>
+ </file>
+
</files>
+
<dependencySets>
<dependencySet>
<scope>runtime</scope>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/98d676b9/slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index 43a1193..d5b3093 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -572,6 +572,7 @@
<exclude>**/*.json</exclude>
<exclude>src/test/python/agent.ini</exclude>
<exclude>src/test/python/version</exclude>
+ <exclude>**/THIRD-PARTY.properties</exclude>
<exclude>src/main/resources/webapps/slideram/.keep</exclude>
<exclude>src/main/resources/webapps/slideragent/.keep</exclude>
<exclude>src/main/resources/webapps/static/yarn.dt.plugins.js</exclude>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/98d676b9/slider-core/src/license/THIRD-PARTY.properties
----------------------------------------------------------------------
diff --git a/slider-core/src/license/THIRD-PARTY.properties b/slider-core/src/license/THIRD-PARTY.properties
new file mode 100644
index 0000000..662fde5
--- /dev/null
+++ b/slider-core/src/license/THIRD-PARTY.properties
@@ -0,0 +1,33 @@
+# Generated by org.codehaus.mojo.license.AddThirdPartyMojo
+#-------------------------------------------------------------------------------
+# Already used licenses in project :
+# - Apache License
+# - BSD
+# - CDDL + GPLv2 with classpath exception
+# - CDDL 1.1
+# - CDDL License
+# - CDDL+GPL
+# - Common Public License Version 1.0
+# - Eclipse Public License - Version 1.0
+# - GNU Lesser General Public License (LGPL), Version 2.1
+# - GPL2 w/ CPE
+# - MIT License
+# - MPL 1.1
+# - New BSD License
+# - Public Domain
+# - Revised BSD
+# - The Apache Software License, Version 2.0
+# - The BSD 3-Clause License
+# - The BSD License
+# - The MIT License
+#-------------------------------------------------------------------------------
+# Please fill the missing licenses for dependencies :
+#
+#
+#Mon Jan 26 15:27:00 GMT 2015
+asm--asm--3.1=The BSD 3-Clause License
+commons-beanutils--commons-beanutils--1.7.0=The Apache Software License, Version 2.0
+javax.servlet--servlet-api--2.5=CDDL License
+javax.servlet.jsp--jsp-api--2.1=CDDL License
+org.apache.zookeeper--zookeeper--3.4.6=The Apache Software License, Version 2.0
+org.codehaus.jettison--jettison--1.1=The Apache Software License, Version 2.0
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/98d676b9/slider-funtest/pom.xml
----------------------------------------------------------------------
diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml
index ac557e4..53d803d 100644
--- a/slider-funtest/pom.xml
+++ b/slider-funtest/pom.xml
@@ -313,6 +313,7 @@
<exclude>**/*.json</exclude>
<exclude>**/httpfs-signature.secret</exclude>
<exclude>**/regionservers</exclude>
+ <exclude>**/THIRD-PARTY.properties</exclude>
</excludes>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/98d676b9/slider-funtest/src/license/THIRD-PARTY.properties
----------------------------------------------------------------------
diff --git a/slider-funtest/src/license/THIRD-PARTY.properties b/slider-funtest/src/license/THIRD-PARTY.properties
new file mode 100644
index 0000000..2279ca6
--- /dev/null
+++ b/slider-funtest/src/license/THIRD-PARTY.properties
@@ -0,0 +1,29 @@
+# Generated by org.codehaus.mojo.license.AddThirdPartyMojo
+#-------------------------------------------------------------------------------
+# Already used licenses in project :
+# - Apache License
+# - BSD
+# - CDDL 1.1
+# - CDDL License
+# - Common Public License Version 1.0
+# - Eclipse Public License - Version 1.0
+# - GNU Lesser General Public License (LGPL), Version 2.1
+# - GPL2 w/ CPE
+# - MIT License
+# - New BSD License
+# - Public Domain
+# - Revised BSD
+# - The Apache Software License, Version 2.0
+# - The BSD 3-Clause License
+# - The BSD License
+#-------------------------------------------------------------------------------
+# Please fill the missing licenses for dependencies :
+#
+#
+#Mon Jan 26 15:56:57 GMT 2015
+asm--asm--3.1=The BSD 3-Clause License
+commons-beanutils--commons-beanutils--1.7.0=The Apache Software License, Version 2.0
+javax.servlet--servlet-api--2.5=CDDL License
+javax.servlet.jsp--jsp-api--2.1=CDDL License
+org.apache.zookeeper--zookeeper--3.4.6=The Apache Software License, Version 2.0
+org.codehaus.jettison--jettison--1.1=The Apache Software License, Version 2.0
[08/28] incubator-slider git commit: SLIDER-719 PUT with Jersey
Posted by el...@apache.org.
SLIDER-719 PUT with Jersey
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/dff5985d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/dff5985d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/dff5985d
Branch: refs/heads/develop
Commit: dff5985d891714e6fee5b01b7e857973c0a52825
Parents: 79d0e14
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jan 20 18:45:34 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jan 20 18:46:16 2015 +0000
----------------------------------------------------------------------
.../agent/rest/JerseyTestDelegates.groovy | 81 ++++++++++++++++----
.../slider/agent/rest/TestStandaloneREST.groovy | 7 +-
2 files changed, 72 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dff5985d/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
index cb89bff..a0fc144 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
@@ -23,6 +23,7 @@ import com.sun.jersey.api.client.WebResource
import com.sun.jersey.api.client.UniformInterfaceException
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.webapp.NotFoundException
import org.apache.slider.api.StateValues
import org.apache.slider.api.types.SerializedComponentInformation
import org.apache.slider.api.types.SerializedContainerInformation
@@ -31,12 +32,12 @@ import org.apache.slider.core.conf.ConfTree
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.core.restclient.HttpOperationResponse
import org.apache.slider.core.restclient.HttpVerb
-import org.apache.slider.core.restclient.UrlConnectionOperations
import org.apache.slider.server.appmaster.web.rest.RestPaths
import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource
import org.apache.slider.server.appmaster.web.rest.application.resources.PingResource
import org.apache.slider.test.Outcome
import org.apache.slider.test.SliderTestUtils
+import org.glassfish.grizzly.servlet.ver25.WebResourceCollectionType
import javax.ws.rs.core.MediaType
@@ -61,12 +62,17 @@ class JerseyTestDelegates extends SliderTestUtils {
final String appmaster;
final String application;
final Client jersey;
+ final WebResource amResource
+ final WebResource appResource
JerseyTestDelegates(String appmaster, Client jersey) {
- this.appmaster = appmaster
- this.application = appendToURL(appmaster, RestPaths.SLIDER_PATH_APPLICATION)
this.jersey = jersey
+ this.appmaster = appmaster
+ application = appendToURL(appmaster, SLIDER_PATH_APPLICATION)
+ amResource = jersey.resource(appmaster)
+ amResource.type(MediaType.APPLICATION_JSON)
+ appResource = amResource.path(SLIDER_PATH_APPLICATION)
}
/**
@@ -86,10 +92,43 @@ class JerseyTestDelegates extends SliderTestUtils {
* @return
*/
public <T> T jExec(HttpVerb method, String subpath, Class<T> c) {
- assert c
- def appPath = appendToURL(SLIDER_PATH_APPLICATION, subpath)
- WebResource webResource = buildResource(appPath)
- (T) webResource.method(method.verb, c)
+ WebResource resource = applicationResource(subpath)
+ jExec(method, resource, c)
+ }
+
+ public <T> T jExec(HttpVerb method, WebResource resource, Class<T> c) {
+ try {
+ assert c
+ (T) resource.method(method.verb, c)
+ } catch (UniformInterfaceException ex) {
+ uprateFaults(method, resource, ex)
+ }
+ }
+
+ /**
+ * Create a resource under the application path
+ * @param subpath
+ * @return
+ */
+ public WebResource applicationResource(String subpath) {
+ return appResource.path(subpath)
+ }
+
+ /**
+ * Convert faults to exceptions; pass through 200 responses
+ * @param method
+ * @param webResource
+ * @param ex
+ * @return
+ */
+ public uprateFaults(
+ HttpVerb method,
+ WebResource webResource,
+ UniformInterfaceException ex) {
+ uprateFaults(method.verb,
+ webResource.URI.toString(),
+ ex.response.status,
+ ex.response.toString())
}
/**
@@ -218,7 +257,7 @@ class JerseyTestDelegates extends SliderTestUtils {
SerializedContainerInformation
)
fail("expected an error, got $result")
- } catch (UniformInterfaceException e) {
+ } catch (NotFoundException e) {
// expected
}
@@ -320,17 +359,20 @@ class JerseyTestDelegates extends SliderTestUtils {
def pinged = jExec(HttpVerb.GET, ACTION_PING, PingResource)
log.info "Ping GET: $pinged"
// HEAD
- jExec(HttpVerb.HEAD, ACTION_PING, PingResource)
+// jExec(HttpVerb.HEAD, ACTION_PING, PingResource)
jExec(HttpVerb.PUT, ACTION_PING, PingResource)
jExec(HttpVerb.DELETE, ACTION_PING, PingResource)
jExec(HttpVerb.POST, ACTION_PING, PingResource)
+ pingPut(ACTION_PING, "ping-text")
}
- private HttpOperationResponse pingAction(
- HttpVerb verb, String subpath, String payload) {
+ private PingResource pingPut(String subpath, String payload) {
def pinged
- jExec(verb,subpath, PingResource)
+ def actionPing = applicationResource(ACTION_PING)
+ def response = actionPing.put(PingResource, payload)
+
+/*
def outcome = ops.execHttpOperation(
verb,
pingUrl,
@@ -347,6 +389,8 @@ class JerseyTestDelegates extends SliderTestUtils {
0, "${bytes.length} bytes of data from ping $verb.verb"
}
return outcome
+*/
+ return response
}
/**
@@ -354,6 +398,8 @@ class JerseyTestDelegates extends SliderTestUtils {
* Important: once executed, the AM is no longer there.
* This must be the last test in the sequence.
*/
+/*
+
public void testStop() {
String target = appendToURL(appmaster, SLIDER_PATH_APPLICATION, ACTION_STOP)
describe "Stop URL $target"
@@ -384,6 +430,7 @@ class JerseyTestDelegates extends SliderTestUtils {
}
}
+*/
/**
* Probe that spins until the url specified by "url") refuses
@@ -391,6 +438,7 @@ class JerseyTestDelegates extends SliderTestUtils {
* @param args argument map
* @return the outcome
*/
+/*
Outcome probePingFailing(Map args) {
String ping = args["url"]
URL pingUrl = new URL(ping)
@@ -402,14 +450,17 @@ class JerseyTestDelegates extends SliderTestUtils {
return Outcome.Success
}
}
+*/
- public void suite() {
+ public void testSuiteGetOperations() {
testCodahaleOperations()
testLiveResources()
testLiveContainers();
- testPing();
-
testRESTModel()
}
+
+ public void testSuiteComplexVerbs() {
+ testPing();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dff5985d/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
index a53f546..5e00da5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
@@ -90,7 +90,12 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
JerseyTestDelegates proxyJerseyTests =
new JerseyTestDelegates(proxyAM, createJerseyClient())
- proxyJerseyTests.suite()
+ proxyJerseyTests.testSuiteGetOperations()
+
+ JerseyTestDelegates directJerseyTests =
+ new JerseyTestDelegates(directAM, createJerseyClient())
+ directJerseyTests.testSuiteGetOperations()
+ directJerseyTests.testSuiteComplexVerbs()
RestTestDelegates proxied = new RestTestDelegates(proxyAM)
RestTestDelegates direct = new RestTestDelegates(directAM)
[27/28] incubator-slider git commit: SLIDER-769 Modify Slider app
packages for seamless integration with AMS provider (Ambari Metrics Service)
Posted by el...@apache.org.
SLIDER-769 Modify Slider app packages for seamless integration with AMS provider (Ambari Metrics Service)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/618cdc4c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/618cdc4c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/618cdc4c
Branch: refs/heads/develop
Commit: 618cdc4c45b77f4f2e82cb5b86aeb1300cc005c7
Parents: 96f890b
Author: Gour Saha <go...@apache.org>
Authored: Tue Jan 27 15:31:38 2015 -0800
Committer: Gour Saha <go...@apache.org>
Committed: Tue Jan 27 15:33:14 2015 -0800
----------------------------------------------------------------------
app-packages/hbase/appConfig-default.json | 7 ++--
.../hbase/appConfig-phoenix-default.json | 7 ++--
.../hbase/appConfig-secured-default.json | 7 ++--
app-packages/hbase/ganglia_metrics.json | 38 --------------------
app-packages/hbase/metainfo.xml | 8 ++---
app-packages/hbase/package/scripts/params.py | 11 ++++--
...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 32 ++++++++++++++---
...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 32 ++++++++++++++---
app-packages/hbase/timeline_metrics.json | 38 ++++++++++++++++++++
9 files changed, 113 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig-default.json b/app-packages/hbase/appConfig-default.json
index f065185..4a91884 100644
--- a/app-packages/hbase/appConfig-default.json
+++ b/app-packages/hbase/appConfig-default.json
@@ -11,10 +11,9 @@
"site.global.app_user": "yarn",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}",
- "site.global.ganglia_server_host": "${NN_HOST}",
- "site.global.ganglia_server_port": "8667",
- "site.global.ganglia_server_id": "Application1",
- "site.global.ganglia_enabled":"true",
+ "site.global.metric_collector_host": "${NN_HOST}",
+ "site.global.metric_collector_port": "6188",
+ "site.global.metric_collector_lib": "",
"site.global.hbase_instance_name": "instancename",
"site.global.hbase_root_password": "secret",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/appConfig-phoenix-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig-phoenix-default.json b/app-packages/hbase/appConfig-phoenix-default.json
index d74562f..68df0cc 100644
--- a/app-packages/hbase/appConfig-phoenix-default.json
+++ b/app-packages/hbase/appConfig-phoenix-default.json
@@ -11,10 +11,9 @@
"site.global.app_user": "yarn",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}",
- "site.global.ganglia_server_host": "${NN_HOST}",
- "site.global.ganglia_server_port": "8667",
- "site.global.ganglia_server_id": "Application1",
- "site.global.ganglia_enabled":"true",
+ "site.global.metric_collector_host": "${NN_HOST}",
+ "site.global.metric_collector_port": "6188",
+ "site.global.metric_collector_lib": "",
"site.global.hbase_instance_name": "instancename",
"site.global.hbase_root_password": "secret",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/appConfig-secured-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig-secured-default.json b/app-packages/hbase/appConfig-secured-default.json
index 2a2b08f..553117e 100644
--- a/app-packages/hbase/appConfig-secured-default.json
+++ b/app-packages/hbase/appConfig-secured-default.json
@@ -11,10 +11,9 @@
"site.global.app_user": "${USER_NAME}",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}",
- "site.global.ganglia_server_host": "${NN_HOST}",
- "site.global.ganglia_server_port": "8667",
- "site.global.ganglia_server_id": "Application1",
- "site.global.ganglia_enabled":"true",
+ "site.global.metric_collector_host": "${NN_HOST}",
+ "site.global.metric_collector_port": "6188",
+ "site.global.metric_collector_lib": "",
"site.global.hbase_instance_name": "instancename",
"site.global.hbase_root_password": "secret",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/ganglia_metrics.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/ganglia_metrics.json b/app-packages/hbase/ganglia_metrics.json
deleted file mode 100644
index da73d48..0000000
--- a/app-packages/hbase/ganglia_metrics.json
+++ /dev/null
@@ -1,38 +0,0 @@
-{
- "Component": {
- "HBASE_REGIONSERVER": {
- "readRequestsCount": {
- "metric": "regionserver.Server.readRequestCount",
- "pointInTime": false,
- "temporal": true
- },
- "regions": {
- "metric": "regionserver.Server.regionCount",
- "pointInTime": false,
- "temporal": true
- },
- "flushQueueSize": {
- "metric": "regionserver.Server.flushQueueLength",
- "pointInTime": false,
- "temporal": true
- }
- },
- "HBASE_MASTER": {
- "cluster_requests": {
- "metric": "master.Server.clusterRequests",
- "pointInTime": false,
- "temporal": true
- },
- "splitTime_avg_time": {
- "metric": "master.FileSystem.HlogSplitTime_mean",
- "pointInTime": false,
- "temporal": true
- },
- "splitSize_avg_time": {
- "metric": "master.FileSystem.HlogSplitSize_mean",
- "pointInTime": false,
- "temporal": true
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml
index d5e07a7..1d01ae7 100644
--- a/app-packages/hbase/metainfo.xml
+++ b/app-packages/hbase/metainfo.xml
@@ -55,11 +55,7 @@
</export>
<export>
<name>org.apache.slider.metrics</name>
- <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
- </export>
- <export>
- <name>org.apache.slider.metrics.ui</name>
- <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
+ <value>http://${site.global.metric_collector_host}:${site.global.metric_collector_port}/ws/v1/timeline/metrics</value>
</export>
</exports>
</exportGroup>
@@ -81,7 +77,7 @@
<name>HBASE_MASTER</name>
<category>MASTER</category>
<minInstanceCount>1</minInstanceCount>
- <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.metrics,QuickLinks-org.apache.slider.metrics.ui</appExports>
+ <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.metrics</appExports>
<componentExports>
<componentExport>
<name>org.apache.slider.jmx</name>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/params.py b/app-packages/hbase/package/scripts/params.py
index 027630b..2da4389 100644
--- a/app-packages/hbase/package/scripts/params.py
+++ b/app-packages/hbase/package/scripts/params.py
@@ -26,6 +26,7 @@ import status_params
config = Script.get_config()
hbase_root = config['configurations']['global']['app_root']
+app_name = config['clusterName']
conf_dir = format("{hbase_root}/conf")
daemon_script = format("{hbase_root}/bin/hbase-daemon.sh")
@@ -59,8 +60,12 @@ client_jaas_config_file = default('hbase_client_jaas_config_file', format("{conf
master_jaas_config_file = default('hbase_master_jaas_config_file', format("{conf_dir}/hbase_master_jaas.conf"))
regionserver_jaas_config_file = default('hbase_regionserver_jaas_config_file', format("{conf_dir}/hbase_regionserver_jaas.conf"))
-ganglia_server_host = default('/configurations/global/ganglia_server_host', '')
-ganglia_server_port = default('/configurations/global/ganglia_server_port', '8663')
+metric_collector_host = config['configurations']['global']['metric_collector_host']
+metric_collector_port = config['configurations']['global']['metric_collector_port']
+metric_collector_lib = config['configurations']['global']['metric_collector_lib']
+has_metric_collector = 1
+if not metric_collector_lib:
+ has_metric_collector = 0
rest_port = config['configurations']['global']['hbase_rest_port']
rest_infoport = config['configurations']['global']['hbase_rest_infoport']
@@ -88,7 +93,7 @@ if security_enabled:
master_jaas_princ = config['configurations']['hbase-site']['hbase.master.kerberos.principal'].replace('_HOST',_hostname_lowercase)
regionserver_jaas_princ = config['configurations']['hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST',_hostname_lowercase)
-
+
master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file']
regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file']
kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 b/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
index a3eb1fc..2b37512 100644
--- a/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
+++ b/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
@@ -28,26 +28,48 @@
# If this variable is left out, then the default is no expiration.
hbase.extendedperiod = 3600
+{% if has_metric_collector %}
+
+*.timeline.plugin.urls=file://{{metric_collector_lib}}
+hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+hbase.period=10
+hbase.collector={{metric_collector_host}}:{{metric_collector_port}}
+
+jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+jvm.period=10
+jvm.collector={{metric_collector_host}}:{{metric_collector_port}}
+
+rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+rpc.period=10
+rpc.collector={{metric_collector_host}}:{{metric_collector_port}}
+
+hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+hbase.sink.timeline.period=10
+hbase.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
+hbase.sink.timeline.serviceName-prefix={{app_name}}-master
+
+{% else %}
+
# Configuration of the "hbase" context for ganglia
# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext
hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
hbase.period=10
-hbase.servers={{ganglia_server_host}}:{{ganglia_server_port}}
+hbase.servers={{metric_collector_host}}:{{metric_collector_port}}
# Configuration of the "jvm" context for ganglia
# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
jvm.period=10
-jvm.servers={{ganglia_server_host}}:{{ganglia_server_port}}
+jvm.servers={{metric_collector_host}}:{{metric_collector_port}}
# Configuration of the "rpc" context for ganglia
# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext
rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
rpc.period=10
-rpc.servers={{ganglia_server_host}}:{{ganglia_server_port}}
+rpc.servers={{metric_collector_host}}:{{metric_collector_port}}
#Ganglia following hadoop example
hbase.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31
@@ -59,4 +81,6 @@ hbase.sink.ganglia.period=10
.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both
.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40
-hbase.sink.ganglia.servers={{ganglia_server_host}}:{{ganglia_server_port}}
+hbase.sink.ganglia.servers={{metric_collector_host}}:{{metric_collector_port}}
+
+{% endif %}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 b/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
index a3eb1fc..1954373 100644
--- a/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
+++ b/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
@@ -28,26 +28,48 @@
# If this variable is left out, then the default is no expiration.
hbase.extendedperiod = 3600
+{% if has_metric_collector %}
+
+*.timeline.plugin.urls=file://{{metric_collector_lib}}
+hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+hbase.period=10
+hbase.collector={{metric_collector_host}}:{{metric_collector_port}}
+
+jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+jvm.period=10
+jvm.collector={{metric_collector_host}}:{{metric_collector_port}}
+
+rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+rpc.period=10
+rpc.collector={{metric_collector_host}}:{{metric_collector_port}}
+
+hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+hbase.sink.timeline.period=10
+hbase.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
+hbase.sink.timeline.serviceName-prefix={{app_name}}-rs
+
+{% else %}
+
# Configuration of the "hbase" context for ganglia
# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext
hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
hbase.period=10
-hbase.servers={{ganglia_server_host}}:{{ganglia_server_port}}
+hbase.servers={{metric_collector_host}}:{{metric_collector_port}}
# Configuration of the "jvm" context for ganglia
# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
jvm.period=10
-jvm.servers={{ganglia_server_host}}:{{ganglia_server_port}}
+jvm.servers={{metric_collector_host}}:{{metric_collector_port}}
# Configuration of the "rpc" context for ganglia
# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext
rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
rpc.period=10
-rpc.servers={{ganglia_server_host}}:{{ganglia_server_port}}
+rpc.servers={{metric_collector_host}}:{{metric_collector_port}}
#Ganglia following hadoop example
hbase.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31
@@ -59,4 +81,6 @@ hbase.sink.ganglia.period=10
.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both
.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40
-hbase.sink.ganglia.servers={{ganglia_server_host}}:{{ganglia_server_port}}
+hbase.sink.ganglia.servers={{metric_collector_host}}:{{metric_collector_port}}
+
+{% endif %}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/timeline_metrics.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/timeline_metrics.json b/app-packages/hbase/timeline_metrics.json
new file mode 100644
index 0000000..da73d48
--- /dev/null
+++ b/app-packages/hbase/timeline_metrics.json
@@ -0,0 +1,38 @@
+{
+ "Component": {
+ "HBASE_REGIONSERVER": {
+ "readRequestsCount": {
+ "metric": "regionserver.Server.readRequestCount",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "regions": {
+ "metric": "regionserver.Server.regionCount",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "flushQueueSize": {
+ "metric": "regionserver.Server.flushQueueLength",
+ "pointInTime": false,
+ "temporal": true
+ }
+ },
+ "HBASE_MASTER": {
+ "cluster_requests": {
+ "metric": "master.Server.clusterRequests",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "splitTime_avg_time": {
+ "metric": "master.FileSystem.HlogSplitTime_mean",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "splitSize_avg_time": {
+ "metric": "master.FileSystem.HlogSplitSize_mean",
+ "pointInTime": false,
+ "temporal": true
+ }
+ }
+ }
+}
[09/28] incubator-slider git commit: SLIDER-719 jersey ping test
supports PUT/DELETE/GET/HEAD/POST
Posted by el...@apache.org.
SLIDER-719 jersey ping test supports PUT/DELETE/GET/HEAD/POST
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4418179a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4418179a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4418179a
Branch: refs/heads/develop
Commit: 4418179aa645a6bb6d4c8cf4fdae76468b9d1d8d
Parents: dff5985
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jan 20 19:06:37 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jan 20 19:06:37 2015 +0000
----------------------------------------------------------------------
.../agent/rest/JerseyTestDelegates.groovy | 42 +++++++++-----------
.../slider/agent/rest/RestTestDelegates.groovy | 21 +++++-----
.../slider/agent/rest/TestStandaloneREST.groovy | 26 ++----------
3 files changed, 35 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4418179a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
index a0fc144..0699800 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
@@ -363,34 +363,30 @@ class JerseyTestDelegates extends SliderTestUtils {
jExec(HttpVerb.PUT, ACTION_PING, PingResource)
jExec(HttpVerb.DELETE, ACTION_PING, PingResource)
jExec(HttpVerb.POST, ACTION_PING, PingResource)
- pingPut(ACTION_PING, "ping-text")
-
+ ping(HttpVerb.PUT, ACTION_PING, "ping-text")
+ ping(HttpVerb.POST, ACTION_PING, "ping-text")
+ ping(HttpVerb.DELETE, ACTION_PING, "ping-text")
}
- private PingResource pingPut(String subpath, String payload) {
- def pinged
+ /**
+ * Execute a ping; assert that a response came back with the relevant
+ * verb if the verb has a response body
+ * @param method method to invoke
+ * @param subpath ping path
+ * @param payload payload
+ * @return the resource if the verb has a response
+ */
+ private PingResource ping(HttpVerb method, String subpath, Object payload) {
def actionPing = applicationResource(ACTION_PING)
- def response = actionPing.put(PingResource, payload)
-
-/*
- def outcome = ops.execHttpOperation(
- verb,
- pingUrl,
- payload.bytes,
- MediaType.TEXT_PLAIN)
- byte[] bytes = outcome.data
- if (verb.hasResponseBody()) {
- assert bytes.length > 0, "0 bytes from ping $verb.verb"
- pinged = deser(PingResource, bytes)
- log.info "Ping $verb.verb: $pinged"
- assert verb.verb == pinged.verb
+ def upload = method.hasUploadBody() ? payload : null
+ if (method.hasResponseBody()) {
+ def pinged = actionPing.method(method.verb, PingResource, upload)
+ assert method.verb == pinged.verb
+ return pinged
} else {
- assert bytes.length ==
- 0, "${bytes.length} bytes of data from ping $verb.verb"
+ actionPing.method(method.verb, upload)
+ return null
}
- return outcome
-*/
- return response
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4418179a/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
index bf009c2..dd23926 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
@@ -193,15 +193,6 @@ class RestTestDelegates extends SliderTestUtils {
TEST_GLOBAL_OPTION_PRESENT
}
-
- public void testRestletGetOperations() throws Throwable {
- Client client = createJerseyClient()
- String path = appendToURL(application, LIVE_RESOURCES)
- WebResource webResource = client.resource(path)
- webResource.type(MediaType.APPLICATION_JSON)
- .get(ConfTree.class);
- }
-
public void testPing() {
// GET
String ping = appendToURL(appmaster, SLIDER_PATH_APPLICATION, ACTION_PING)
@@ -306,4 +297,16 @@ class RestTestDelegates extends SliderTestUtils {
}
}
+
+ public void testSuiteGetOperations() {
+
+ testCodahaleOperations()
+ testLiveResources()
+ testLiveContainers();
+ testRESTModel()
+ }
+
+ public void testSuiteComplexVerbs() {
+ testPing();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4418179a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
index 5e00da5..9cea46c 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
@@ -100,36 +100,18 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
RestTestDelegates proxied = new RestTestDelegates(proxyAM)
RestTestDelegates direct = new RestTestDelegates(directAM)
- direct.testRestletGetOperations();
- proxied.testCodahaleOperations()
- direct.testCodahaleOperations()
-
- describe "base entry lists"
-
- assertPathServesList(proxyAM, LIVE, ApplicationResource.LIVE_ENTRIES)
-
- // now some REST gets
- describe "Application REST ${LIVE_RESOURCES}"
- proxied.testLiveResources()
-
- proxied.testRESTModel()
-
- // PUT & POST &c must go direct for now
- direct.testPing()
- // PUT & POST &c direct
- direct.testPing()
+ proxied.testSuiteGetOperations()
+ direct.testSuiteGetOperations()
+ direct.testSuiteComplexVerbs()
if (!wsBackDoorRequired) {
// and via the proxy
- proxied.testRESTModel()
+ proxied.testSuiteComplexVerbs()
}
direct.logCodahaleMetrics();
// this MUST be the final test
direct.testStop();
-
-
-
}
[18/28] incubator-slider git commit: SLIDER-782 Failed attempt to
integrate with the AmWebClient logic for handling redirects across http/https
boundary. That code only works for GET operations.
Posted by el...@apache.org.
SLIDER-782 Failed attempt to integrate with the AmWebClient logic for handling redirects across http/https boundary. That code only works for GET operations.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6c038b41
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6c038b41
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6c038b41
Branch: refs/heads/develop
Commit: 6c038b4132fe8bfdca493d3ef9158bfd838db9db
Parents: 73e1639
Author: Steve Loughran <st...@apache.org>
Authored: Fri Jan 23 12:46:34 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Jan 23 12:48:16 2015 +0000
----------------------------------------------------------------------
.../core/restclient/UgiJerseyBinding.java | 12 +++-
.../restclient/UrlConnectionOperations.java | 69 ++++++++++++++++++--
2 files changed, 73 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6c038b41/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
index 76407c6..10ce1ce 100644
--- a/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.webapp.NotFoundException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
@@ -74,7 +75,7 @@ public class UgiJerseyBinding implements
/**
* Get a URL connection.
- * @param url
+ * @param url URL to connect to
* @return the connection
* @throws IOException any problem. {@link AuthenticationException}
* errors are wrapped
@@ -82,6 +83,10 @@ public class UgiJerseyBinding implements
@Override
public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
try {
+ // open a connection handling status codes and so redirections
+ // but as it opens a connection, it's less useful than you think.
+// return operations.openConnectionRedirecting(url);
+
return operations.openConnection(url);
} catch (AuthenticationException e) {
throw new IOException(e);
@@ -124,10 +129,11 @@ public class UgiJerseyBinding implements
ClientResponse response = ex.getResponse();
int resultCode = response.getStatus();
String msg = verb.toString() + " " + url;
- if (resultCode == 404) {
+ if (resultCode == HttpServletResponse.SC_NOT_FOUND) {
return (IOException) new PathNotFoundException(url).initCause(ex);
}
- if (resultCode == 401) {
+ if (resultCode == HttpServletResponse.SC_UNAUTHORIZED
+ || resultCode == HttpServletResponse.SC_FORBIDDEN) {
return (IOException) new PathAccessDeniedException(url).initCause(ex);
}
// all other error codes
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6c038b41/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
index 328684e..e6b08c1 100644
--- a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
@@ -31,10 +31,13 @@ import org.apache.hadoop.yarn.webapp.NotFoundException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.core.HttpHeaders;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
+import java.net.URI;
import java.net.URL;
/**
@@ -74,7 +77,8 @@ public class UrlConnectionOperations extends Configured {
}
/**
- * Opens a url with read and connect timeouts
+ * Opens a url with cache disabled, redirect handled in
+ * (JDK) implementation.
*
* @param url to open
* @return URLConnection
@@ -84,14 +88,70 @@ public class UrlConnectionOperations extends Configured {
public HttpURLConnection openConnection(URL url) throws
IOException,
AuthenticationException {
- Preconditions.checkArgument(url.getPort() != 0, "no port");
- HttpURLConnection conn =
- (HttpURLConnection) connectionFactory.openConnection(url, useSpnego);
+
+ HttpURLConnection conn = innerOpenConnection(url);
conn.setUseCaches(false);
conn.setInstanceFollowRedirects(true);
return conn;
}
+
+ /**
+ * Opens a url.
+ * <p>
+ * This implementation
+ * <ol>
+ * <li>Handles protocol switching during redirects</li>
+ * <li>Handles 307 responses "redirect with same verb"</li>
+ * </ol>
+ *
+ * @param url to open
+ * @return URLConnection
+ * @throws IOException
+ * @throws AuthenticationException authentication failure
+ */
+ public HttpURLConnection openConnectionRedirecting(URL url) throws
+ IOException,
+ AuthenticationException {
+ HttpURLConnection connection = innerOpenConnection(url);
+ connection.setUseCaches(false);
+ int responseCode = connection.getResponseCode();
+ if (responseCode == HttpServletResponse.SC_MOVED_TEMPORARILY
+ || responseCode == HttpServletResponse.SC_TEMPORARY_REDIRECT) {
+ log.debug("Redirected with response {}", responseCode);
+ // is a redirect - are we changing schemes?
+ String redirectLocation = connection.getHeaderField(HttpHeaders.LOCATION);
+ String originalScheme = url.getProtocol();
+ String redirectScheme = URI.create(redirectLocation).getScheme();
+ boolean buildNewUrl = false;
+ if (!originalScheme.equals(redirectScheme)) {
+ // need to fake it out by doing redirect ourselves
+ log.debug("Protocol change during redirect");
+ buildNewUrl = true;
+ } else if (responseCode == HttpServletResponse.SC_TEMPORARY_REDIRECT) {
+ // 307 response
+ buildNewUrl = true;
+ }
+
+ if (buildNewUrl) {
+ // perform redirect ourselves
+ log.debug("Redirecting {} to URL {}",
+ url, redirectLocation);
+ URL redirectURL = new URL(redirectLocation);
+ connection = innerOpenConnection(url);
+ }
+ }
+
+ return connection;
+ }
+
+ protected HttpURLConnection innerOpenConnection(URL url) throws
+ IOException,
+ AuthenticationException {
+ Preconditions.checkArgument(url.getPort() != 0, "no port");
+ return (HttpURLConnection) connectionFactory.openConnection(url, useSpnego);
+ }
+
public HttpOperationResponse execGet(URL url) throws
IOException,
AuthenticationException {
@@ -121,7 +181,6 @@ public class UrlConnectionOperations extends Configured {
if (doOutput) {
conn.setRequestProperty("Content-Type", contentType);
}
-
// now do the connection
conn.connect();
[20/28] incubator-slider git commit: SLIDER-762 Rest client has
ability to bind via YARN registry
Posted by el...@apache.org.
SLIDER-762 Rest client has ability to bind via YARN registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/beda1b2d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/beda1b2d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/beda1b2d
Branch: refs/heads/develop
Commit: beda1b2d83c214fced12eae801ad966dfc1a173b
Parents: 6c038b4
Author: Steve Loughran <st...@apache.org>
Authored: Fri Jan 23 19:42:08 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Jan 23 19:42:08 2015 +0000
----------------------------------------------------------------------
.../slider/client/rest/RestClientFactory.java | 61 ++++++
.../client/rest/RestClientRegistryBinder.java | 201 +++++++++++++++++++
.../core/exceptions/ExceptionConverter.java | 20 +-
.../registry/info/CustomRegistryConstants.java | 5 +-
.../registry/retrieve/RegistryRetriever.java | 67 ++-----
.../core/restclient/UgiJerseyBinding.java | 35 +---
.../slideram/SliderAMProviderService.java | 5 +
.../rest/SliderRestClientTestDelegates.groovy | 4 +-
.../slider/agent/rest/TestStandaloneREST.groovy | 29 ++-
.../funtest/lifecycle/AgentWebPagesIT.groovy | 32 ++-
10 files changed, 358 insertions(+), 101 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/beda1b2d/slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java b/slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java
new file mode 100644
index 0000000..1b7553e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client.rest;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.WebResource;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.slider.core.registry.info.CustomRegistryConstants;
+
+import java.io.IOException;
+
+public class RestClientFactory {
+
+ private final RestClientRegistryBinder binder;
+ private final Client jerseyClient;
+ private final String user, serviceclass, instance;
+
+ public RestClientFactory(RegistryOperations operations,
+ Client jerseyClient,
+ String user,
+ String serviceclass,
+ String instance) {
+ this.jerseyClient = jerseyClient;
+ this.user = user;
+ this.serviceclass = serviceclass;
+ this.instance = instance;
+ binder = new RestClientRegistryBinder(operations);
+ }
+
+ /**
+ * Locate the AM
+ * @return a resource to the AM
+ * @throws IOException any failure to resolve to the AM
+ */
+ WebResource locateAppmaster() throws IOException {
+ String restAPI = binder.lookupExternalRestAPI(user, serviceclass, instance,
+ CustomRegistryConstants.AM_REST_BASE);
+ return jerseyClient.resource(restAPI);
+ }
+
+ public SliderApplicationAPI createSliderApplicationApi() throws IOException {
+ return new SliderApplicationAPI(jerseyClient, locateAppmaster());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/beda1b2d/slider-core/src/main/java/org/apache/slider/client/rest/RestClientRegistryBinder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/rest/RestClientRegistryBinder.java b/slider-core/src/main/java/org/apache/slider/client/rest/RestClientRegistryBinder.java
new file mode 100644
index 0000000..048a07b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/client/rest/RestClientRegistryBinder.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client.rest;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
+import org.apache.hadoop.registry.client.impl.zk.RegistryInternalConstants;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.encodeForRegistry;
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.convertUsername;
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.getCurrentUsernameUnencoded;
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath;
+
+/**
+ * Generic code to get the URLs for clients via the registry
+ */
+public class RestClientRegistryBinder {
+ private static final Logger log =
+ LoggerFactory.getLogger(RestClientRegistryBinder.class);
+
+ private final RegistryOperations operations;
+
+ public RestClientRegistryBinder(RegistryOperations operations) {
+ this.operations = operations;
+ }
+
+ /**
+ * Buld the user path -switches to the system path if the user is "".
+ * It also cross-converts the username to ascii via punycode
+ * @param username username or ""
+ * @return the path to the user
+ */
+ public static String homePathForUser(String username) {
+ Preconditions.checkArgument(username != null, "null user");
+
+ // catch recursion
+ if (username.startsWith(RegistryConstants.PATH_USERS)) {
+ return username;
+ }
+
+ if (username.isEmpty()) {
+ return RegistryConstants.PATH_SYSTEM_SERVICES;
+ }
+
+ // convert username to registry name
+ String convertedName = convertUsername(username);
+
+ return RegistryPathUtils.join(RegistryConstants.PATH_USERS,
+ encodeForRegistry(convertedName));
+ }
+
+ /**
+ * Get the current username, before any encoding has been applied.
+ * @return the current user from the kerberos identity, falling back
+ * to the user and/or env variables.
+ */
+ public static String currentUsernameUnencoded() {
+ String env_hadoop_username = System.getenv(
+ RegistryInternalConstants.HADOOP_USER_NAME);
+ return getCurrentUsernameUnencoded(env_hadoop_username);
+ }
+
+ /**
+ * Qualify a user.
+ * <ol>
+ * <li> <code>"~"</code> maps to user home path home</li>
+ * <li> <code>"~user"</code> maps to <code>/users/$user</code></li>
+ * <li> <code>"/"</code> maps to <code>/services/</code></li>
+ * </ol>
+ * @param user the username
+ * @return the base path
+ */
+ public static String qualifyUser(String user) {
+ // qualify the user
+ String t = user.trim();
+ if (t.startsWith("/")) {
+ // already resolved
+ return t;
+ } else if (t.equals("~")) {
+ // self
+ return currentUsernameUnencoded();
+ } else if (t.startsWith("~")) {
+ // another user
+ // convert username to registry name
+ String convertedName = convertUsername(t.substring(1));
+
+ return RegistryPathUtils.join(RegistryConstants.PATH_USERS,
+ encodeForRegistry(convertedName));
+ } else {
+ return "/" + t;
+ }
+ }
+
+ /**
+ * Look up an external REST API
+ * @param user user which will be qualified as per {@link #qualifyUser(String)}
+ * @param serviceClass service class
+ * @param instance instance name
+ * @param api API
+ * @return the API, or an exception is raised.
+ * @throws IOException
+ */
+ public String lookupExternalRestAPI(String user,
+ String serviceClass,
+ String instance,
+ String api)
+ throws IOException {
+ String qualified = qualifyUser(user);
+ String path = servicePath(qualified, serviceClass, instance);
+ String restAPI = resolveExternalRestAPI(api, path);
+ if (restAPI == null) {
+ throw new PathNotFoundException(path + " API " + api);
+ }
+ return restAPI;
+ }
+
+ /**
+ * Resolve a service record then return an external REST API exported it.
+ *
+ * @param api API to resolve
+ * @param path path of the service record
+ * @return null if the record exists but the API is absent or it has no
+ * REST endpoints.
+ * @throws IOException resolution problems, as covered in
+ * {@link RegistryOperations#resolve(String)}
+ */
+ protected String resolveExternalRestAPI(String api, String path) throws
+ IOException {
+ ServiceRecord record = operations.resolve(path);
+ return lookupRestAPI(record, api, true);
+ }
+
+ /**
+ * Look up an external REST API endpoint
+ * @param record service record
+ * @param api URI of api
+ * @param external flag to indicate this is an external record
+ * @return the first endpoint of the implementation, or null if there
+ * is no entry for the API, implementation or it's the wrong type.
+ */
+ public static String lookupRestAPI(ServiceRecord record,
+ String api, boolean external) throws InvalidRecordException {
+ try {
+ String url = null;
+ Endpoint endpoint = getEndpoint(record, api, external);
+ List<String> addresses =
+ RegistryTypeUtils.retrieveAddressesUriType(endpoint);
+ if (addresses != null && !addresses.isEmpty()) {
+ url = addresses.get(0);
+ }
+ return url;
+ } catch (InvalidRecordException e) {
+ log.debug("looking for API {}", api, e);
+ return null;
+ }
+ }
+
+ /**
+ * Get an endpont by API
+ * @param record service record
+ * @param api API
+ * @param external flag to indicate this is an external record
+ * @return the endpoint or null
+ */
+ public static Endpoint getEndpoint(ServiceRecord record,
+ String api,
+ boolean external) {
+ return external ? record.getExternalEndpoint(api)
+ : record.getInternalEndpoint(api);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/beda1b2d/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
index ed5ceb2..e3cd508 100644
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
@@ -21,7 +21,10 @@ package org.apache.slider.core.exceptions;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.UniformInterfaceException;
import org.apache.hadoop.fs.PathAccessDeniedException;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.yarn.webapp.*;
+import javax.servlet.http.HttpServletResponse;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -32,19 +35,27 @@ import java.io.IOException;
public class ExceptionConverter {
/**
- * Convert a Jersey Exception into an IOE or subclass
+ * Uprate error codes 400 and up into faults;
+ * 404 is converted to a {@link FileNotFoundException},
+ * 401 to {@link ForbiddenException}
+ * FileNotFoundException for an unknown resource
+ * PathAccessDeniedException for access denied
+ * PathIOException for anything else
+ * @param verb HTTP Verb used
* @param targetURL URL being targeted
* @param exception original exception
* @return a new exception, the original one nested as a cause
*/
- public static IOException convertJerseyException(String targetURL,
+ public static IOException convertJerseyException(String verb,
+ String targetURL,
UniformInterfaceException exception) {
IOException ioe = null;
ClientResponse response = exception.getResponse();
if (response != null) {
int status = response.getStatus();
- if (status == 401) {
+ if (status == HttpServletResponse.SC_UNAUTHORIZED
+ || status == HttpServletResponse.SC_FORBIDDEN) {
ioe = new PathAccessDeniedException(targetURL);
}
if (status >= 400 && status < 500) {
@@ -53,7 +64,8 @@ public class ExceptionConverter {
}
if (ioe == null) {
- ioe = new IOException("Failed to GET " + targetURL + ": " + exception);
+ ioe = new PathIOException(targetURL,
+ verb + " " + targetURL + " failed: " + exception);
}
ioe.initCause(exception);
return ioe;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/beda1b2d/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
index 0a3476c..13ad5c5 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
@@ -48,7 +48,10 @@ public class CustomRegistryConstants {
"classpath:org.apache.slider.agents.oneway";
public static final String AM_IPC_PROTOCOL =
- "classpath:org.apache.slider.appmaster";
+ "classpath:org.apache.slider.appmaster.ipc";
+
+ public static final String AM_REST_BASE =
+ "classpath:org.apache.slider.client.rest";
public static final String WEB_UI = "http://";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/beda1b2d/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
index 0d32807..3ed7cc6 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
@@ -21,23 +21,21 @@ package org.apache.slider.core.registry.retrieve;
import com.beust.jcommander.Strings;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
import org.apache.hadoop.registry.client.exceptions.RegistryIOException;
-import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ServiceRecord;
+import static org.apache.slider.client.rest.RestClientRegistryBinder.*;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.ExceptionConverter;
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.core.registry.docstore.PublishedExports;
import org.apache.slider.core.registry.docstore.PublishedExportsSet;
-import org.apache.slider.core.registry.info.CustomRegistryConstants;
+import static org.apache.slider.core.registry.info.CustomRegistryConstants.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.util.List;
/**
* Registry retriever.
@@ -69,51 +67,14 @@ public class RegistryRetriever extends AMWebClient {
* not match that expected (i.e. not a list of URLs), missing endpoint...
*/
public RegistryRetriever(ServiceRecord record) throws RegistryIOException {
- Endpoint internal = record.getInternalEndpoint(
- CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API);
- String url = null;
- if (internal != null) {
- List<String> addresses = RegistryTypeUtils.retrieveAddressesUriType(
- internal);
- if (addresses != null && !addresses.isEmpty()) {
- url = addresses.get(0);
- }
- }
- internalConfigurationURL = url;
- Endpoint external = record.getExternalEndpoint(
- CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API);
- url = null;
- if (external != null) {
- List<String> addresses =
- RegistryTypeUtils.retrieveAddressesUriType(external);
- if (addresses != null && !addresses.isEmpty()) {
- url = addresses.get(0);
- }
- }
- externalConfigurationURL = url;
-
- internal = record.getInternalEndpoint(
- CustomRegistryConstants.PUBLISHER_EXPORTS_API);
- url = null;
- if (internal != null) {
- List<String> addresses = RegistryTypeUtils.retrieveAddressesUriType(
- internal);
- if (addresses != null && !addresses.isEmpty()) {
- url = addresses.get(0);
- }
- }
- internalExportsURL = url;
- external = record.getExternalEndpoint(
- CustomRegistryConstants.PUBLISHER_EXPORTS_API);
- url = null;
- if (external != null) {
- List<String> addresses =
- RegistryTypeUtils.retrieveAddressesUriType(external);
- if (addresses != null && !addresses.isEmpty()) {
- url = addresses.get(0);
- }
- }
- externalExportsURL = url;
+ internalConfigurationURL = lookupRestAPI(record,
+ PUBLISHER_CONFIGURATIONS_API, true);
+ externalConfigurationURL = lookupRestAPI(record,
+ PUBLISHER_CONFIGURATIONS_API, false);
+ externalExportsURL = lookupRestAPI(record,
+ PUBLISHER_EXPORTS_API, true);
+ internalExportsURL = lookupRestAPI(record,
+ PUBLISHER_EXPORTS_API, false);
}
/**
@@ -141,7 +102,7 @@ public class RegistryRetriever extends AMWebClient {
PublishedConfigSet configSet = webResource.get(PublishedConfigSet.class);
return configSet;
} catch (UniformInterfaceException e) {
- throw ExceptionConverter.convertJerseyException(confURL, e);
+ throw ExceptionConverter.convertJerseyException("GET", confURL, e);
}
}
@@ -176,7 +137,7 @@ public class RegistryRetriever extends AMWebClient {
PublishedExportsSet exportSet = webResource.get(PublishedExportsSet.class);
return exportSet;
} catch (UniformInterfaceException e) {
- throw ExceptionConverter.convertJerseyException(exportsUrl, e);
+ throw ExceptionConverter.convertJerseyException("GET", exportsUrl, e);
}
}
@@ -203,7 +164,7 @@ public class RegistryRetriever extends AMWebClient {
webResource.get(PublishedConfiguration.class);
return publishedConf;
} catch (UniformInterfaceException e) {
- throw ExceptionConverter.convertJerseyException(confURL, e);
+ throw ExceptionConverter.convertJerseyException("GET", confURL, e);
}
}
@@ -229,7 +190,7 @@ public class RegistryRetriever extends AMWebClient {
webResource.get(PublishedExports.class);
return publishedExports;
} catch (UniformInterfaceException e) {
- throw ExceptionConverter.convertJerseyException(exportsURL, e);
+ throw ExceptionConverter.convertJerseyException("GET", exportsURL, e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/beda1b2d/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
index 10ce1ce..cad3c69 100644
--- a/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
@@ -30,10 +30,12 @@ import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
+import org.apache.slider.core.exceptions.ExceptionConverter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.http.HttpServletResponse;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
@@ -112,39 +114,14 @@ public class UgiJerseyBinding implements
/**
* Uprate error codes 400 and up into faults;
- * 404 is converted to a {@link NotFoundException},
- * 401 to {@link ForbiddenException}
- *
- * @param verb HTTP Verb used
- * @param url URL as string
- * @param ex exception
- * @throws PathNotFoundException for an unknown resource
- * @throws PathAccessDeniedException for access denied
- * @throws PathIOException for anything else
+ * <p>
+ * see {@link ExceptionConverter#convertJerseyException(String, String, UniformInterfaceException)}
*/
public static IOException uprateFaults(HttpVerb verb, String url,
UniformInterfaceException ex)
throws IOException {
-
- ClientResponse response = ex.getResponse();
- int resultCode = response.getStatus();
- String msg = verb.toString() + " " + url;
- if (resultCode == HttpServletResponse.SC_NOT_FOUND) {
- return (IOException) new PathNotFoundException(url).initCause(ex);
- }
- if (resultCode == HttpServletResponse.SC_UNAUTHORIZED
- || resultCode == HttpServletResponse.SC_FORBIDDEN) {
- return (IOException) new PathAccessDeniedException(url).initCause(ex);
- }
- // all other error codes
-
-
- // get a string respnse
- String message = msg +
- " failed with exit code " + resultCode
- + ", message " + ex.toString();
- log.error(message, ex);
- return (IOException) new PathIOException(url, message).initCause(ex);
+ return ExceptionConverter.convertJerseyException(verb.getVerb(),
+ url, ex);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/beda1b2d/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
index afe6428..408e311 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
@@ -152,6 +152,11 @@ public class SliderAMProviderService extends AbstractProviderService implements
serviceRecord.addExternalEndpoint(
RegistryTypeUtils.webEndpoint(
CustomRegistryConstants.WEB_UI, amWebURI.toURI()));
+
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.webEndpoint(
+ CustomRegistryConstants.AM_REST_BASE, amWebURI.toURI()));
+
serviceRecord.addExternalEndpoint(
RegistryTypeUtils.restEndpoint(
CustomRegistryConstants.MANAGEMENT_REST_API,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/beda1b2d/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy
index 71f3933..ef070e0 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy
@@ -127,9 +127,9 @@ class SliderRestClientTestDelegates extends SliderTestUtils {
// fetch missing
try {
- def result = appAPI.getContainer("/unknown")
+ def result = appAPI.getContainer("unknown")
fail("expected an error, got $result")
- } catch (PathNotFoundException e) {
+ } catch (FileNotFoundException e) {
// expected
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/beda1b2d/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
index 8ee88c3..3c3dd52 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
@@ -28,12 +28,17 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.commons.httpclient.HttpClient
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager
+import org.apache.hadoop.registry.client.api.RegistryOperations
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.client.SliderClient
+import org.apache.slider.client.rest.RestClientFactory
+import org.apache.slider.client.rest.SliderApplicationAPI
+import org.apache.slider.common.SliderKeys
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.registry.info.CustomRegistryConstants
import org.apache.slider.core.restclient.HttpOperationResponse
import org.junit.Test
@@ -109,16 +114,16 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
log.info "Content type: ${response.contentType}"
-
+ def ugiClient = createUGIJerseyClient();
describe "Proxy SliderRestClient Tests"
SliderRestClientTestDelegates proxySliderRestClient =
- new SliderRestClientTestDelegates(proxyAM, createUGIJerseyClient())
+ new SliderRestClientTestDelegates(proxyAM, ugiClient)
proxySliderRestClient.testSuiteGetOperations()
describe "Direct SliderRestClient Tests"
SliderRestClientTestDelegates directSliderRestClient =
- new SliderRestClientTestDelegates(directAM, createUGIJerseyClient())
+ new SliderRestClientTestDelegates(directAM, ugiClient)
directSliderRestClient.testSuiteGetOperations()
directSliderRestClient.testSuiteComplexVerbs()
@@ -126,13 +131,13 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
describe "Proxy Jersey Tests"
JerseyTestDelegates proxyJerseyTests =
- new JerseyTestDelegates(proxyAM, createUGIJerseyClient())
+ new JerseyTestDelegates(proxyAM, ugiClient)
proxyJerseyTests.testSuiteGetOperations()
describe "Direct Jersey Tests"
JerseyTestDelegates directJerseyTests =
- new JerseyTestDelegates(directAM, createUGIJerseyClient())
+ new JerseyTestDelegates(directAM, ugiClient)
directJerseyTests.testSuiteGetOperations()
directJerseyTests.testSuiteComplexVerbs()
@@ -150,7 +155,19 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
// and via the proxy
proxied.testSuiteComplexVerbs()
}
-
+
+ // create the Rest client via the registry
+
+ RegistryOperations operations = client.registryOperations;
+ def restClientFactory = new RestClientFactory(
+ operations,
+ ugiClient,
+ "~", SliderKeys.APP_TYPE,
+ clustername)
+ def sliderApplicationApi = restClientFactory.createSliderApplicationApi();
+ sliderApplicationApi.desiredModel
+ sliderApplicationApi.resolvedModel
+ sliderApplicationApi.ping("registry located")
/* DISABLED: this client does not pass the tests.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/beda1b2d/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
index fd54b55..e6b8bc4 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
@@ -18,14 +18,19 @@
package org.apache.slider.funtest.lifecycle
+import com.sun.jersey.api.client.Client
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.registry.client.api.RegistryOperations
import org.apache.hadoop.security.UserGroupInformation
import org.apache.hadoop.yarn.webapp.ForbiddenException
import org.apache.slider.agent.rest.JerseyTestDelegates
import org.apache.slider.agent.rest.RestTestDelegates
import org.apache.slider.agent.rest.SliderRestClientTestDelegates
+import org.apache.slider.client.SliderClient
+import org.apache.slider.client.rest.RestClientFactory
import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderKeys
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
@@ -118,26 +123,27 @@ public class AgentWebPagesIT extends AgentCommandTestBase
describe "Proxy Jersey Tests"
+ Client ugiClient = createUGIJerseyClient()
JerseyTestDelegates proxyJerseyTests =
- new JerseyTestDelegates(proxyAM, createUGIJerseyClient())
+ new JerseyTestDelegates(proxyAM, ugiClient)
proxyJerseyTests.testSuiteGetOperations()
describe "Direct Jersey Tests"
JerseyTestDelegates directJerseyTests =
- new JerseyTestDelegates(directAM, createUGIJerseyClient())
+ new JerseyTestDelegates(directAM, ugiClient)
directJerseyTests.testSuiteGetOperations()
directJerseyTests.testSuiteComplexVerbs()
describe "Proxy SliderRestClient Tests"
SliderRestClientTestDelegates proxySliderRestClient =
- new SliderRestClientTestDelegates(proxyAM, createUGIJerseyClient())
+ new SliderRestClientTestDelegates(proxyAM, ugiClient)
proxySliderRestClient.testSuiteGetOperations()
if (!wsBackDoorRequired) {
proxySliderRestClient.testSuiteComplexVerbs()
}
describe "Direct SliderRestClient Tests"
SliderRestClientTestDelegates directSliderRestClient =
- new SliderRestClientTestDelegates(directAM, createUGIJerseyClient())
+ new SliderRestClientTestDelegates(directAM, ugiClient)
directSliderRestClient.testSuiteGetOperations()
directSliderRestClient.testSuiteComplexVerbs()
@@ -155,10 +161,24 @@ public class AgentWebPagesIT extends AgentCommandTestBase
// these tests use the Jersey client without the Hadoop-specific
// SPNEGO
- JerseyTestDelegates baseicJerseyClientTests =
+ JerseyTestDelegates basicJerseyClientTests =
new JerseyTestDelegates(proxyAM, createBasicJerseyClient())
- baseicJerseyClientTests.testSuiteGetOperations()
+ basicJerseyClientTests.testSuiteGetOperations()
}
+
+ // create the Rest client via the registry
+
+ //get a slider client against the cluster
+
+ SliderClient sliderClient = bondToCluster(SLIDER_CONFIG, CLUSTER)
+ RegistryOperations operations = sliderClient.registryOperations;
+ def restClientFactory = new RestClientFactory(
+ operations, ugiClient,
+ "~", SliderKeys.APP_TYPE, CLUSTER)
+ def sliderApplicationApi = restClientFactory.createSliderApplicationApi();
+ sliderApplicationApi.desiredModel
+ sliderApplicationApi.resolvedModel
+ sliderApplicationApi.ping("registry located")
// finally, stop the AM
direct.testStop();
[16/28] incubator-slider git commit: SLIDER-782 factoring out base
for other jersey clients;
Posted by el...@apache.org.
SLIDER-782 factoring out base for other jersey clients;
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b0963bd0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b0963bd0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b0963bd0
Branch: refs/heads/develop
Commit: b0963bd02a3cd1994848facec16c2076d79f9a38
Parents: b7d5099
Author: Steve Loughran <st...@apache.org>
Authored: Fri Jan 23 12:45:22 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Jan 23 12:45:22 2015 +0000
----------------------------------------------------------------------
.../apache/slider/client/SliderRestClient.java | 343 -------------------
.../slider/client/rest/BaseRestClient.java | 154 +++++++++
.../client/rest/SliderApplicationAPI.java | 274 +++++++++++++++
.../rest/SliderRestClientTestDelegates.groovy | 38 +-
4 files changed, 447 insertions(+), 362 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b0963bd0/slider-core/src/main/java/org/apache/slider/client/SliderRestClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderRestClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderRestClient.java
deleted file mode 100644
index 9de2551..0000000
--- a/slider-core/src/main/java/org/apache/slider/client/SliderRestClient.java
+++ /dev/null
@@ -1,343 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.client;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.GenericType;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.representation.Form;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.slider.api.types.SerializedComponentInformation;
-import org.apache.slider.api.types.SerializedContainerInformation;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTree;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.restclient.HttpVerb;
-import org.apache.slider.core.restclient.UgiJerseyBinding;
-import org.apache.slider.server.appmaster.web.rest.application.resources.PingResource;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.core.MediaType;
-
-import java.io.IOException;
-import java.util.Map;
-
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
-
-public class SliderRestClient extends AbstractService {
- private static final Logger log =
- LoggerFactory.getLogger(SliderRestClient.class);
- private final Client jersey;
- private WebResource appmaster;
- private WebResource appResource;
-
- public SliderRestClient(String name, Client jersey, WebResource appmaster) {
- super(name);
- Preconditions.checkNotNull(jersey, "null jersey");
- this.jersey = jersey;
- if (appmaster !=null) {
- bindToAppmaster(appmaster);
- }
- }
-
- public SliderRestClient(Client jersey, WebResource appmaster) {
- this("SliderRestClient", jersey, appmaster);
- }
-
- public Client getJersey() {
- return jersey;
- }
-
- /**
- * Bind/rebind to the AM
- * @param appmaster AM
- */
- public void bindToAppmaster(WebResource appmaster) {
- this.appmaster = appmaster;
- this.appResource = appmaster.path(SLIDER_PATH_APPLICATION);
- }
-
- public WebResource getAppmaster() {
- return appmaster;
- }
-
- /**
- * Create a resource under the application path
- * @param subpath
- * @return an resource under the application path
- */
- public WebResource applicationResource(String subpath) {
- Preconditions.checkArgument(!StringUtils.isEmpty(subpath),
- "empty path");
- Preconditions.checkNotNull(appResource, "Null app resource");
- return appResource.path(subpath);
- }
-
- /**
- * Get operation against a path under the Application
- * @param <T> type expected
- * @param subpath path
- * @param c class to instantiate
- * @return instance
- * @throws IOException on any problem
- */
- public <T> T getApplicationResource(String subpath, Class<T> c)
- throws IOException {
- return appResourceOperation(HttpVerb.GET, subpath, c);
- }
-
- /**
- * Get operation against a path under the Application
- * @param <T> type expected
- * @param subpath path
- * @param t type info
- * @return instance
- * @throws IOException on any problem
- */
- public <T> T getApplicationResource(String subpath, GenericType<T> t)
- throws IOException {
- return appResourceOperation(HttpVerb.GET, subpath, t);
- }
-
- /**
- *
- * @param method method to exec
- * @param <T> type expected
- * @param subpath path
- * @param c class to instantiate
- * @return instance
- * @throws IOException on any problem
- */
- public <T> T appResourceOperation(HttpVerb method, String subpath, Class<T> c)
- throws IOException {
- WebResource resource = applicationResource(subpath);
- return exec(method, resource, c);
- }
-
-
- /**
- * Get operation against a path under the Application
- * @param <T> type expected
- * @param subpath path
- * @param t type info
- * @return instance
- * @throws IOException on any problem
- */
- public <T> T appResourceOperation(HttpVerb method, String subpath,
- GenericType<T> t)
- throws IOException {
- WebResource resource = applicationResource(subpath);
- return exec(method, resource, t);
- }
-
- /**
- * Execute the operation. Failures are raised as IOException subclasses
- * @param method method to execute
- * @param resource resource to work against
- * @param c class to build
- * @param <T> type expected
- * @return an instance of the type T
- * @throws IOException on any failure
- */
- public <T> T exec(HttpVerb method, WebResource resource, Class<T> c)
- throws IOException {
- try {
- Preconditions.checkArgument(c != null);
- resource.accept(MediaType.APPLICATION_JSON_TYPE);
- return (T) resource.method(method.getVerb(), c);
- } catch (UniformInterfaceException ex) {
- throw UgiJerseyBinding.uprateFaults(method, resource.getURI().toString(),
- ex);
- }
- }
-
-
- /**
- * Execute the operation. Failures are raised as IOException subclasses
- * @param method method to execute
- * @param resource resource to work against
- * @param generic type to work with
- * @param <T> type expected
- * @return an instance of the type T
- * @throws IOException on any failure
- */
- public <T> T exec(HttpVerb method, WebResource resource, GenericType<T> t)
- throws IOException {
- try {
- Preconditions.checkArgument(t != null);
- resource.accept(MediaType.APPLICATION_JSON_TYPE);
- return resource.method(method.getVerb(), t);
- } catch (UniformInterfaceException ex) {
- throw UgiJerseyBinding.uprateFaults(method, resource.getURI().toString(),
- ex);
- }
- }
-
-
-
- /**
- * Get the aggregate desired model
- * @return the aggregate configuration of what was asked for
- * —before resolution has taken place
- * @throws IOException on any failure
- */
- public AggregateConf getDesiredModel() throws IOException {
- return getApplicationResource(MODEL_DESIRED, AggregateConf.class);
- }
-
-
- /**
- * Get the desired application configuration
- * @return the application configuration asked for
- * —before resolution has taken place
- * @throws IOException on any failure
- */
- public ConfTreeOperations getDesiredAppconf() throws IOException {
- ConfTree resource =
- getApplicationResource(MODEL_DESIRED_APPCONF, ConfTree.class);
- return new ConfTreeOperations(resource);
- }
-
- /**
- * Get the desired YARN resources
- * @return the resources asked for
- * —before resolution has taken place
- * @throws IOException on any failure
- */
- public ConfTreeOperations getDesiredYarnResources() throws IOException {
- ConfTree resource =
- getApplicationResource(MODEL_DESIRED_RESOURCES, ConfTree.class);
- return new ConfTreeOperations(resource);
- }
-
- /**
- * Get the aggregate resolved model
- * @return the aggregate configuration of what was asked for
- * —after resolution has taken place
- * @throws IOException on any failure
- */
- public AggregateConf getResolvedModel() throws IOException {
- return getApplicationResource(MODEL_RESOLVED, AggregateConf.class);
- }
-
-
- /**
- * Get the resolved application configuration
- * @return the application configuration asked for
- * —after resolution has taken place
- * @throws IOException on any failure
- */
- public ConfTreeOperations getResolvedAppconf() throws IOException {
- ConfTree resource =
- getApplicationResource(MODEL_RESOLVED_APPCONF, ConfTree.class);
- return new ConfTreeOperations(resource);
- }
-
- /**
- * Get the resolved YARN resources
- * @return the resources asked for
- * —after resolution has taken place
- * @throws IOException on any failure
- */
- public ConfTreeOperations getResolvedYarnResources() throws IOException {
- ConfTree resource =
- getApplicationResource(MODEL_RESOLVED_RESOURCES, ConfTree.class);
- return new ConfTreeOperations(resource);
- }
-
- /**
- * Get the live YARN resources
- * @return the live set of resources in the cluster
- * @throws IOException on any failure
- */
- public ConfTreeOperations getLiveYarnResources() throws IOException {
- ConfTree resource =
- getApplicationResource(LIVE_RESOURCES, ConfTree.class);
- return new ConfTreeOperations(resource);
- }
-
- /**
- * Get a map of live containers [containerId:info]
- * @return a possibly empty list of serialized containers
- * @throws IOException on any failure
- */
- public Map<String, SerializedContainerInformation> enumContainers() throws
- IOException {
- return getApplicationResource(LIVE_CONTAINERS,
- new GenericType<Map<String, SerializedContainerInformation>>() {
- });
- }
-
- /**
- * Get a container from the container Id
- * @param containerId YARN container ID
- * @return the container information
- * @throws IOException on any failure
- */
- public SerializedContainerInformation getContainer( String containerId) throws
- IOException {
- return getApplicationResource(LIVE_CONTAINERS + "/" + containerId,
- SerializedContainerInformation.class);
- }
-
- /**
- * List all components into a map of [name:info]
- * @return a possibly empty map of components
- * @throws IOException on any failure
- */
- public Map<String, SerializedComponentInformation> enumComponents() throws
- IOException {
- return getApplicationResource(LIVE_COMPONENTS,
- new GenericType<Map<String, SerializedComponentInformation>>() {
- });
- }
-
- /**
- * Get information about a component
- * @param componentName name of the component
- * @return the component details
- * @throws IOException on any failure
- */
- public SerializedComponentInformation getComponent(String componentName) throws
- IOException {
- return getApplicationResource(LIVE_COMPONENTS + "/" + componentName,
- SerializedComponentInformation.class);
- }
-
- /**
- * Ping as a post
- * @param text text to include
- * @return the response
- * @throws IOException on any failure
- */
- public PingResource ping(String text) throws IOException {
- WebResource pingOut = applicationResource(ACTION_PING);
- pingOut.accept(MediaType.APPLICATION_JSON_TYPE);
- pingOut.type(MediaType.APPLICATION_JSON_TYPE);
- Form f = new Form();
- f.add("text", text);
- return pingOut.post(PingResource.class, f);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b0963bd0/slider-core/src/main/java/org/apache/slider/client/rest/BaseRestClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/rest/BaseRestClient.java b/slider-core/src/main/java/org/apache/slider/client/rest/BaseRestClient.java
new file mode 100644
index 0000000..46be2aa
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/client/rest/BaseRestClient.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client.rest;
+
+import com.google.common.base.Preconditions;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.GenericType;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
+import org.apache.slider.core.restclient.HttpVerb;
+import org.apache.slider.core.restclient.UgiJerseyBinding;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.MediaType;
+import java.io.IOException;
+import java.net.URI;
+
+
+/**
+ * This is a base class for Jersey Rest clients in Slider.
+ * It supports bonding to an AM and the execution of operations —with
+ * exceptions uprated to IOExceptions when needed.
+ * <p>
+ * Subclasses can use these operations to provide an API-like view
+ * of the REST model
+ */
+public class BaseRestClient {
+ private static final Logger log =
+ LoggerFactory.getLogger(BaseRestClient.class);
+ private final Client client;
+ private WebResource appmaster;
+
+ public BaseRestClient(
+ Client client,
+ WebResource appmaster) {
+ Preconditions.checkNotNull(client, "null jersey client");
+ this.client = client;
+ if (appmaster != null) {
+ bindToAppmaster(appmaster);
+ }
+ }
+
+ public Client getClient() {
+ return client;
+ }
+
+ /**
+ * Bind/rebind to the AM
+ * @param appmaster AM
+ */
+ public void bindToAppmaster(WebResource appmaster) {
+ Preconditions.checkArgument(appmaster != null, " Null appmaster");
+ this.appmaster = appmaster;
+ }
+
+ public WebResource getAppmaster() {
+ return appmaster;
+ }
+
+ /**
+ * Execute the operation. Failures are raised as IOException subclasses
+ * @param method method to execute
+ * @param resource resource to work against
+ * @param c class to build
+ * @param <T> type expected
+ * @return an instance of the type T
+ * @throws IOException on any failure
+ */
+ public <T> T exec(HttpVerb method, WebResource resource, Class<T> c)
+ throws IOException {
+ try {
+ Preconditions.checkArgument(c != null);
+ resource.accept(MediaType.APPLICATION_JSON_TYPE);
+ return (T) resource.method(method.getVerb(), c);
+ } catch (UniformInterfaceException ex) {
+ throw UgiJerseyBinding.uprateFaults(method,
+ resource.getURI().toString(),
+ ex);
+ }
+ }
+
+ /**
+ * Execute the operation. Failures are raised as IOException subclasses
+ * @param method method to execute
+ * @param resource resource to work against
+ * @param t type to work with
+ * @param <T> type expected
+ * @return an instance of the type T
+ * @throws IOException on any failure
+ */
+ public <T> T exec(HttpVerb method, WebResource resource, GenericType<T> t)
+ throws IOException {
+ try {
+ Preconditions.checkArgument(t != null);
+ resource.accept(MediaType.APPLICATION_JSON_TYPE);
+ return resource.method(method.getVerb(), t);
+ } catch (UniformInterfaceException ex) {
+ throw UgiJerseyBinding.uprateFaults(method, resource.getURI().toString(),
+ ex);
+ }
+ }
+
+ /**
+ * Create a Web resource from the client.
+ *
+ * @param u the URI of the resource.
+ * @return the Web resource.
+ */
+ protected WebResource resource(URI u) {
+ return client.resource(u);
+ }
+
+ /**
+ * Create a Web resource from the client.
+ *
+ * @param u the URI of the resource.
+ * @return the Web resource.
+ */
+
+ protected WebResource resource(String url) {
+ WebResource resource = client.resource(url);
+ return resource;
+ }
+
+ protected WebResource jsonResource(String url) {
+ WebResource resource = resource(url);
+ resource.type(MediaType.APPLICATION_JSON);
+ return resource;
+ }
+
+
+ protected WebResource jsonResource(URI u) {
+ WebResource resource = resource(u);
+ resource.type(MediaType.APPLICATION_JSON);
+ return resource;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b0963bd0/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationAPI.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationAPI.java b/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationAPI.java
new file mode 100644
index 0000000..16af359
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationAPI.java
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client.rest;
+
+import com.google.common.base.Preconditions;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.GenericType;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.representation.Form;
+import org.apache.commons.lang.StringUtils;
+import org.apache.slider.api.types.SerializedComponentInformation;
+import org.apache.slider.api.types.SerializedContainerInformation;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.restclient.HttpVerb;
+import org.apache.slider.server.appmaster.web.rest.application.resources.PingResource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.MediaType;
+import java.io.IOException;
+import java.util.Map;
+
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
+
+public class SliderApplicationAPI extends BaseRestClient {
+ private static final Logger log =
+ LoggerFactory.getLogger(SliderApplicationAPI.class);
+ private WebResource appResource;
+
+ public SliderApplicationAPI(Client jerseyClient, WebResource appmaster) {
+ super(jerseyClient, appmaster);
+ }
+
+ /**
+ * Bind/rebind to the AM
+ * @param appmaster AM
+ */
+ public void bindToAppmaster(WebResource appmaster) {
+ super.bindToAppmaster(appmaster);
+ this.appResource = appmaster.path(SLIDER_PATH_APPLICATION);
+ }
+
+ /**
+ * Create a resource under the application path
+ * @param subpath
+ * @return an resource under the application path
+ */
+ public WebResource applicationResource(String subpath) {
+ Preconditions.checkArgument(!StringUtils.isEmpty(subpath),
+ "empty path");
+ Preconditions.checkNotNull(appResource, "Null app resource");
+ return appResource.path(subpath);
+ }
+
+ /**
+ * Get operation against a path under the Application
+ * @param <T> type expected
+ * @param subpath path
+ * @param c class to instantiate
+ * @return instance
+ * @throws IOException on any problem
+ */
+ public <T> T getApplicationResource(String subpath, Class<T> c)
+ throws IOException {
+ return appResourceOperation(HttpVerb.GET, subpath, c);
+ }
+
+ /**
+ * Get operation against a path under the Application
+ * @param <T> type expected
+ * @param subpath path
+ * @param t type info
+ * @return instance
+ * @throws IOException on any problem
+ */
+ public <T> T getApplicationResource(String subpath, GenericType<T> t)
+ throws IOException {
+ return appResourceOperation(HttpVerb.GET, subpath, t);
+ }
+
+ /**
+ *
+ * @param method method to exec
+ * @param <T> type expected
+ * @param subpath path
+ * @param c class to instantiate
+ * @return instance
+ * @throws IOException on any problem
+ */
+ public <T> T appResourceOperation(HttpVerb method, String subpath, Class<T> c)
+ throws IOException {
+ WebResource resource = applicationResource(subpath);
+ return exec(method, resource, c);
+ }
+
+
+ /**
+ * Get operation against a path under the Application
+ * @param <T> type expected
+ * @param subpath path
+ * @param t type info
+ * @return instance
+ * @throws IOException on any problem
+ */
+ public <T> T appResourceOperation(HttpVerb method, String subpath,
+ GenericType<T> t)
+ throws IOException {
+ WebResource resource = applicationResource(subpath);
+ return exec(method, resource, t);
+ }
+
+
+ /**
+ * Get the aggregate desired model
+ * @return the aggregate configuration of what was asked for
+ * —before resolution has taken place
+ * @throws IOException on any failure
+ */
+ public AggregateConf getDesiredModel() throws IOException {
+ return getApplicationResource(MODEL_DESIRED, AggregateConf.class);
+ }
+
+ /**
+ * Get the desired application configuration
+ * @return the application configuration asked for
+ * —before resolution has taken place
+ * @throws IOException on any failure
+ */
+ public ConfTreeOperations getDesiredAppconf() throws IOException {
+ ConfTree resource =
+ getApplicationResource(MODEL_DESIRED_APPCONF, ConfTree.class);
+ return new ConfTreeOperations(resource);
+ }
+
+ /**
+ * Get the desired YARN resources
+ * @return the resources asked for
+ * —before resolution has taken place
+ * @throws IOException on any failure
+ */
+ public ConfTreeOperations getDesiredYarnResources() throws IOException {
+ ConfTree resource =
+ getApplicationResource(MODEL_DESIRED_RESOURCES, ConfTree.class);
+ return new ConfTreeOperations(resource);
+ }
+
+ /**
+ * Get the aggregate resolved model
+ * @return the aggregate configuration of what was asked for
+ * —after resolution has taken place
+ * @throws IOException on any failure
+ */
+ public AggregateConf getResolvedModel() throws IOException {
+ return getApplicationResource(MODEL_RESOLVED, AggregateConf.class);
+ }
+
+
+ /**
+ * Get the resolved application configuration
+ * @return the application configuration asked for
+ * —after resolution has taken place
+ * @throws IOException on any failure
+ */
+ public ConfTreeOperations getResolvedAppconf() throws IOException {
+ ConfTree resource =
+ getApplicationResource(MODEL_RESOLVED_APPCONF, ConfTree.class);
+ return new ConfTreeOperations(resource);
+ }
+
+ /**
+ * Get the resolved YARN resources
+ * @return the resources asked for
+ * —after resolution has taken place
+ * @throws IOException on any failure
+ */
+ public ConfTreeOperations getResolvedYarnResources() throws IOException {
+ ConfTree resource =
+ getApplicationResource(MODEL_RESOLVED_RESOURCES, ConfTree.class);
+ return new ConfTreeOperations(resource);
+ }
+
+ /**
+ * Get the live YARN resources
+ * @return the live set of resources in the cluster
+ * @throws IOException on any failure
+ */
+ public ConfTreeOperations getLiveYarnResources() throws IOException {
+ ConfTree resource =
+ getApplicationResource(LIVE_RESOURCES, ConfTree.class);
+ return new ConfTreeOperations(resource);
+ }
+
+ /**
+ * Get a map of live containers [containerId:info]
+ * @return a possibly empty list of serialized containers
+ * @throws IOException on any failure
+ */
+ public Map<String, SerializedContainerInformation> enumContainers() throws
+ IOException {
+ return getApplicationResource(LIVE_CONTAINERS,
+ new GenericType<Map<String, SerializedContainerInformation>>() {
+ });
+ }
+
+ /**
+ * Get a container from the container Id
+ * @param containerId YARN container ID
+ * @return the container information
+ * @throws IOException on any failure
+ */
+ public SerializedContainerInformation getContainer( String containerId) throws
+ IOException {
+ return getApplicationResource(LIVE_CONTAINERS + "/" + containerId,
+ SerializedContainerInformation.class);
+ }
+
+ /**
+ * List all components into a map of [name:info]
+ * @return a possibly empty map of components
+ * @throws IOException on any failure
+ */
+ public Map<String, SerializedComponentInformation> enumComponents() throws
+ IOException {
+ return getApplicationResource(LIVE_COMPONENTS,
+ new GenericType<Map<String, SerializedComponentInformation>>() {
+ });
+ }
+
+ /**
+ * Get information about a component
+ * @param componentName name of the component
+ * @return the component details
+ * @throws IOException on any failure
+ */
+ public SerializedComponentInformation getComponent(String componentName) throws
+ IOException {
+ return getApplicationResource(LIVE_COMPONENTS + "/" + componentName,
+ SerializedComponentInformation.class);
+ }
+
+ /**
+ * Ping as a post
+ * @param text text to include
+ * @return the response
+ * @throws IOException on any failure
+ */
+ public PingResource ping(String text) throws IOException {
+ WebResource pingOut = applicationResource(ACTION_PING);
+ pingOut.accept(MediaType.APPLICATION_JSON_TYPE);
+ pingOut.type(MediaType.APPLICATION_JSON_TYPE);
+ Form f = new Form();
+ f.add("text", text);
+ return pingOut.post(PingResource.class, f);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b0963bd0/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy
index 46dba1d..71f3933 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy
@@ -26,7 +26,7 @@ import org.apache.hadoop.fs.PathNotFoundException
import org.apache.slider.api.StateValues
import org.apache.slider.api.types.SerializedComponentInformation
import org.apache.slider.api.types.SerializedContainerInformation
-import org.apache.slider.client.SliderRestClient
+import org.apache.slider.client.rest.SliderApplicationAPI
import org.apache.slider.core.conf.ConfTree
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource
@@ -55,7 +55,7 @@ class SliderRestClientTestDelegates extends SliderTestUtils {
final String appmaster;
final String application;
final Client jersey;
- final SliderRestClient restClient;
+ final SliderApplicationAPI appAPI;
SliderRestClientTestDelegates(String appmaster, Client jersey) {
@@ -64,27 +64,27 @@ class SliderRestClientTestDelegates extends SliderTestUtils {
application = appendToURL(appmaster, SLIDER_PATH_APPLICATION)
WebResource amResource = jersey.resource(appmaster)
amResource.type(MediaType.APPLICATION_JSON)
- restClient = new SliderRestClient(jersey, amResource)
+ appAPI = new SliderApplicationAPI(jersey, amResource)
}
public void testGetDesiredModel() throws Throwable {
- restClient.getDesiredModel()
- restClient.getDesiredAppconf()
- restClient.getDesiredYarnResources()
+ appAPI.getDesiredModel()
+ appAPI.getDesiredAppconf()
+ appAPI.getDesiredYarnResources()
}
public void testGetResolvedModel() throws Throwable {
- restClient.getResolvedModel()
- restClient.getResolvedAppconf()
- restClient.getResolvedYarnResources()
+ appAPI.getResolvedModel()
+ appAPI.getResolvedAppconf()
+ appAPI.getResolvedYarnResources()
}
public void testLiveResources() throws Throwable {
describe "Live Resources"
- ConfTreeOperations tree = restClient.getLiveYarnResources()
+ ConfTreeOperations tree = appAPI.getLiveYarnResources()
log.info tree.toString()
def liveAM = tree.getComponent(COMPONENT_AM)
@@ -102,7 +102,7 @@ class SliderRestClientTestDelegates extends SliderTestUtils {
public void testLiveContainers() throws Throwable {
describe "Application REST ${LIVE_CONTAINERS}"
- Map<String, SerializedContainerInformation> containers = restClient.enumContainers()
+ Map<String, SerializedContainerInformation> containers = appAPI.enumContainers()
assert containers.size() == 1
log.info "${containers}"
SerializedContainerInformation amContainerInfo =
@@ -122,12 +122,12 @@ class SliderRestClientTestDelegates extends SliderTestUtils {
describe "containers"
SerializedContainerInformation amContainerInfo2 =
- restClient.getContainer(amContainerId)
+ appAPI.getContainer(amContainerId)
assert amContainerInfo2.containerId == amContainerId
// fetch missing
try {
- def result = restClient.getContainer("/unknown")
+ def result = appAPI.getContainer("/unknown")
fail("expected an error, got $result")
} catch (PathNotFoundException e) {
// expected
@@ -137,7 +137,7 @@ class SliderRestClientTestDelegates extends SliderTestUtils {
describe "components"
Map<String, SerializedComponentInformation> components =
- restClient.enumComponents()
+ appAPI.enumComponents()
// two components
assert components.size() >= 1
@@ -146,7 +146,7 @@ class SliderRestClientTestDelegates extends SliderTestUtils {
SerializedComponentInformation amComponentInfo =
(SerializedComponentInformation) components[COMPONENT_AM]
- SerializedComponentInformation amFullInfo = restClient.getComponent(COMPONENT_AM)
+ SerializedComponentInformation amFullInfo = appAPI.getComponent(COMPONENT_AM)
assert amFullInfo.containers.size() == 1
assert amFullInfo.containers[0] == amContainerId
@@ -166,14 +166,14 @@ class SliderRestClientTestDelegates extends SliderTestUtils {
MODEL,
ApplicationResource.MODEL_ENTRIES)
- def unresolvedConf = restClient.getDesiredModel()
+ def unresolvedConf = appAPI.getDesiredModel()
// log.info "Unresolved \n$unresolvedConf"
def unresolvedAppConf = unresolvedConf.appConfOperations
def sam = "slider-appmaster"
assert unresolvedAppConf.getComponentOpt(sam,
TEST_GLOBAL_OPTION, "") == ""
- def resolvedConf = restClient.getResolvedModel()
+ def resolvedConf = appAPI.getResolvedModel()
assert resolvedConf.appConfOperations.getComponentOpt(
sam, TEST_GLOBAL_OPTION, "") == TEST_GLOBAL_OPTION_PRESENT
@@ -184,7 +184,7 @@ class SliderRestClientTestDelegates extends SliderTestUtils {
- def resolvedAppconf = restClient.getResolvedAppconf()
+ def resolvedAppconf = appAPI.getResolvedAppconf()
assert resolvedAppconf.
components[sam][TEST_GLOBAL_OPTION] == TEST_GLOBAL_OPTION_PRESENT
}
@@ -193,7 +193,7 @@ class SliderRestClientTestDelegates extends SliderTestUtils {
// GET
describe "pinging"
- restClient.ping("hello")
+ appAPI.ping("hello")
}
[06/28] incubator-slider git commit: SLIDER-719 initial jersey tests
standalone
Posted by el...@apache.org.
SLIDER-719 initial jersey tests standalone
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/72e47903
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/72e47903
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/72e47903
Branch: refs/heads/develop
Commit: 72e47903eca51ca54b9e53b05816eea32ebe08ca
Parents: 54f1f12
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jan 20 17:04:49 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jan 20 17:04:49 2015 +0000
----------------------------------------------------------------------
.../agent/rest/JerseyTestDelegates.groovy | 116 +++++++++++++------
.../slider/agent/rest/TestStandaloneREST.groovy | 34 ++++--
2 files changed, 105 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/72e47903/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
index 32ca2cc..a06be23 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
@@ -20,6 +20,7 @@ package org.apache.slider.agent.rest
import com.sun.jersey.api.client.Client
import com.sun.jersey.api.client.WebResource
+import com.sun.jersey.api.client.UniformInterfaceException
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.webapp.NotFoundException
@@ -65,20 +66,42 @@ class JerseyTestDelegates extends SliderTestUtils {
JerseyTestDelegates(String appmaster, Client jersey) {
this.appmaster = appmaster
- application = appendToURL(appmaster, RestPaths.SLIDER_PATH_APPLICATION)
+ this.application = appendToURL(appmaster, RestPaths.SLIDER_PATH_APPLICATION)
this.jersey = jersey
}
/**
* <T> T get(Class<T> c)
- * Get operation against the EM
+ * Get operation against a path under the Application
+ * @param subpath path
+ * @return
+ */
+ public <T> T jGetApplicationResource(String subpath, Class<T> c) {
+ assert c
+ def appPath = appendToURL(SLIDER_PATH_APPLICATION, subpath)
+ WebResource webResource = buildResource(appPath)
+ (T)webResource.get(c)
+ }
+
+ /**
+ * <T> T get(Class<T> c)
+ * Get operation against a path under the AM
* @param path path
* @return
*/
- public <T> T jerseyGet(String path, Class<T> c) {
+ public <T> T jGetAMResource(String path, Class<T> c) {
assert c
WebResource webResource = buildResource(path)
- webResource.get(c)
+ (T)webResource.get(c)
+ }
+
+ /**
+ * Get operation against a path under the AM
+ * @param path path
+ * @return the string value
+ */
+ public String jerseyGet(String path) {
+ return jGetAMResource(path, String.class)
}
/**
@@ -88,7 +111,7 @@ class JerseyTestDelegates extends SliderTestUtils {
*/
public WebResource buildResource(String path) {
assert path
- String fullpath = appendToURL(application, path)
+ String fullpath = appendToURL(appmaster, path)
WebResource webResource = jersey.resource(fullpath)
webResource.type(MediaType.APPLICATION_JSON)
log.info("HTTP operation against $fullpath");
@@ -96,27 +119,39 @@ class JerseyTestDelegates extends SliderTestUtils {
}
public void testJerseyGetConftree() throws Throwable {
- jerseyGet(LIVE_RESOURCES, ConfTree.class);
+ jGetApplicationResource(LIVE_RESOURCES, ConfTree.class);
}
public void testCodahaleOperations() throws Throwable {
describe "Codahale operations"
- jerseyGet(appmaster)
- jerseyGet(appmaster, SYSTEM_THREADS)
- jerseyGet(appmaster, SYSTEM_HEALTHCHECK)
- jerseyGet(appmaster, SYSTEM_PING)
- jerseyGet(appmaster, SYSTEM_METRICS_JSON)
+ jerseyGet("/")
+ jerseyGet(SYSTEM_THREADS)
+ jerseyGet(SYSTEM_HEALTHCHECK)
+ jerseyGet(SYSTEM_PING)
+ jerseyGet(SYSTEM_METRICS_JSON)
}
public void logCodahaleMetrics() {
// query Coda Hale metrics
- log.info jerseyGet(appmaster, SYSTEM_HEALTHCHECK)
- log.info jerseyGet(appmaster, SYSTEM_METRICS)
+ log.info jerseyGet(SYSTEM_HEALTHCHECK)
+ log.info jerseyGet(SYSTEM_METRICS)
}
- public void testLiveResources() throws Throwable {
+ public <T> T fetchJType(
+ String subpath, Class<T> clazz) {
+ (T)jGetApplicationResource(subpath, clazz)
+ }
+
+ public ConfTreeOperations jGetConfigTree(
+ String path) {
+ ConfTree ctree = jGetApplicationResource(path, ConfTree)
+ ConfTreeOperations tree = new ConfTreeOperations(ctree)
+ return tree
+ }
+
+ public void testLiveResources() throws Throwable {
describe "Live Resources"
- ConfTreeOperations tree = fetchConfigTree(appmaster, LIVE_RESOURCES)
+ ConfTreeOperations tree = jGetConfigTree(LIVE_RESOURCES)
log.info tree.toString()
def liveAM = tree.getComponent(COMPONENT_AM)
@@ -135,7 +170,7 @@ class JerseyTestDelegates extends SliderTestUtils {
describe "Application REST ${LIVE_CONTAINERS}"
Map<String, SerializedContainerInformation> containers =
- fetchType(HashMap, appmaster, LIVE_CONTAINERS)
+ jGetApplicationResource(LIVE_CONTAINERS, HashMap)
assert containers.size() == 1
log.info "${containers}"
SerializedContainerInformation amContainerInfo =
@@ -155,16 +190,20 @@ class JerseyTestDelegates extends SliderTestUtils {
describe "containers"
SerializedContainerInformation retrievedContainerInfo =
- fetchType(SerializedContainerInformation, appmaster,
- LIVE_CONTAINERS + "/${amContainerId}")
+ fetchJType(
+ LIVE_CONTAINERS + "/${amContainerId}",
+ SerializedContainerInformation
+ )
assert retrievedContainerInfo.containerId == amContainerId
// fetch missing
try {
- def result = fetchType(SerializedContainerInformation, appmaster,
- LIVE_CONTAINERS + "/unknown")
+ def result = fetchJType(
+ LIVE_CONTAINERS + "/unknown",
+ SerializedContainerInformation
+ )
fail("expected an error, got $result")
- } catch (NotFoundException e) {
+ } catch (UniformInterfaceException e) {
// expected
}
@@ -172,7 +211,7 @@ class JerseyTestDelegates extends SliderTestUtils {
describe "components"
Map<String, SerializedComponentInformation> components =
- fetchType(HashMap, appmaster, LIVE_COMPONENTS)
+ fetchJType(LIVE_COMPONENTS, HashMap)
// two components
assert components.size() == 1
log.info "${components}"
@@ -180,10 +219,10 @@ class JerseyTestDelegates extends SliderTestUtils {
SerializedComponentInformation amComponentInfo =
(SerializedComponentInformation) components[COMPONENT_AM]
- SerializedComponentInformation amFullInfo = fetchType(
- SerializedComponentInformation,
- appmaster,
- LIVE_COMPONENTS + "/${COMPONENT_AM}")
+ SerializedComponentInformation amFullInfo = fetchJType(
+ LIVE_COMPONENTS + "/${COMPONENT_AM}",
+ SerializedComponentInformation
+ )
assert amFullInfo.containers.size() == 1
assert amFullInfo.containers[0] == amContainerId
@@ -202,14 +241,14 @@ class JerseyTestDelegates extends SliderTestUtils {
MODEL,
ApplicationResource.MODEL_ENTRIES)
- def unresolvedConf = fetchType(AggregateConf, appmaster, MODEL_DESIRED)
+ def unresolvedConf = fetchJType(MODEL_DESIRED, AggregateConf)
// log.info "Unresolved \n$unresolvedConf"
def unresolvedAppConf = unresolvedConf.appConfOperations
def sam = "slider-appmaster"
assert unresolvedAppConf.getComponentOpt(sam,
TEST_GLOBAL_OPTION, "") == ""
- def resolvedConf = fetchType(AggregateConf, appmaster, MODEL_RESOLVED)
+ def resolvedConf = fetchJType(MODEL_RESOLVED, AggregateConf)
// log.info "Resolved \n$resolvedConf"
assert resolvedConf.appConfOperations.getComponentOpt(
sam, TEST_GLOBAL_OPTION, "") == TEST_GLOBAL_OPTION_PRESENT
@@ -240,7 +279,10 @@ class JerseyTestDelegates extends SliderTestUtils {
// GET
String ping = appendToURL(appmaster, SLIDER_PATH_APPLICATION, ACTION_PING)
describe "ping to AM URL $appmaster, ping URL $ping"
- def pinged = fetchType(PingResource, appmaster, ACTION_PING + "?body=hello")
+ def pinged = fetchJType(
+ ACTION_PING + "?body=hello",
+ PingResource
+ )
log.info "Ping GET: $pinged"
URL pingUrl = new URL(ping)
@@ -313,10 +355,9 @@ class JerseyTestDelegates extends SliderTestUtils {
[url: ping],
true,
"AM failed to shut down") {
- def pinged = fetchType(
- PingResource,
- appmaster,
- ACTION_PING + "?body=hello")
+ def pinged = fetchJType(ACTION_PING + "?body=hello",
+ PingResource
+ )
fail("AM didn't shut down; Ping GET= $pinged")
}
@@ -340,4 +381,13 @@ class JerseyTestDelegates extends SliderTestUtils {
}
}
+ public void suite() {
+
+ testRestletGetOperations();
+ testCodahaleOperations()
+ testLiveResources()
+ testLiveContainers();
+
+ testRESTModel()
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/72e47903/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
index fd676df..a53f546 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
@@ -59,36 +59,41 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
addToTeardown(client);
ApplicationReport report = waitForClusterLive(client)
- def realappmaster = report.originalTrackingUrl
+ def directAM = report.originalTrackingUrl
// set up url config to match
initHttpTestSupport(launcher.configuration)
execOperation(WEB_STARTUP_TIME) {
- GET(realappmaster)
+ GET(directAM)
}
execOperation(WEB_STARTUP_TIME) {
- def metrics = GET(realappmaster, SYSTEM_METRICS)
+ def metrics = GET(directAM, SYSTEM_METRICS)
log.info metrics
}
- def appmaster = report.trackingUrl
+ def proxyAM = report.trackingUrl
- GET(appmaster)
+ GET(proxyAM)
- log.info GET(appmaster, SYSTEM_PING)
- log.info GET(appmaster, SYSTEM_THREADS)
- log.info GET(appmaster, SYSTEM_HEALTHCHECK)
- log.info GET(appmaster, SYSTEM_METRICS_JSON)
+ log.info GET(proxyAM, SYSTEM_PING)
+ log.info GET(proxyAM, SYSTEM_THREADS)
+ log.info GET(proxyAM, SYSTEM_HEALTHCHECK)
+ log.info GET(proxyAM, SYSTEM_METRICS_JSON)
def wsBackDoorRequired = conf.getBoolean(
SliderXmlConfKeys.X_DEV_INSECURE_WS,
true)
+
+
+ JerseyTestDelegates proxyJerseyTests =
+ new JerseyTestDelegates(proxyAM, createJerseyClient())
+ proxyJerseyTests.suite()
- RestTestDelegates proxied = new RestTestDelegates(appmaster)
- RestTestDelegates direct = new RestTestDelegates(realappmaster)
+ RestTestDelegates proxied = new RestTestDelegates(proxyAM)
+ RestTestDelegates direct = new RestTestDelegates(directAM)
direct.testRestletGetOperations();
proxied.testCodahaleOperations()
@@ -96,7 +101,7 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
describe "base entry lists"
- assertPathServesList(appmaster, LIVE, ApplicationResource.LIVE_ENTRIES)
+ assertPathServesList(proxyAM, LIVE, ApplicationResource.LIVE_ENTRIES)
// now some REST gets
describe "Application REST ${LIVE_RESOURCES}"
@@ -114,7 +119,12 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
}
direct.logCodahaleMetrics();
+
+ // this MUST be the final test
direct.testStop();
+
+
+
}
[14/28] incubator-slider git commit: SLIDER-734 add check for command
line argument
Posted by el...@apache.org.
SLIDER-734 add check for command line argument
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b74b1f59
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b74b1f59
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b74b1f59
Branch: refs/heads/develop
Commit: b74b1f593c74fe122c05da37f06dc6ce64c9db79
Parents: 1282802
Author: tedyu <yu...@gmail.com>
Authored: Thu Jan 22 13:50:36 2015 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Thu Jan 22 13:50:36 2015 -0800
----------------------------------------------------------------------
app-packages/hbase/hbase-wrapper.py | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b74b1f59/app-packages/hbase/hbase-wrapper.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/hbase-wrapper.py b/app-packages/hbase/hbase-wrapper.py
index d3bb6ee..1fae0c2 100644
--- a/app-packages/hbase/hbase-wrapper.py
+++ b/app-packages/hbase/hbase-wrapper.py
@@ -104,6 +104,10 @@ def writePropertiesToConfigXMLFile(infile, outfile, propertyMap):
writeToFile(xmldoc.toxml(), outfile)
home = expanduser("~")
+if len(sys.argv) < 2:
+ print "the name of cluster instance is required"
+ sys.exit(1)
+
cluster_instance=sys.argv[1]
hbase_conf_dir="/etc/hbase/conf"
[25/28] incubator-slider git commit: SLIDER-767 rename
"apache-slider-command-logger" and "command-logger" modules
Posted by el...@apache.org.
SLIDER-767 rename "apache-slider-command-logger" and "command-logger" modules
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f5b0940f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f5b0940f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f5b0940f
Branch: refs/heads/develop
Commit: f5b0940f03c72281e8e51fab122b8118a1b810c3
Parents: 657f734
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jan 27 13:32:57 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jan 27 13:32:57 2015 +0000
----------------------------------------------------------------------
app-packages/command-logger/application-pkg/README.txt | 2 +-
app-packages/command-logger/application-pkg/pom.xml | 12 ++++++++----
.../application-pkg/src/packages/tarball/all.xml | 2 +-
app-packages/command-logger/slider-pkg/appConfig.json | 2 +-
app-packages/command-logger/slider-pkg/pom.xml | 12 ++++++------
.../test/app_packages/test_command_log/appConfig.json | 2 +-
.../test_command_log/appConfig_fast_no_reg.json | 2 +-
.../app_packages/test_command_log/appConfig_no_hb.json | 2 +-
slider-funtest/pom.xml | 8 ++++----
9 files changed, 24 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f5b0940f/app-packages/command-logger/application-pkg/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/command-logger/application-pkg/README.txt b/app-packages/command-logger/application-pkg/README.txt
index 5596f30..ea5b2e8 100644
--- a/app-packages/command-logger/application-pkg/README.txt
+++ b/app-packages/command-logger/application-pkg/README.txt
@@ -14,7 +14,7 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-Command logger is a simple application that does not have any moving parts yet.
+Command logger is a simple application that does not have any moving parts.
Its sole purpose is to get packaged as a tarball that includes some static files
suggesting how commands may be logged.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f5b0940f/app-packages/command-logger/application-pkg/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/command-logger/application-pkg/pom.xml b/app-packages/command-logger/application-pkg/pom.xml
index 7ea5a3b..3c38a62 100644
--- a/app-packages/command-logger/application-pkg/pom.xml
+++ b/app-packages/command-logger/application-pkg/pom.xml
@@ -23,12 +23,16 @@
<relativePath>../../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>command-logger</artifactId>
+ <artifactId>command-logger-sample</artifactId>
<packaging>pom</packaging>
- <name>Command Logger</name>
- <description>Command Logger</description>
+ <name>Command Logger Sample Application</name>
+ <description>
+ Command logger is a simple application that does not have any moving parts.
+ Its sole purpose is to get packaged as a tarball that includes some static files
+ suggesting how commands may be logged, for use in test deployments.
+ </description>
<properties>
- <app.package.name>command-logger</app.package.name>
+ <app.package.name>command-logger-sample</app.package.name>
</properties>
<build>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f5b0940f/app-packages/command-logger/application-pkg/src/packages/tarball/all.xml
----------------------------------------------------------------------
diff --git a/app-packages/command-logger/application-pkg/src/packages/tarball/all.xml b/app-packages/command-logger/application-pkg/src/packages/tarball/all.xml
index 4538d57..b1a0e93 100644
--- a/app-packages/command-logger/application-pkg/src/packages/tarball/all.xml
+++ b/app-packages/command-logger/application-pkg/src/packages/tarball/all.xml
@@ -18,7 +18,7 @@
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1 http://maven.apache.org/xsd/assembly-1.1.1.xsd">
- <!--This 'agent-all' id is not appended to the produced bundle because we do this:
+ <!--This 'command-logger' id is not appended to the produced bundle because we do this:
http://maven.apache.org/plugins/maven-assembly-plugin/faq.html#required-classifiers
-->
<id>command-logger</id>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f5b0940f/app-packages/command-logger/slider-pkg/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/command-logger/slider-pkg/appConfig.json b/app-packages/command-logger/slider-pkg/appConfig.json
index d4082a8..09e9252 100644
--- a/app-packages/command-logger/slider-pkg/appConfig.json
+++ b/app-packages/command-logger/slider-pkg/appConfig.json
@@ -3,7 +3,7 @@
"metadata": {
},
"global": {
- "application.def": ".slider/package/CMD_LOGGER/apache-slider-command-logger.zip",
+ "application.def": ".slider/package/CMD_LOGGER/command-logger-slider-package.zip",
"java_home": "/usr/jdk64/jdk1.7.0_67",
"site.global.application_id": "CommandLogger",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/command-logger",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f5b0940f/app-packages/command-logger/slider-pkg/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/command-logger/slider-pkg/pom.xml b/app-packages/command-logger/slider-pkg/pom.xml
index bb2aaac..3e8e1e4 100644
--- a/app-packages/command-logger/slider-pkg/pom.xml
+++ b/app-packages/command-logger/slider-pkg/pom.xml
@@ -24,13 +24,13 @@
<relativePath>../../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>apache-slider-command-logger</artifactId>
+ <artifactId>command-logger-slider-package</artifactId>
<packaging>pom</packaging>
- <name>Slider Command Logger App Package</name>
+ <name>Slider Command Logger Slider App Package</name>
<description>Slider Command Logger App Package</description>
<properties>
<work.dir>package-tmp</work.dir>
- <app.package.name>apache-slider-command-logger</app.package.name>
+ <app.package.name>command-logger-slider-package</app.package.name>
</properties>
<build>
@@ -67,7 +67,7 @@
<goal>copy-dependencies</goal>
</goals>
<configuration>
- <includeArtifactIds>command-logger</includeArtifactIds>
+ <includeArtifactIds>command-logger-sample</includeArtifactIds>
<includeTypes>tar</includeTypes>
<outputDirectory>${project.build.directory}/${work.dir}</outputDirectory>
</configuration>
@@ -82,7 +82,7 @@
<artifactItems>
<artifactItem>
<groupId>org.apache.slider</groupId>
- <artifactId>command-logger</artifactId>
+ <artifactId>command-logger-sample</artifactId>
<type>tar</type>
<overWrite>false</overWrite>
<outputDirectory>${project.build.directory}/${work.dir}</outputDirectory>
@@ -100,7 +100,7 @@
<dependencies>
<dependency>
<groupId>org.apache.slider</groupId>
- <artifactId>command-logger</artifactId>
+ <artifactId>command-logger-sample</artifactId>
<version>${project.version}</version>
<type>tar</type>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f5b0940f/slider-core/src/test/app_packages/test_command_log/appConfig.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_command_log/appConfig.json b/slider-core/src/test/app_packages/test_command_log/appConfig.json
index a8d3d4b..b9383ec 100644
--- a/slider-core/src/test/app_packages/test_command_log/appConfig.json
+++ b/slider-core/src/test/app_packages/test_command_log/appConfig.json
@@ -3,7 +3,7 @@
"metadata": {
},
"global": {
- "application.def": ".slider/package/CMD_LOGGER/apache-slider-command-logger.zip",
+ "application.def": ".slider/package/CMD_LOGGER/command-logger-slider-package.zip",
"java_home": "/usr/jdk64/jdk1.7.0_67",
"site.global.application_id": "CommandLogger",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/command-logger",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f5b0940f/slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json b/slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json
index ac11c08..fa7312a 100644
--- a/slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json
+++ b/slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json
@@ -5,7 +5,7 @@
"global": {
"heartbeat.monitor.interval": "20000",
"agent.instance.debug.data": "ANY:DO_NOT_REGISTER:NONE",
- "application.def": ".slider/package/CMD_LOGGER/apache-slider-command-logger.zip",
+ "application.def": ".slider/package/CMD_LOGGER/command-logger-slider-package.zip",
"java_home": "/usr/jdk64/jdk1.7.0_67",
"site.global.application_id": "CommandLogger",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/command-logger",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f5b0940f/slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json b/slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json
index 9d70879..eae57e7 100644
--- a/slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json
+++ b/slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json
@@ -5,7 +5,7 @@
"global": {
"heartbeat.monitor.interval": "20000",
"agent.instance.debug.data": "ANY:DO_NOT_HEARTBEAT:DO_NOT_HEARTBEAT:NONE",
- "application.def": ".slider/package/CMD_LOGGER/apache-slider-command-logger.zip",
+ "application.def": ".slider/package/CMD_LOGGER/command-logger-slider-package.zip",
"java_home": "/usr/jdk64/jdk1.7.0_67",
"site.global.application_id": "CommandLogger",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/command-logger",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f5b0940f/slider-funtest/pom.xml
----------------------------------------------------------------------
diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml
index 53d803d..2231274 100644
--- a/slider-funtest/pom.xml
+++ b/slider-funtest/pom.xml
@@ -119,7 +119,7 @@
<slider.conf.dir>${slider.conf.dir}</slider.conf.dir>
<slider.bin.dir>../slider-assembly/target/slider-${project.version}-all/slider-${project.version}</slider.bin.dir>
<test.app.pkg.dir>../app-packages/command-logger/slider-pkg/target</test.app.pkg.dir>
- <test.app.pkg.file>apache-slider-command-logger.zip</test.app.pkg.file>
+ <test.app.pkg.file>command-logger-slider-package.zip</test.app.pkg.file>
<test.app.pkg.name>CMD_LOGGER</test.app.pkg.name>
<test.app.resource>../slider-core/src/test/app_packages/test_command_log/resources.json</test.app.resource>
<test.app.template>../slider-core/src/test/app_packages/test_command_log/appConfig.json</test.app.template>
@@ -155,11 +155,11 @@
<artifactItems>
<artifactItem>
<groupId>org.apache.slider</groupId>
- <artifactId>apache-slider-command-logger</artifactId>
+ <artifactId>command-logger-slider-package</artifactId>
<type>zip</type>
<overWrite>false</overWrite>
<outputDirectory>${project.build.directory}/${work.dir}</outputDirectory>
- <destFileName>apache-slider-command-logger.zip</destFileName>
+ <destFileName>command-logger-slider-package.zip</destFileName>
</artifactItem>
</artifactItems>
</configuration>
@@ -221,7 +221,7 @@
<dependency>
<groupId>org.apache.slider</groupId>
- <artifactId>apache-slider-command-logger</artifactId>
+ <artifactId>command-logger-slider-package</artifactId>
<version>${project.version}</version>
<type>zip</type>
</dependency>
[21/28] incubator-slider git commit: Merge branch
'feature/SLIDER-762_Add_API-level_slider_REST_client' into develop
Posted by el...@apache.org.
Merge branch 'feature/SLIDER-762_Add_API-level_slider_REST_client' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6bd71659
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6bd71659
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6bd71659
Branch: refs/heads/develop
Commit: 6bd71659e932fb7e6a29777599a650df26214096
Parents: ea6eca1 beda1b2
Author: Steve Loughran <st...@apache.org>
Authored: Fri Jan 23 19:43:08 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Jan 23 19:43:08 2015 +0000
----------------------------------------------------------------------
.../slider/client/rest/BaseRestClient.java | 154 +++++++++++
.../slider/client/rest/RestClientFactory.java | 61 +++++
.../client/rest/RestClientRegistryBinder.java | 201 ++++++++++++++
.../client/rest/SliderApplicationAPI.java | 274 +++++++++++++++++++
.../core/exceptions/ExceptionConverter.java | 20 +-
.../registry/info/CustomRegistryConstants.java | 5 +-
.../registry/retrieve/RegistryRetriever.java | 67 +----
.../core/restclient/UgiJerseyBinding.java | 32 ++-
.../restclient/UrlConnectionOperations.java | 94 +++++--
.../slideram/SliderAMProviderService.java | 5 +
.../server/appmaster/state/RoleStatus.java | 5 +
.../rest/application/ApplicationResource.java | 2 +-
.../agent/rest/JerseyTestDelegates.groovy | 24 +-
.../rest/SliderRestClientTestDelegates.groovy | 251 +++++++++++++++++
.../slider/agent/rest/TestStandaloneREST.groovy | 37 ++-
.../funtest/lifecycle/AgentWebPagesIT.groovy | 46 +++-
16 files changed, 1183 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
[13/28] incubator-slider git commit: SLIDER-762 API-level REST client
implemented as far as API goes today
Posted by el...@apache.org.
SLIDER-762 API-level REST client implemented as far as API goes today
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b7d5099d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b7d5099d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b7d5099d
Branch: refs/heads/develop
Commit: b7d5099d3ab2ddfb54a97ec0b70dc7c4ac7e8ed9
Parents: 5b5dac8
Author: Steve Loughran <st...@apache.org>
Authored: Thu Jan 22 19:11:49 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Jan 22 19:11:49 2015 +0000
----------------------------------------------------------------------
.../apache/slider/client/SliderRestClient.java | 13 +-
.../core/restclient/UgiJerseyBinding.java | 3 +
.../rest/SliderRestClientTestDelegates.groovy | 289 ++-----------------
.../slider/agent/rest/TestStandaloneREST.groovy | 14 +
4 files changed, 61 insertions(+), 258 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b7d5099d/slider-core/src/main/java/org/apache/slider/client/SliderRestClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderRestClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderRestClient.java
index 14b85e9..9de2551 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderRestClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderRestClient.java
@@ -24,6 +24,7 @@ import com.sun.jersey.api.client.GenericType;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.representation.Form;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.service.AbstractService;
import org.apache.slider.api.types.SerializedComponentInformation;
import org.apache.slider.api.types.SerializedContainerInformation;
@@ -55,6 +56,9 @@ public class SliderRestClient extends AbstractService {
super(name);
Preconditions.checkNotNull(jersey, "null jersey");
this.jersey = jersey;
+ if (appmaster !=null) {
+ bindToAppmaster(appmaster);
+ }
}
public SliderRestClient(Client jersey, WebResource appmaster) {
@@ -65,6 +69,10 @@ public class SliderRestClient extends AbstractService {
return jersey;
}
+ /**
+ * Bind/rebind to the AM
+ * @param appmaster AM
+ */
public void bindToAppmaster(WebResource appmaster) {
this.appmaster = appmaster;
this.appResource = appmaster.path(SLIDER_PATH_APPLICATION);
@@ -80,6 +88,9 @@ public class SliderRestClient extends AbstractService {
* @return an resource under the application path
*/
public WebResource applicationResource(String subpath) {
+ Preconditions.checkArgument(!StringUtils.isEmpty(subpath),
+ "empty path");
+ Preconditions.checkNotNull(appResource, "Null app resource");
return appResource.path(subpath);
}
@@ -273,7 +284,7 @@ public class SliderRestClient extends AbstractService {
*/
public Map<String, SerializedContainerInformation> enumContainers() throws
IOException {
- return getApplicationResource(LIVE_RESOURCES,
+ return getApplicationResource(LIVE_CONTAINERS,
new GenericType<Map<String, SerializedContainerInformation>>() {
});
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b7d5099d/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
index f2991d2..76407c6 100644
--- a/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
@@ -113,6 +113,9 @@ public class UgiJerseyBinding implements
* @param verb HTTP Verb used
* @param url URL as string
* @param ex exception
+ * @throws PathNotFoundException for an unknown resource
+ * @throws PathAccessDeniedException for access denied
+ * @throws PathIOException for anything else
*/
public static IOException uprateFaults(HttpVerb verb, String url,
UniformInterfaceException ex)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b7d5099d/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy
index a97d136..46dba1d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/SliderRestClientTestDelegates.groovy
@@ -18,24 +18,18 @@
package org.apache.slider.agent.rest
-import com.google.common.base.Preconditions
import com.sun.jersey.api.client.Client
-import com.sun.jersey.api.client.ClientResponse
-import com.sun.jersey.api.client.UniformInterfaceException
import com.sun.jersey.api.client.WebResource
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.webapp.NotFoundException
+import org.apache.hadoop.fs.PathNotFoundException
import org.apache.slider.api.StateValues
import org.apache.slider.api.types.SerializedComponentInformation
import org.apache.slider.api.types.SerializedContainerInformation
-import org.apache.slider.common.tools.SliderUtils
-import org.apache.slider.core.conf.AggregateConf
+import org.apache.slider.client.SliderRestClient
import org.apache.slider.core.conf.ConfTree
import org.apache.slider.core.conf.ConfTreeOperations
-import org.apache.slider.core.restclient.HttpVerb
import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource
-import org.apache.slider.server.appmaster.web.rest.application.resources.PingResource
import org.apache.slider.test.SliderTestUtils
import javax.ws.rs.core.MediaType
@@ -61,164 +55,36 @@ class SliderRestClientTestDelegates extends SliderTestUtils {
final String appmaster;
final String application;
final Client jersey;
- final WebResource amResource
- final WebResource appResource
+ final SliderRestClient restClient;
SliderRestClientTestDelegates(String appmaster, Client jersey) {
this.jersey = jersey
this.appmaster = appmaster
application = appendToURL(appmaster, SLIDER_PATH_APPLICATION)
- amResource = jersey.resource(appmaster)
+ WebResource amResource = jersey.resource(appmaster)
amResource.type(MediaType.APPLICATION_JSON)
- appResource = amResource.path(SLIDER_PATH_APPLICATION)
+ restClient = new SliderRestClient(jersey, amResource)
}
- /**
- * <T> T get(Class<T> c)
- * Get operation against a path under the Application
- * @param subpath path
- * @return
- */
- public <T> T jGetApplicationResource(String subpath, Class<T> c) {
- return (T)jExec(HttpVerb.GET, subpath, c);
- }
-
- /**
- * <T> T get(Class<T> c)
- * Get operation against a path under the Application
- * @param subpath path
- * @return
- */
- public <T> T jExec(HttpVerb method, String subpath, Class<T> c) {
- WebResource resource = applicationResource(subpath);
- jExec(method, resource, c);
- }
-
- public <T> T jExec(HttpVerb method, WebResource resource, Class<T> c) {
- try {
- Preconditions.checkArgument(c != null);
- resource.accept(MediaType.APPLICATION_JSON_TYPE);
- (T) resource.method(method.verb, c);
- } catch (UniformInterfaceException ex) {
- uprateFaults(method, resource, ex);
- }
- }
- /**
- * Create a resource under the application path
- * @param subpath
- * @return
- */
- public WebResource applicationResource(String subpath) {
- return appResource.path(subpath)
+ public void testGetDesiredModel() throws Throwable {
+ restClient.getDesiredModel()
+ restClient.getDesiredAppconf()
+ restClient.getDesiredYarnResources()
}
- /**
- * Convert faults to exceptions; pass through 200 responses
- * @param method
- * @param webResource
- * @param ex
- * @return
- */
- public uprateFaults(
- HttpVerb method,
- WebResource webResource,
- UniformInterfaceException ex) {
- uprateFaults(method.verb,
- webResource.URI.toString(),
- ex.response.status,
- ex.response.toString())
+ public void testGetResolvedModel() throws Throwable {
+ restClient.getResolvedModel()
+ restClient.getResolvedAppconf()
+ restClient.getResolvedYarnResources()
}
- /**
- * <T> T get(Class<T> c)
- * Get operation against a path under the AM
- * @param path path
- * @return
- */
- public <T> T jGetAMResource(String path, Class<T> c) {
- assert c
- WebResource webResource = buildResource(path)
- (T)webResource.get(c)
- }
-
- /**
- * Get operation against a path under the AM
- * @param path path
- * @return the string value
- */
- public String jerseyGet(String path) {
- return jGetAMResource(path, String.class)
- }
-
- /**
- * Build a resource against a path under the AM API
- * @param path path
- * @return a resource for use
- */
- public WebResource buildResource(String path) {
- assert path
- String fullpath = SliderUtils.appendToURL(appmaster, path);
- WebResource webResource = jersey.resource(fullpath);
- webResource.type(MediaType.APPLICATION_JSON);
- log.info("HTTP operation against $fullpath");
- return webResource;
- }
-
- public void testJerseyGetConftree() throws Throwable {
- jGetApplicationResource(LIVE_RESOURCES, ConfTree.class);
- }
- public void testCodahaleOperations() throws Throwable {
- describe "Codahale operations"
-
- jerseyGet("/")
- jerseyGet(SYSTEM_THREADS)
- jerseyGet(SYSTEM_HEALTHCHECK)
- jerseyGet(SYSTEM_PING)
- jerseyGet(SYSTEM_METRICS_JSON)
- }
-
- public void logCodahaleMetrics() {
- // query Coda Hale metrics
- log.info jerseyGet(SYSTEM_HEALTHCHECK)
- log.info jerseyGet(SYSTEM_METRICS)
- }
-
- /**
- * Fetch a typed entry <i>under the application path</i>
- * @param subpath
- * @param clazz
- * @return
- */
- public <T> T jFetchType(
- String subpath, Class<T> clazz) {
- (T)jGetApplicationResource(subpath, clazz)
- }
-
- public ConfTreeOperations jGetConfigTree(
- String path) {
- ConfTree ctree = jGetApplicationResource(path, ConfTree)
- ConfTreeOperations tree = new ConfTreeOperations(ctree)
- return tree
- }
-
-
- public void testMimeTypes() throws Throwable {
- describe "Mime Types"
-
- WebResource resource = applicationResource(LIVE_RESOURCES)
- def response = resource.get(ClientResponse)
- response.headers.each {key, val -> log.info("$key: $val")}
- log.info response.toString()
- assert response.type.equals(MediaType.APPLICATION_JSON_TYPE)
- }
-
public void testLiveResources() throws Throwable {
describe "Live Resources"
- ConfTreeOperations tree = jGetConfigTree(LIVE_RESOURCES)
+ ConfTreeOperations tree = restClient.getLiveYarnResources()
log.info tree.toString()
def liveAM = tree.getComponent(COMPONENT_AM)
@@ -236,8 +102,7 @@ class SliderRestClientTestDelegates extends SliderTestUtils {
public void testLiveContainers() throws Throwable {
describe "Application REST ${LIVE_CONTAINERS}"
- Map<String, SerializedContainerInformation> containers =
- jGetApplicationResource(LIVE_CONTAINERS, HashMap)
+ Map<String, SerializedContainerInformation> containers = restClient.enumContainers()
assert containers.size() == 1
log.info "${containers}"
SerializedContainerInformation amContainerInfo =
@@ -256,21 +121,15 @@ class SliderRestClientTestDelegates extends SliderTestUtils {
describe "containers"
- SerializedContainerInformation retrievedContainerInfo =
- jFetchType(
- LIVE_CONTAINERS + "/${amContainerId}",
- SerializedContainerInformation
- )
- assert retrievedContainerInfo.containerId == amContainerId
+ SerializedContainerInformation amContainerInfo2 =
+ restClient.getContainer(amContainerId)
+ assert amContainerInfo2.containerId == amContainerId
// fetch missing
try {
- def result = jFetchType(
- LIVE_CONTAINERS + "/unknown",
- SerializedContainerInformation
- )
+ def result = restClient.getContainer("/unknown")
fail("expected an error, got $result")
- } catch (NotFoundException e) {
+ } catch (PathNotFoundException e) {
// expected
}
@@ -278,7 +137,8 @@ class SliderRestClientTestDelegates extends SliderTestUtils {
describe "components"
Map<String, SerializedComponentInformation> components =
- jFetchType(LIVE_COMPONENTS, HashMap)
+ restClient.enumComponents()
+
// two components
assert components.size() >= 1
log.info "${components}"
@@ -286,49 +146,14 @@ class SliderRestClientTestDelegates extends SliderTestUtils {
SerializedComponentInformation amComponentInfo =
(SerializedComponentInformation) components[COMPONENT_AM]
- SerializedComponentInformation amFullInfo = jFetchType(
- LIVE_COMPONENTS + "/${COMPONENT_AM}",
- SerializedComponentInformation
- )
+ SerializedComponentInformation amFullInfo = restClient.getComponent(COMPONENT_AM)
assert amFullInfo.containers.size() == 1
assert amFullInfo.containers[0] == amContainerId
}
- /**
- * Assert that a path resolves to an array list that contains
- * those entries (and only those entries) expected
- * @param appmaster AM ref
- * @param path path under AM
- * @param entries entries to assert the presence of
- */
- public void assertPathServesList(
- String appmaster,
- String path,
- List<String> entries) {
- def list = jFetchType(path, ArrayList)
- assert list.size() == entries.size()
- assert entries.containsAll(list)
- }
-
- /**
- * Fetch a list of URLs, all of which must be of the same type
- * @param clazz class of resolved values
- * @param appmaster URL to app master
- * @param subpaths list of subpaths
- * @return a map of paths to values
- */
- public <T> Map<String, T> fetchTypeList(
- Class<T> clazz, String appmaster, List<String> subpaths
- ) {
- Map<String, T> results = [:]
- subpaths.each { String it ->
- results[it] = (jFetchType(it, clazz))
- }
- return results;
- }
-
+
/**
* Test the rest model. For this to work the cluster has to be configured
* with the global option
@@ -341,15 +166,14 @@ class SliderRestClientTestDelegates extends SliderTestUtils {
MODEL,
ApplicationResource.MODEL_ENTRIES)
- def unresolvedConf = jFetchType(MODEL_DESIRED, AggregateConf)
+ def unresolvedConf = restClient.getDesiredModel()
// log.info "Unresolved \n$unresolvedConf"
def unresolvedAppConf = unresolvedConf.appConfOperations
def sam = "slider-appmaster"
assert unresolvedAppConf.getComponentOpt(sam,
TEST_GLOBAL_OPTION, "") == ""
- def resolvedConf = jFetchType(MODEL_RESOLVED, AggregateConf)
-// log.info "Resolved \n$resolvedConf"
+ def resolvedConf = restClient.getResolvedModel()
assert resolvedConf.appConfOperations.getComponentOpt(
sam, TEST_GLOBAL_OPTION, "") == TEST_GLOBAL_OPTION_PRESENT
@@ -359,9 +183,9 @@ class SliderRestClientTestDelegates extends SliderTestUtils {
[TEST_GLOBAL_OPTION] == null
- def resolved = fetchTypeList(ConfTree, appmaster,
- [MODEL_RESOLVED_APPCONF, MODEL_RESOLVED_RESOURCES])
- assert resolved[MODEL_RESOLVED_APPCONF].
+
+ def resolvedAppconf = restClient.getResolvedAppconf()
+ assert resolvedAppconf.
components[sam][TEST_GLOBAL_OPTION] == TEST_GLOBAL_OPTION_PRESENT
}
@@ -369,38 +193,9 @@ class SliderRestClientTestDelegates extends SliderTestUtils {
// GET
describe "pinging"
- def pinged = jExec(HttpVerb.GET, ACTION_PING, PingResource)
- log.info "Ping GET: $pinged"
- // HEAD
-// jExec(HttpVerb.HEAD, ACTION_PING, PingResource)
- jExec(HttpVerb.PUT, ACTION_PING, PingResource)
- jExec(HttpVerb.DELETE, ACTION_PING, PingResource)
- jExec(HttpVerb.POST, ACTION_PING, PingResource)
- ping(HttpVerb.PUT, ACTION_PING, "ping-text")
- ping(HttpVerb.POST, ACTION_PING, "ping-text")
- ping(HttpVerb.DELETE, ACTION_PING, "ping-text")
+ restClient.ping("hello")
}
- /**
- * Execute a ping; assert that a response came back with the relevant
- * verb if the verb has a response body
- * @param method method to invoke
- * @param subpath ping path
- * @param payload payload
- * @return the resource if the verb has a response
- */
- private PingResource ping(HttpVerb method, String subpath, Object payload) {
- def actionPing = applicationResource(ACTION_PING)
- def upload = method.hasUploadBody() ? payload : null
- if (method.hasResponseBody()) {
- def pinged = actionPing.method(method.verb, PingResource, upload)
- assert method.verb == pinged.verb
- return pinged
- } else {
- actionPing.method(method.verb, upload)
- return null
- }
- }
/**
* Test the stop command.
@@ -441,30 +236,10 @@ class SliderRestClientTestDelegates extends SliderTestUtils {
}
*/
- /**
- * Probe that spins until the url specified by "url") refuses
- * connections
- * @param args argument map
- * @return the outcome
- */
-/*
- Outcome probePingFailing(Map args) {
- String ping = args["url"]
- URL pingUrl = new URL(ping)
- try {
- def response = pingAction(HttpVerb.HEAD, pingUrl, "should not be running")
- return Outcome.Retry
- } catch (IOException e) {
- // expected
- return Outcome.Success
- }
- }
-*/
-
public void testSuiteGetOperations() {
- testCodahaleOperations()
- testMimeTypes()
+ testGetDesiredModel()
+ testGetResolvedModel()
testLiveResources()
testLiveContainers();
testRESTModel()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b7d5099d/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
index 0abf264..8ee88c3 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
@@ -110,6 +110,20 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
+
+ describe "Proxy SliderRestClient Tests"
+ SliderRestClientTestDelegates proxySliderRestClient =
+ new SliderRestClientTestDelegates(proxyAM, createUGIJerseyClient())
+ proxySliderRestClient.testSuiteGetOperations()
+
+ describe "Direct SliderRestClient Tests"
+ SliderRestClientTestDelegates directSliderRestClient =
+ new SliderRestClientTestDelegates(directAM, createUGIJerseyClient())
+ directSliderRestClient.testSuiteGetOperations()
+ directSliderRestClient.testSuiteComplexVerbs()
+
+
+
describe "Proxy Jersey Tests"
JerseyTestDelegates proxyJerseyTests =
new JerseyTestDelegates(proxyAM, createUGIJerseyClient())
[05/28] incubator-slider git commit: SLIDER-719 factoring out jersey
tests into own class
Posted by el...@apache.org.
SLIDER-719 factoring out jersey tests into own class
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c0a39d92
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c0a39d92
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c0a39d92
Branch: refs/heads/develop
Commit: c0a39d926b231167199655290bcbb43972cb86b2
Parents: c97f760
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jan 20 10:32:24 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jan 20 14:50:24 2015 +0000
----------------------------------------------------------------------
.../agent/rest/JerseyTestDelegates.groovy | 343 +++++++++++++++++++
.../apache/slider/test/SliderTestUtils.groovy | 5 +-
2 files changed, 345 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0a39d92/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
new file mode 100644
index 0000000..2f20bf5
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.rest
+
+import com.sun.jersey.api.client.Client
+import com.sun.jersey.api.client.WebResource
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.webapp.NotFoundException
+import org.apache.slider.api.StateValues
+import org.apache.slider.api.types.SerializedComponentInformation
+import org.apache.slider.api.types.SerializedContainerInformation
+import org.apache.slider.core.conf.AggregateConf
+import org.apache.slider.core.conf.ConfTree
+import org.apache.slider.core.conf.ConfTreeOperations
+import org.apache.slider.core.restclient.HttpOperationResponse
+import org.apache.slider.core.restclient.HttpVerb
+import org.apache.slider.core.restclient.UrlConnectionOperations
+import org.apache.slider.server.appmaster.web.rest.RestPaths
+import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource
+import org.apache.slider.server.appmaster.web.rest.application.resources.PingResource
+import org.apache.slider.test.Outcome
+import org.apache.slider.test.SliderTestUtils
+
+import javax.ws.rs.core.MediaType
+
+import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES
+import static org.apache.slider.api.StatusKeys.*
+import static org.apache.slider.common.SliderKeys.COMPONENT_AM
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.*
+
+/**
+ * This class contains parts of tests that can be run
+ * against a deployed AM: local or remote.
+ * It uses Jersey ... and must be passed a client that is either secure
+ * or not
+ *
+ */
+@CompileStatic
+@Slf4j
+class JerseyTestDelegates extends SliderTestUtils {
+ public static final String TEST_GLOBAL_OPTION = "test.global.option"
+ public static final String TEST_GLOBAL_OPTION_PRESENT = "present"
+
+ final String appmaster;
+ final String application;
+ final Client jersey;
+
+
+ JerseyTestDelegates(String appmaster, Client jersey) {
+ this.appmaster = appmaster
+ application = appendToURL(appmaster, RestPaths.SLIDER_PATH_APPLICATION)
+ this.jersey = jersey
+ }
+
+ /**
+ * <T> T get(Class<T> c)
+ * Get operation against the EM
+ * @param path path
+ * @return
+ */
+ public <T> T jerseyGet(String path, Class<T> c) {
+ assert c
+ WebResource webResource = buildResource(path)
+ webResource.get(c)
+ }
+
+ /**
+ * Build a resource against a path under the AM API
+ * @param path path
+ * @return a resource for use
+ */
+ public WebResource buildResource(String path) {
+ assert path
+ String fullpath = appendToURL(application, path)
+ WebResource webResource = jersey.resource(fullpath)
+ webResource.type(MediaType.APPLICATION_JSON)
+ log.info("HTTP operation against $fullpath");
+ return webResource
+ }
+
+ public void testJerseyGetConftree() throws Throwable {
+ jerseyGet(LIVE_RESOURCES, ConfTree.class);
+ }
+ public void testCodahaleOperations() throws Throwable {
+ describe "Codahale operations"
+
+ getWebPage(appmaster)
+ getWebPage(appmaster, SYSTEM_THREADS)
+ getWebPage(appmaster, SYSTEM_HEALTHCHECK)
+ getWebPage(appmaster, SYSTEM_PING)
+ getWebPage(appmaster, SYSTEM_METRICS_JSON)
+ }
+
+ public void logCodahaleMetrics() {
+ // query Coda Hale metrics
+ log.info getWebPage(appmaster, SYSTEM_HEALTHCHECK)
+ log.info getWebPage(appmaster, SYSTEM_METRICS)
+ }
+
+ public void testLiveResources() throws Throwable {
+ describe "Live Resources"
+ ConfTreeOperations tree = fetchConfigTree(appmaster, LIVE_RESOURCES)
+
+ log.info tree.toString()
+ def liveAM = tree.getComponent(COMPONENT_AM)
+ def desiredInstances = liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES);
+ assert desiredInstances ==
+ liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_ACTUAL)
+
+ assert 1 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_STARTED)
+ assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_REQUESTING)
+ assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_FAILED)
+ assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_COMPLETED)
+ assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_RELEASING)
+ }
+
+ public void testLiveContainers() throws Throwable {
+ describe "Application REST ${LIVE_CONTAINERS}"
+
+ Map<String, SerializedContainerInformation> containers =
+ fetchType(HashMap, appmaster, LIVE_CONTAINERS)
+ assert containers.size() == 1
+ log.info "${containers}"
+ SerializedContainerInformation amContainerInfo =
+ (SerializedContainerInformation) containers.values()[0]
+ assert amContainerInfo.containerId
+
+ def amContainerId = amContainerInfo.containerId
+ assert containers[amContainerId]
+
+ assert amContainerInfo.component == COMPONENT_AM
+ assert amContainerInfo.createTime > 0
+ assert amContainerInfo.exitCode == null
+ assert amContainerInfo.output == null
+ assert amContainerInfo.released == null
+ assert amContainerInfo.state == StateValues.STATE_LIVE
+
+ describe "containers"
+
+ SerializedContainerInformation retrievedContainerInfo =
+ fetchType(SerializedContainerInformation, appmaster,
+ LIVE_CONTAINERS + "/${amContainerId}")
+ assert retrievedContainerInfo.containerId == amContainerId
+
+ // fetch missing
+ try {
+ def result = fetchType(SerializedContainerInformation, appmaster,
+ LIVE_CONTAINERS + "/unknown")
+ fail("expected an error, got $result")
+ } catch (NotFoundException e) {
+ // expected
+ }
+
+
+ describe "components"
+
+ Map<String, SerializedComponentInformation> components =
+ fetchType(HashMap, appmaster, LIVE_COMPONENTS)
+ // two components
+ assert components.size() == 1
+ log.info "${components}"
+
+ SerializedComponentInformation amComponentInfo =
+ (SerializedComponentInformation) components[COMPONENT_AM]
+
+ SerializedComponentInformation amFullInfo = fetchType(
+ SerializedComponentInformation,
+ appmaster,
+ LIVE_COMPONENTS + "/${COMPONENT_AM}")
+
+ assert amFullInfo.containers.size() == 1
+ assert amFullInfo.containers[0] == amContainerId
+
+ }
+
+ /**
+ * Test the rest model. For this to work the cluster has to be configured
+ * with the global option
+ * @param appmaster
+ */
+ public void testRESTModel() {
+ describe "model"
+
+ assertPathServesList(appmaster,
+ MODEL,
+ ApplicationResource.MODEL_ENTRIES)
+
+ def unresolvedConf = fetchType(AggregateConf, appmaster, MODEL_DESIRED)
+// log.info "Unresolved \n$unresolvedConf"
+ def unresolvedAppConf = unresolvedConf.appConfOperations
+
+ def sam = "slider-appmaster"
+ assert unresolvedAppConf.getComponentOpt(sam,
+ TEST_GLOBAL_OPTION, "") == ""
+ def resolvedConf = fetchType(AggregateConf, appmaster, MODEL_RESOLVED)
+// log.info "Resolved \n$resolvedConf"
+ assert resolvedConf.appConfOperations.getComponentOpt(
+ sam, TEST_GLOBAL_OPTION, "") == TEST_GLOBAL_OPTION_PRESENT
+
+ def unresolved = fetchTypeList(ConfTree, appmaster,
+ [MODEL_DESIRED_APPCONF, MODEL_DESIRED_RESOURCES])
+ assert unresolved[MODEL_DESIRED_APPCONF].components[sam]
+ [TEST_GLOBAL_OPTION] == null
+
+
+ def resolved = fetchTypeList(ConfTree, appmaster,
+ [MODEL_RESOLVED_APPCONF, MODEL_RESOLVED_RESOURCES])
+ assert resolved[MODEL_RESOLVED_APPCONF].components[sam]
+ [TEST_GLOBAL_OPTION] ==
+ TEST_GLOBAL_OPTION_PRESENT
+ }
+
+
+ public void testRestletGetOperations() throws Throwable {
+ Client client = createJerseyClient()
+ String path = appendToURL(application, LIVE_RESOURCES)
+ WebResource webResource = client.resource(path)
+ webResource.type(MediaType.APPLICATION_JSON)
+ .get(ConfTree.class);
+ }
+
+ public void testPing() {
+ // GET
+ String ping = appendToURL(appmaster, SLIDER_PATH_APPLICATION, ACTION_PING)
+ describe "ping to AM URL $appmaster, ping URL $ping"
+ def pinged = fetchType(PingResource, appmaster, ACTION_PING + "?body=hello")
+ log.info "Ping GET: $pinged"
+
+ URL pingUrl = new URL(ping)
+ def message = "hello"
+
+ // HEAD
+ pingAction(HttpVerb.HEAD, pingUrl, message)
+
+ // Other verbs
+ pingAction(HttpVerb.POST, pingUrl, message)
+ pingAction(HttpVerb.PUT, pingUrl, message)
+ pingAction(HttpVerb.DELETE, pingUrl, message)
+
+ }
+
+
+ private HttpOperationResponse pingAction(
+ HttpVerb verb,
+ URL pingUrl,
+ String payload) {
+ return pingAction(connectionOperations, verb, pingUrl, payload)
+ }
+
+ private HttpOperationResponse pingAction(
+ UrlConnectionOperations ops, HttpVerb verb, URL pingUrl, String payload) {
+ def pinged
+ def outcome = ops.execHttpOperation(
+ verb,
+ pingUrl,
+ payload.bytes,
+ MediaType.TEXT_PLAIN)
+ byte[] bytes = outcome.data
+ if (verb.hasResponseBody()) {
+ assert bytes.length > 0, "0 bytes from ping $verb.verb"
+ pinged = deser(PingResource, bytes)
+ log.info "Ping $verb.verb: $pinged"
+ assert verb.verb == pinged.verb
+ } else {
+ assert bytes.length ==
+ 0, "${bytes.length} bytes of data from ping $verb.verb"
+ }
+ return outcome
+ }
+
+ /**
+ * Test the stop command.
+ * Important: once executed, the AM is no longer there.
+ * This must be the last test in the sequence.
+ */
+ public void testStop() {
+ String target = appendToURL(appmaster, SLIDER_PATH_APPLICATION, ACTION_STOP)
+ describe "Stop URL $target"
+ URL targetUrl = new URL(target)
+ def outcome = connectionOperations.execHttpOperation(
+ HttpVerb.POST,
+ targetUrl,
+ new byte[0],
+ MediaType.TEXT_PLAIN)
+ log.info "Stopped: $outcome"
+
+ // await the shutdown
+ sleep(1000)
+
+ // now a ping is expected to fail
+ String ping = appendToURL(appmaster, SLIDER_PATH_APPLICATION, ACTION_PING)
+ URL pingUrl = new URL(ping)
+
+ repeatUntilSuccess("probe for missing registry entry",
+ this.&probePingFailing, 30000, 500,
+ [url: ping],
+ true,
+ "AM failed to shut down") {
+ def pinged = fetchType(
+ PingResource,
+ appmaster,
+ ACTION_PING + "?body=hello")
+ fail("AM didn't shut down; Ping GET= $pinged")
+ }
+
+ }
+
+ /**
+ * Probe that spins until the url specified by "url") refuses
+ * connections
+ * @param args argument map
+ * @return the outcome
+ */
+ Outcome probePingFailing(Map args) {
+ String ping = args["url"]
+ URL pingUrl = new URL(ping)
+ try {
+ def response = pingAction(HttpVerb.HEAD, pingUrl, "should not be running")
+ return Outcome.Retry
+ } catch (IOException e) {
+ // expected
+ return Outcome.Success
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c0a39d92/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index df62057..897d7e3 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -643,7 +643,7 @@ class SliderTestUtils extends Assert {
assert path
assertHttpSupportInitialized()
- log.info("Fetching HTTP content at " + path);
+ log.info("Fetching HTTP content at $path");
URL url = new URL(path)
def outcome = connectionOperations.execGet(url)
String body = new String(outcome.data)
@@ -675,8 +675,7 @@ class SliderTestUtils extends Assert {
log.info("Asserting component $component expected count $expected}",)
int actual = extractLiveContainerCount(clusterDescription, component)
if (expected != actual) {
- log.warn(
- "$component actual=$actual, expected $expected in \n$clusterDescription")
+ log.warn("$component actual=$actual, expected $expected in \n$clusterDescription")
}
assert expected == actual
}
[03/28] incubator-slider git commit: SLIDER-719: bonding jersey to
UrlConnectionOperations
Posted by el...@apache.org.
SLIDER-719: bonding jersey to UrlConnectionOperations
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/61f4743f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/61f4743f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/61f4743f
Branch: refs/heads/develop
Commit: 61f4743f83d73e255eee5744be3e72bbb9c60754
Parents: 901f38c
Author: Steve Loughran <st...@apache.org>
Authored: Fri Jan 16 16:57:09 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jan 20 14:50:23 2015 +0000
----------------------------------------------------------------------
pom.xml | 5 +-
.../core/restclient/UgiJerseyBinding.java | 79 ++++++++++++++++++++
.../restclient/UrlConnectionOperations.java | 2 +-
.../slider/agent/rest/RestTestDelegates.groovy | 20 ++++-
.../slider/agent/rest/TestStandaloneREST.groovy | 7 +-
.../agent/TestAgentAMManagementWS.groovy | 4 +-
.../publisher/TestPublisherRestResources.groovy | 2 +-
.../apache/slider/test/SliderTestUtils.groovy | 61 ++++++++++++---
.../funtest/lifecycle/AgentWebPagesIT.groovy | 3 +-
9 files changed, 159 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index cf989e4..b206d66 100644
--- a/pom.xml
+++ b/pom.xml
@@ -154,6 +154,7 @@
<gson.version>2.2.2</gson.version>
<guice.version>3.0</guice.version>
<httpclient.version>3.1</httpclient.version>
+ <httpclient4.version>4.2.5</httpclient4.version>
<jackson.version>1.9.13</jackson.version>
<jcommander.version>1.30</jcommander.version>
@@ -645,13 +646,13 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
- <version>4.2.5</version>
+ <version>${httpclient4.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
- <version>4.2.5</version>
+ <version>${httpclient4.version}</version>
</dependency>
<!-- ======================================================== -->
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
new file mode 100644
index 0000000..6af8e23
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.restclient;
+
+import com.google.common.base.Preconditions;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+/**
+ * Class to bond to a Jersey client, for UGI integration and SPNEGO.
+ * <p>
+ * Usage: create an instance, then when creating a Jersey <code>Client</code>
+ * pass in to the constructor the handler provided by {@link #getHandler()}
+ *
+ * @see https://jersey.java.net/apidocs/1.17/jersey/com/sun/jersey/client/urlconnection/HttpURLConnectionFactory.html
+ */
+public class UgiJerseyBinding implements
+ HttpURLConnectionFactory {
+ private final UrlConnectionOperations operations;
+ private final URLConnectionClientHandler handler;
+
+ /**
+ * Construct an instance
+ * @param operations operations instance
+ */
+ @SuppressWarnings("ThisEscapedInObjectConstruction")
+ public UgiJerseyBinding(UrlConnectionOperations operations) {
+ Preconditions.checkArgument(operations != null, "Null operations");
+ this.operations = operations;
+ handler = new URLConnectionClientHandler(this);
+ }
+
+ /**
+ * Get a URL connection.
+ * @param url
+ * @return the connection
+ * @throws IOException any problem. {@link AuthenticationException}
+ * errors are wrapped
+ */
+ @Override
+ public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
+ try {
+ return operations.openConnection(url);
+ } catch (AuthenticationException e) {
+ throw new IOException(e);
+ }
+ }
+
+ public UrlConnectionOperations getOperations() {
+ return operations;
+ }
+
+ public URLConnectionClientHandler getHandler() {
+ return handler;
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
index fa0da5a..c51914a 100644
--- a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
@@ -40,7 +40,7 @@ import java.net.URL;
* Operations on the JDK UrlConnection class. This uses WebHDFS
* methods to set up the operations.
*/
-public class UrlConnectionOperations extends Configured {
+public class UrlConnectionOperations extends Configured {
private static final Logger log =
LoggerFactory.getLogger(UrlConnectionOperations.class);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
index 3431175..0826204 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestTestDelegates.groovy
@@ -18,6 +18,8 @@
package org.apache.slider.agent.rest
+import com.sun.jersey.api.client.Client
+import com.sun.jersey.api.client.WebResource
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.webapp.NotFoundException
@@ -27,9 +29,11 @@ import org.apache.slider.api.types.SerializedContainerInformation
import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.conf.ConfTree
import org.apache.slider.core.conf.ConfTreeOperations
+import org.apache.slider.core.registry.docstore.PublishedConfiguration
import org.apache.slider.core.restclient.HttpOperationResponse
import org.apache.slider.core.restclient.HttpVerb
import org.apache.slider.core.restclient.UrlConnectionOperations
+import org.apache.slider.server.appmaster.web.rest.RestPaths
import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource
import org.apache.slider.server.appmaster.web.rest.application.resources.PingResource
import org.apache.slider.test.Outcome
@@ -53,11 +57,14 @@ class RestTestDelegates extends SliderTestUtils {
public static final String TEST_GLOBAL_OPTION_PRESENT = "present"
final String appmaster;
+ final String application;
RestTestDelegates(String appmaster) {
this.appmaster = appmaster
+ application = appendToURL(appmaster, RestPaths.SLIDER_PATH_APPLICATION)
}
+
public void testCodahaleOperations() throws Throwable {
describe "Codahale operations"
getWebPage(appmaster)
@@ -89,6 +96,15 @@ class RestTestDelegates extends SliderTestUtils {
assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_COMPLETED)
assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_RELEASING)
}
+
+
+ public void testRestletOperations() throws Throwable {
+ Client client = createJerseyClient()
+ String path = appendToURL(application, LIVE_RESOURCES)
+ WebResource webResource = client.resource(path)
+ webResource.type(MediaType.APPLICATION_JSON)
+ .get(ConfTree.class);
+ }
public void testLiveContainers() throws Throwable {
describe "Application REST ${LIVE_CONTAINERS}"
@@ -211,7 +227,7 @@ class RestTestDelegates extends SliderTestUtils {
HttpVerb verb,
URL pingUrl,
String payload) {
- return pingAction(connectionFactory, verb, pingUrl, payload)
+ return pingAction(connectionOperations, verb, pingUrl, payload)
}
private HttpOperationResponse pingAction(
@@ -244,7 +260,7 @@ class RestTestDelegates extends SliderTestUtils {
String target = appendToURL(appmaster, SLIDER_PATH_APPLICATION, ACTION_STOP)
describe "Stop URL $target"
URL targetUrl = new URL(target)
- def outcome = connectionFactory.execHttpOperation(
+ def outcome = connectionOperations.execHttpOperation(
HttpVerb.POST,
targetUrl,
new byte[0],
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
index a3378a8..22bc4ae 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
@@ -62,14 +62,14 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
def realappmaster = report.originalTrackingUrl
// set up url config to match
- initConnectionFactory(launcher.configuration)
+ initHttpTestSupport(launcher.configuration)
- execHttpRequest(WEB_STARTUP_TIME) {
+ execOperation(WEB_STARTUP_TIME) {
GET(realappmaster)
}
- execHttpRequest(WEB_STARTUP_TIME) {
+ execOperation(WEB_STARTUP_TIME) {
def metrics = GET(realappmaster, SYSTEM_METRICS)
log.info metrics
}
@@ -90,6 +90,7 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
RestTestDelegates proxied = new RestTestDelegates(appmaster)
RestTestDelegates direct = new RestTestDelegates(realappmaster)
+ direct.testRestletOperations();
proxied.testCodahaleOperations()
direct.testCodahaleOperations()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
index 2e2e62b..7eb1d17 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -153,7 +153,7 @@ class TestAgentAMManagementWS extends AgentTestBase {
log.info("tracking URL is $proxyAM")
// spin awaiting the agent web page coming up.
- execHttpRequest(WEB_STARTUP_TIME) {
+ execOperation(WEB_STARTUP_TIME) {
GET(proxyAM)
}
@@ -173,7 +173,7 @@ class TestAgentAMManagementWS extends AgentTestBase {
log.info "AM live, now fetching agent at $agent_url"
// spin awaiting the agent web page coming up.
- execHttpRequest(WEB_STARTUP_TIME) {
+ execOperation(WEB_STARTUP_TIME) {
GET(agent_url)
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
index e3f38eb..29f0510 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
@@ -107,7 +107,7 @@ class TestPublisherRestResources extends AgentTestBase {
webResource = client.resource(sliderConfigset + "dummy-site");
- execHttpRequest(30000) {
+ execOperation(30000) {
GET(sliderConfigset)
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 2c6b5fe..9dd2828 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -18,6 +18,10 @@
package org.apache.slider.test
+import com.sun.jersey.api.client.Client
+import com.sun.jersey.api.client.config.ClientConfig
+import com.sun.jersey.api.client.config.DefaultClientConfig
+import com.sun.jersey.api.json.JSONConfiguration
import groovy.json.JsonOutput
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
@@ -55,6 +59,7 @@ import org.apache.slider.core.main.ServiceLaunchException
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.persist.JsonSerDeser
import org.apache.slider.core.registry.docstore.PublishedConfigSet
+import org.apache.slider.core.restclient.UgiJerseyBinding
import org.apache.slider.core.restclient.UrlConnectionOperations
import org.apache.slider.server.appmaster.web.HttpCacheHeaders
import org.apache.slider.server.appmaster.web.rest.RestPaths
@@ -450,7 +455,8 @@ class SliderTestUtils extends Assert {
}
/**
- * Fetch a web page
+ * Fetch a web page using HttpClient 3.1.
+ * This <i>DOES NOT</i> work with secure connections.
* @param url URL
* @return the response body
*/
@@ -478,6 +484,9 @@ class SliderTestUtils extends Assert {
/**
* Fetches a web page asserting that the response code is between 200 and 400.
+ * This <i>DOES NOT</i> work with secure connections.
+ * <p>
+ *
* Will error on 400 and 500 series response codes and let 200 and 300 through.
* @param url URL to get as string
* @return body of response
@@ -548,7 +557,7 @@ class SliderTestUtils extends Assert {
/**
* Fetches a web page asserting that the response code is between 200 and 400.
* Will error on 400 and 500 series response codes and let 200 and 300 through.
- *
+ * <p>
* if security is enabled, this uses SPNEGO to auth
* @param page
* @return body of response
@@ -559,13 +568,14 @@ class SliderTestUtils extends Assert {
}
/**
- * Execute any of the http requests, swallowing exceptions until
- * eventually they time out
+ * Execute any operation provided as a closure which returns a string, swallowing exceptions until
+ * eventually they time out.
+ *
* @param timeout
* @param operation
* @return
*/
- public static String execHttpRequest(int timeout, Closure operation) {
+ public static String execOperation(int timeout, Closure operation) {
Duration duration = new Duration(timeout).start()
Exception ex = new IOException("limit exceeded before starting");
while (!duration.limitExceeded) {
@@ -581,12 +591,41 @@ class SliderTestUtils extends Assert {
throw ex;
}
- static UrlConnectionOperations connectionFactory
+ /**
+ * Static factory for URL connections
+ */
+ static UrlConnectionOperations connectionOperations
+ static UgiJerseyBinding jerseyBinding;
- public static def initConnectionFactory(Configuration conf) {
- connectionFactory = new UrlConnectionOperations(conf);
+
+ /**
+ * Static initializer of the connection operations
+ * @param conf config
+ */
+ public static synchronized void initHttpTestSupport(Configuration conf) {
+ connectionOperations = new UrlConnectionOperations(conf);
+ jerseyBinding = new UgiJerseyBinding(connectionOperations)
+
}
+ /**
+ * Check for the HTTP support being initialized
+ */
+ public static synchronized void assertHttpSupportInitialized() {
+ assert connectionOperations
+ assert jerseyBinding
+ }
+
+ /**
+ * Create Jersey client
+ * @return
+ */
+ public static Client createJerseyClient() {
+ assertHttpSupportInitialized()
+ ClientConfig clientConfig = new DefaultClientConfig();
+ clientConfig.features[JSONConfiguration.FEATURE_POJO_MAPPING] =Boolean.TRUE;
+ return new Client(jerseyBinding.handler, clientConfig);
+ }
/**
* Fetches a web page asserting that the response code is between 200 and 400.
@@ -594,7 +633,7 @@ class SliderTestUtils extends Assert {
*
* if security is enabled, this uses SPNEGO to auth
* <p>
- * Relies on {@link #initConnectionFactory(org.apache.hadoop.conf.Configuration)}
+ * Relies on {@link #initHttpTestSupport(org.apache.hadoop.conf.Configuration)}
* to have been called.
*
* @param path path to page
@@ -603,11 +642,11 @@ class SliderTestUtils extends Assert {
*/
public static String getWebPage(String path, Closure connectionChecks = null) {
assert path
- assert null != connectionFactory
+ assertHttpSupportInitialized()
log.info("Fetching HTTP content at " + path);
URL url = new URL(path)
- def outcome = connectionFactory.execGet(url)
+ def outcome = connectionOperations.execGet(url)
String body = new String(outcome.data)
return body;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61f4743f/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
index ce1d955..de8dcdf 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
@@ -29,7 +29,6 @@ import org.apache.slider.common.tools.ConfigHelper
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
-import org.apache.slider.server.appmaster.web.rest.RestPaths
import org.junit.After
import org.junit.Before
import org.junit.Test
@@ -82,7 +81,7 @@ public class AgentWebPagesIT extends AgentCommandTestBase
def conf = SLIDER_CONFIG
- initConnectionFactory(conf)
+ initHttpTestSupport(conf)
def appId = ensureYarnApplicationIsUp(launchReportFile)
assert appId
[07/28] incubator-slider git commit: SLIDER-719 jersey tests about to
migrate to WebTarget
Posted by el...@apache.org.
SLIDER-719 jersey tests about to migrate to WebTarget
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/79d0e14f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/79d0e14f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/79d0e14f
Branch: refs/heads/develop
Commit: 79d0e14f1d63b1e1a41edb77dce191238fb52eea
Parents: 72e4790
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jan 20 17:47:51 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jan 20 17:47:51 2015 +0000
----------------------------------------------------------------------
.../agent/rest/JerseyTestDelegates.groovy | 120 +++++++++++--------
1 file changed, 71 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/79d0e14f/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
index a06be23..cb89bff 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
@@ -23,7 +23,6 @@ import com.sun.jersey.api.client.WebResource
import com.sun.jersey.api.client.UniformInterfaceException
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.webapp.NotFoundException
import org.apache.slider.api.StateValues
import org.apache.slider.api.types.SerializedComponentInformation
import org.apache.slider.api.types.SerializedContainerInformation
@@ -77,10 +76,20 @@ class JerseyTestDelegates extends SliderTestUtils {
* @return
*/
public <T> T jGetApplicationResource(String subpath, Class<T> c) {
+ return (T)jExec(HttpVerb.GET, subpath, c)
+ }
+
+ /**
+ * <T> T get(Class<T> c)
+ * Get operation against a path under the Application
+ * @param subpath path
+ * @return
+ */
+ public <T> T jExec(HttpVerb method, String subpath, Class<T> c) {
assert c
def appPath = appendToURL(SLIDER_PATH_APPLICATION, subpath)
WebResource webResource = buildResource(appPath)
- (T)webResource.get(c)
+ (T) webResource.method(method.verb, c)
}
/**
@@ -137,7 +146,13 @@ class JerseyTestDelegates extends SliderTestUtils {
log.info jerseyGet(SYSTEM_METRICS)
}
- public <T> T fetchJType(
+ /**
+ * Fetch a typed entry <i>under the application path</i>
+ * @param subpath
+ * @param clazz
+ * @return
+ */
+ public <T> T jFetchType(
String subpath, Class<T> clazz) {
(T)jGetApplicationResource(subpath, clazz)
}
@@ -149,7 +164,7 @@ class JerseyTestDelegates extends SliderTestUtils {
return tree
}
- public void testLiveResources() throws Throwable {
+ public void testLiveResources() throws Throwable {
describe "Live Resources"
ConfTreeOperations tree = jGetConfigTree(LIVE_RESOURCES)
@@ -190,7 +205,7 @@ class JerseyTestDelegates extends SliderTestUtils {
describe "containers"
SerializedContainerInformation retrievedContainerInfo =
- fetchJType(
+ jFetchType(
LIVE_CONTAINERS + "/${amContainerId}",
SerializedContainerInformation
)
@@ -198,7 +213,7 @@ class JerseyTestDelegates extends SliderTestUtils {
// fetch missing
try {
- def result = fetchJType(
+ def result = jFetchType(
LIVE_CONTAINERS + "/unknown",
SerializedContainerInformation
)
@@ -211,7 +226,7 @@ class JerseyTestDelegates extends SliderTestUtils {
describe "components"
Map<String, SerializedComponentInformation> components =
- fetchJType(LIVE_COMPONENTS, HashMap)
+ jFetchType(LIVE_COMPONENTS, HashMap)
// two components
assert components.size() == 1
log.info "${components}"
@@ -219,7 +234,7 @@ class JerseyTestDelegates extends SliderTestUtils {
SerializedComponentInformation amComponentInfo =
(SerializedComponentInformation) components[COMPONENT_AM]
- SerializedComponentInformation amFullInfo = fetchJType(
+ SerializedComponentInformation amFullInfo = jFetchType(
LIVE_COMPONENTS + "/${COMPONENT_AM}",
SerializedComponentInformation
)
@@ -230,6 +245,39 @@ class JerseyTestDelegates extends SliderTestUtils {
}
/**
+ * Assert that a path resolves to an array list that contains
+ * those entries (and only those entries) expected
+ * @param appmaster AM ref
+ * @param path path under AM
+ * @param entries entries to assert the presence of
+ */
+ public void assertPathServesList(
+ String appmaster,
+ String path,
+ List<String> entries) {
+ def list = jFetchType(path, ArrayList)
+ assert list.size() == entries.size()
+ assert entries.containsAll(list)
+ }
+
+ /**
+ * Fetch a list of URLs, all of which must be of the same type
+ * @param clazz class of resolved values
+ * @param appmaster URL to app master
+ * @param subpaths list of subpaths
+ * @return a map of paths to values
+ */
+ public <T> Map<String, T> fetchTypeList(
+ Class<T> clazz, String appmaster, List<String> subpaths
+ ) {
+ Map<String, T> results = [:]
+ subpaths.each { String it ->
+ results[it] = (jFetchType(it, clazz))
+ }
+ return results;
+ }
+
+ /**
* Test the rest model. For this to work the cluster has to be configured
* with the global option
* @param appmaster
@@ -241,14 +289,14 @@ class JerseyTestDelegates extends SliderTestUtils {
MODEL,
ApplicationResource.MODEL_ENTRIES)
- def unresolvedConf = fetchJType(MODEL_DESIRED, AggregateConf)
+ def unresolvedConf = jFetchType(MODEL_DESIRED, AggregateConf)
// log.info "Unresolved \n$unresolvedConf"
def unresolvedAppConf = unresolvedConf.appConfOperations
def sam = "slider-appmaster"
assert unresolvedAppConf.getComponentOpt(sam,
TEST_GLOBAL_OPTION, "") == ""
- def resolvedConf = fetchJType(MODEL_RESOLVED, AggregateConf)
+ def resolvedConf = jFetchType(MODEL_RESOLVED, AggregateConf)
// log.info "Resolved \n$resolvedConf"
assert resolvedConf.appConfOperations.getComponentOpt(
sam, TEST_GLOBAL_OPTION, "") == TEST_GLOBAL_OPTION_PRESENT
@@ -261,54 +309,28 @@ class JerseyTestDelegates extends SliderTestUtils {
def resolved = fetchTypeList(ConfTree, appmaster,
[MODEL_RESOLVED_APPCONF, MODEL_RESOLVED_RESOURCES])
- assert resolved[MODEL_RESOLVED_APPCONF].components[sam]
- [TEST_GLOBAL_OPTION] ==
- TEST_GLOBAL_OPTION_PRESENT
- }
-
-
- public void testRestletGetOperations() throws Throwable {
- Client client = createJerseyClient()
- String path = appendToURL(application, LIVE_RESOURCES)
- WebResource webResource = client.resource(path)
- webResource.type(MediaType.APPLICATION_JSON)
- .get(ConfTree.class);
+ assert resolved[MODEL_RESOLVED_APPCONF].
+ components[sam][TEST_GLOBAL_OPTION] == TEST_GLOBAL_OPTION_PRESENT
}
public void testPing() {
// GET
- String ping = appendToURL(appmaster, SLIDER_PATH_APPLICATION, ACTION_PING)
- describe "ping to AM URL $appmaster, ping URL $ping"
- def pinged = fetchJType(
- ACTION_PING + "?body=hello",
- PingResource
- )
+ describe "pinging"
+
+ def pinged = jExec(HttpVerb.GET, ACTION_PING, PingResource)
log.info "Ping GET: $pinged"
-
- URL pingUrl = new URL(ping)
- def message = "hello"
-
// HEAD
- pingAction(HttpVerb.HEAD, pingUrl, message)
-
- // Other verbs
- pingAction(HttpVerb.POST, pingUrl, message)
- pingAction(HttpVerb.PUT, pingUrl, message)
- pingAction(HttpVerb.DELETE, pingUrl, message)
+ jExec(HttpVerb.HEAD, ACTION_PING, PingResource)
+ jExec(HttpVerb.PUT, ACTION_PING, PingResource)
+ jExec(HttpVerb.DELETE, ACTION_PING, PingResource)
+ jExec(HttpVerb.POST, ACTION_PING, PingResource)
}
-
- private HttpOperationResponse pingAction(
- HttpVerb verb,
- URL pingUrl,
- String payload) {
- return pingAction(connectionOperations, verb, pingUrl, payload)
- }
-
private HttpOperationResponse pingAction(
- UrlConnectionOperations ops, HttpVerb verb, URL pingUrl, String payload) {
+ HttpVerb verb, String subpath, String payload) {
def pinged
+ jExec(verb,subpath, PingResource)
def outcome = ops.execHttpOperation(
verb,
pingUrl,
@@ -355,7 +377,7 @@ class JerseyTestDelegates extends SliderTestUtils {
[url: ping],
true,
"AM failed to shut down") {
- def pinged = fetchJType(ACTION_PING + "?body=hello",
+ def pinged = jFetchType(ACTION_PING + "?body=hello",
PingResource
)
fail("AM didn't shut down; Ping GET= $pinged")
@@ -383,10 +405,10 @@ class JerseyTestDelegates extends SliderTestUtils {
public void suite() {
- testRestletGetOperations();
testCodahaleOperations()
testLiveResources()
testLiveContainers();
+ testPing();
testRESTModel()
}