You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/12/18 22:33:15 UTC
[01/26] incubator-slider git commit: SLIDER-724 Implement OfflineFilter to return 503 "unavailable", for AM bootstrap —removing filter
Repository: incubator-slider
Updated Branches:
refs/heads/develop 8609f16aa -> 57d0d2ed2
SLIDER-724 Implement OfflineFilter to return 503 "unavailable", for AM bootstrap —removing filter
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b09bd848
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b09bd848
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b09bd848
Branch: refs/heads/develop
Commit: b09bd848069f1fff450e4d4690cafcf11678f544
Parents: fe3ffc9
Author: Steve Loughran <st...@apache.org>
Authored: Mon Dec 15 14:27:32 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Dec 16 20:25:39 2014 +0000
----------------------------------------------------------------------
.../appmaster/web/utils/OfflineFilter.java | 104 -------------------
1 file changed, 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b09bd848/slider-core/src/main/java/org/apache/slider/server/appmaster/web/utils/OfflineFilter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/utils/OfflineFilter.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/utils/OfflineFilter.java
deleted file mode 100644
index ad16f4e..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/utils/OfflineFilter.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.http.FilterContainer;
-
-import javax.servlet.Filter;
-import javax.servlet.FilterChain;
-import javax.servlet.FilterConfig;
-import javax.servlet.ServletException;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
-import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Offline filter.
- * All filter instances share the static offline flag.
- * <p>
- * Callers must use the method {@link #bindFilter(FilterContainer)}
- * to bind to the web container.
- */
-public class OfflineFilter implements Filter {
-
- private static final AtomicBoolean offline = new AtomicBoolean(true);
-
- private static String offlineMessage = "offline";
- private static final AtomicInteger retry = new AtomicInteger(10);
-
- public OfflineFilter() {
- }
-
- public static synchronized void goOffline(String message) {
- Preconditions.checkArgument(message != null, "null message");
- offline.set(true);
- offlineMessage = message;
- }
-
- public static synchronized String getOfflineMessage() {
- return offlineMessage;
- }
-
- public static int getRetry() {
- return retry.intValue();
- }
-
- public static void setRetry(int retryCount) {
- retry.set(retryCount);
- }
-
- @Override
- public void init(FilterConfig filterConfig) throws ServletException {
-
- }
-
- @Override
- public void doFilter(ServletRequest request,
- ServletResponse response,
- FilterChain chain) throws IOException, ServletException {
-
- if (!offline.get()) {
- chain.doFilter(request, response);
- } else {
- // service is offline
- HttpServletResponse httpResponse = (HttpServletResponse) response;
- httpResponse.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE,
- getOfflineMessage());
- }
- }
-
- @Override
- public void destroy() {
-
- }
-
- /**
- * Add the filter to a container
- * @param container container
- */
- public static void bindFilter(FilterContainer container) {
- container.addFilter("OfflineFilter",
- "org.apache.slider.server.appmaster.web.utils.OfflineFilter",
- null);
- }
-}
[25/26] incubator-slider git commit: SLIDER-711 components/ and
components/$component
Posted by st...@apache.org.
SLIDER-711 components/ and components/$component
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3c450b93
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3c450b93
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3c450b93
Branch: refs/heads/develop
Commit: 3c450b938c8b3144f527a7c14790fd2e1d491549
Parents: 08b979c
Author: Steve Loughran <st...@apache.org>
Authored: Thu Dec 18 20:43:53 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Dec 18 21:01:30 2014 +0000
----------------------------------------------------------------------
.../types/SerializedComponentInformation.java | 41 ++++++++
.../server/appmaster/RoleLaunchService.java | 2 +-
.../slider/server/appmaster/state/AppState.java | 3 +-
.../server/appmaster/state/RoleStatus.java | 39 ++++---
.../web/rest/AbstractSliderResource.java | 3 +
.../server/appmaster/web/rest/RestPaths.java | 2 +
.../rest/application/ApplicationResource.java | 102 +++++++++++++++++--
.../resources/ContainerListRefresher.java | 51 ----------
.../resources/LiveComponentsRefresher.java | 52 ++++++++++
.../resources/LiveContainersRefresher.java | 51 ++++++++++
.../standalone/TestStandaloneAgentWeb.groovy | 73 ++++++++++---
.../TestMockAppStateAppRestIntegration.groovy | 8 +-
12 files changed, 322 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c450b93/slider-core/src/main/java/org/apache/slider/api/types/SerializedComponentInformation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/SerializedComponentInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/SerializedComponentInformation.java
new file mode 100644
index 0000000..8eac3db
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/types/SerializedComponentInformation.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.types;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.List;
+
+/**
+ * Serializable version of component data
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+
+public class SerializedComponentInformation {
+
+ public String name;
+ public int priority;
+ public int desired, actual, requested, releasing;
+ public int failed, started, startFailed, completed, totalRequested;
+ public String failureMessage;
+ public int placementPolicy;
+ public List<String> containers;
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c450b93/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
index 7701cfe..4333a09 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
@@ -131,7 +131,7 @@ public class RoleLaunchService
new RoleLaunchService.RoleLauncher(container,
role.getProviderRole(),
clusterSpec,
- clusterSpec.getResourceOperations() .getOrAddComponent(roleName),
+ clusterSpec.getResourceOperations().getOrAddComponent(roleName),
clusterSpec.getAppConfOperations().getOrAddComponent(roleName));
execute(launcher);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c450b93/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index e9d0371..9ae20a5 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -839,8 +839,7 @@ public class AppState {
}
return rs;
}
-
-
+
public RoleStatus lookupRoleStatus(Container c) throws YarnRuntimeException {
return lookupRoleStatus(ContainerPriority.extractRole(c));
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c450b93/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 3c860d6..74a1c9c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -19,6 +19,7 @@
package org.apache.slider.server.appmaster.state;
import org.apache.slider.api.StatusKeys;
+import org.apache.slider.api.types.SerializedComponentInformation;
import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
@@ -37,8 +38,7 @@ public final class RoleStatus implements Cloneable {
private final String name;
/**
- * Role key in the container details stored in the AM,
- * currently mapped to priority
+ * Role priority
*/
private final int key;
@@ -47,23 +47,6 @@ public final class RoleStatus implements Cloneable {
private int desired, actual, requested, releasing;
private int failed, started, startFailed, completed, totalRequested;
- /**
- * value to use when specifiying "no limit" for instances: {@value}
- */
- public static final int UNLIMITED_INSTANCES = 1;
-
- /**
- * minimum number of instances of a role permitted in a valid
- * configuration. Default: 0.
- */
- private int minimum = 0;
-
- /**
- * maximum number of instances of a role permitted in a valid
- * configuration. Default: unlimited.
- */
- private int maximum = UNLIMITED_INSTANCES;
-
private String failureMessage = "";
public RoleStatus(ProviderRole providerRole) {
@@ -260,8 +243,6 @@ public final class RoleStatus implements Cloneable {
return "RoleStatus{" +
"name='" + name + '\'' +
", key=" + key +
- ", minimum=" + minimum +
- ", maximum=" + maximum +
", desired=" + desired +
", actual=" + actual +
", requested=" + requested +
@@ -304,6 +285,22 @@ public final class RoleStatus implements Cloneable {
return stats;
}
+ public SerializedComponentInformation serialize() {
+ SerializedComponentInformation info = new SerializedComponentInformation();
+ info.name = name;
+ info.priority = getPriority();
+ info.desired = desired;
+ info.actual = actual;
+ info.requested = requested;
+ info.releasing = releasing;
+ info.failed = failed;
+ info.startFailed = startFailed;
+ info.requested = requested;
+ info.placementPolicy = getPlacementPolicy();
+ info.failureMessage = failureMessage;
+ return info;
+ }
+
/**
* Compare two role status entries by name
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c450b93/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
index ede46f0..dc07c10 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
@@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;
+import java.io.FileNotFoundException;
import java.net.URI;
import java.net.URL;
@@ -79,6 +80,8 @@ public abstract class AbstractSliderResource {
} catch (WebApplicationException e) {
// rethrow direct
throw e;
+ } catch (FileNotFoundException e) {
+ return new NotFoundException("Not found: " + path);
} catch (PathNotFoundException e) {
return new NotFoundException("Not found: " + path);
} catch (AuthenticationFailedException e) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c450b93/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
index 28c0fab..011ec3a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
@@ -130,8 +130,10 @@ public class RestPaths {
public static final String APPLICATION_WADL = "/application.wadl";
+ public static final String LIVE = "/live";
public static final String LIVE_RESOURCES = "/live/resources";
public static final String LIVE_CONTAINERS = "/live/containers";
+ public static final String LIVE_COMPONENTS = "/live/components";
public static final String MODEL_DESIRED = "/model/desired";
public static final String MODEL_DESIRED_APPCONF = MODEL_DESIRED +"/appconf";
public static final String MODEL_DESIRED_RESOURCES = MODEL_DESIRED +"/resources";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c450b93/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
index e4be96f..9419765 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
@@ -19,15 +19,23 @@
package org.apache.slider.server.appmaster.web.rest.application;
import com.google.common.collect.Lists;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.webapp.NotFoundException;
+import org.apache.slider.api.types.SerializedComponentInformation;
import org.apache.slider.api.types.SerializedContainerInformation;
import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.exceptions.NoSuchNodeException;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
import org.apache.slider.server.appmaster.web.rest.application.resources.CachedContent;
-import org.apache.slider.server.appmaster.web.rest.application.resources.ContainerListRefresher;
+import org.apache.slider.server.appmaster.web.rest.application.resources.LiveContainersRefresher;
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
+import org.apache.slider.server.appmaster.web.rest.application.resources.LiveComponentsRefresher;
import org.apache.slider.server.appmaster.web.rest.application.resources.LiveResourcesRefresher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,28 +43,43 @@ import org.slf4j.LoggerFactory;
import javax.inject.Singleton;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Singleton
+@SuppressWarnings("unchecked")
public class ApplicationResource extends AbstractSliderResource {
private static final Logger log =
LoggerFactory.getLogger(ApplicationResource.class);
public static final int LIFESPAN = 1000;
+ public static final List<String> LIVE_ENTRIES = toJsonList("resources",
+ "containers",
+ "components",
+ "nodes",
+ "statistics",
+ "internal");
+ public static final List<String> ROOT_ENTRIES =
+ toJsonList("model", "live", "actions");
private final ContentCache cache = new ContentCache();
+ private final StateAccessForProviders state;
public ApplicationResource(WebAppApi slider) {
super(slider);
- StateAccessForProviders state = slider.getAppState();
+ state = slider.getAppState();
cache.put(RestPaths.LIVE_RESOURCES,
new CachedContent<ConfTree>(LIFESPAN,
new LiveResourcesRefresher(state)));
cache.put(RestPaths.LIVE_CONTAINERS,
new CachedContent<Map<String, SerializedContainerInformation>>(LIFESPAN,
- new ContainerListRefresher(state)));
+ new LiveContainersRefresher(state)));
+ cache.put(RestPaths.LIVE_COMPONENTS,
+ new CachedContent<Map<String, SerializedComponentInformation>> (LIFESPAN,
+ new LiveComponentsRefresher(state)));
}
/**
@@ -64,7 +87,7 @@ public class ApplicationResource extends AbstractSliderResource {
* @param elements elements
* @return something that can be returned
*/
- private List<String> toJsonList(String... elements) {
+ private static List<String> toJsonList(String... elements) {
return Lists.newArrayList(elements);
}
@@ -72,7 +95,7 @@ public class ApplicationResource extends AbstractSliderResource {
@Path("/")
@Produces({MediaType.APPLICATION_JSON})
public List<String> getRoot() {
- return toJsonList("model", "live", "actions");
+ return ROOT_ENTRIES;
}
@GET
@@ -86,12 +109,7 @@ public class ApplicationResource extends AbstractSliderResource {
@Path("/live")
@Produces({MediaType.APPLICATION_JSON})
public List<String> getLive() {
- return toJsonList("resources",
- "containers",
- "components",
- "nodes",
- "statistics",
- "internal");
+ return LIVE_ENTRIES;
}
@GET
@@ -104,6 +122,7 @@ public class ApplicationResource extends AbstractSliderResource {
throw buildException(RestPaths.LIVE_RESOURCES, e);
}
}
+
@GET
@Path(RestPaths.LIVE_CONTAINERS)
@Produces({MediaType.APPLICATION_JSON})
@@ -116,4 +135,65 @@ public class ApplicationResource extends AbstractSliderResource {
}
}
+ @GET
+ @Path(RestPaths.LIVE_CONTAINERS + "/{containerId}")
+ @Produces({MediaType.APPLICATION_JSON})
+ public SerializedContainerInformation getLiveContainer(
+ @PathParam("containerId") String containerId) {
+ try {
+ RoleInstance id = state.getLiveInstanceByContainerID(containerId);
+ return id.serialize();
+ } catch (NoSuchNodeException e) {
+ throw new NotFoundException("Unknown container: " + containerId);
+ } catch (Exception e) {
+ throw buildException(RestPaths.LIVE_CONTAINERS + "/"+ containerId, e);
+ }
+ }
+
+ @GET
+ @Path(RestPaths.LIVE_COMPONENTS)
+ @Produces({MediaType.APPLICATION_JSON})
+ public Map<String, SerializedComponentInformation> getLiveComponents() {
+ try {
+ return (Map<String, SerializedComponentInformation>) cache.lookup(
+ RestPaths.LIVE_COMPONENTS);
+ } catch (Exception e) {
+ throw buildException(RestPaths.LIVE_COMPONENTS, e);
+ }
+ }
+
+ @GET
+ @Path(RestPaths.LIVE_COMPONENTS+"/{component}")
+ @Produces({MediaType.APPLICATION_JSON})
+ public SerializedComponentInformation getLiveComponent(
+ @PathParam("component") String component) {
+ try {
+ RoleStatus roleStatus = state.lookupRoleStatus(component);
+ SerializedComponentInformation info = roleStatus.serialize();
+ List<RoleInstance> containers = lookupRoleContainers(component);
+ info.containers = new ArrayList<String>(containers.size());
+ for (RoleInstance container : containers) {
+ info.containers.add(container.id);
+ }
+ return info;
+ } catch (YarnRuntimeException e) {
+ throw new NotFoundException("Unknown component: " + component);
+ } catch (Exception e) {
+ throw buildException(RestPaths.LIVE_CONTAINERS, e);
+ }
+ }
+
+ List<RoleInstance> lookupRoleContainers(String component) {
+ RoleStatus roleStatus = state.lookupRoleStatus(component);
+ List<RoleInstance> ownedContainerList = state.cloneOwnedContainerList();
+ List<RoleInstance> matching = new ArrayList<RoleInstance>(ownedContainerList.size());
+ int roleId = roleStatus.getPriority();
+ for (RoleInstance instance : ownedContainerList) {
+ if (instance.roleId == roleId) {
+ matching.add(instance);
+ }
+ }
+ return matching;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c450b93/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java
deleted file mode 100644
index 7e74062..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.application.resources;
-
-import org.apache.slider.api.types.SerializedContainerInformation;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Refresh the container list.
- */
-public class ContainerListRefresher implements ResourceRefresher<Map<String, SerializedContainerInformation>> {
-
- private final StateAccessForProviders state;
-
- public ContainerListRefresher(StateAccessForProviders state) {
- this.state = state;
- }
-
- @Override
- public Map<String, SerializedContainerInformation> refresh() {
- List<RoleInstance> containerList = state.cloneOwnedContainerList();
-
- Map<String, SerializedContainerInformation> map = new HashMap<String, SerializedContainerInformation>();
- for (RoleInstance instance : containerList) {
- SerializedContainerInformation serialized = instance.serialize();
- map.put(serialized.containerId, serialized);
- }
- return map;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c450b93/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveComponentsRefresher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveComponentsRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveComponentsRefresher.java
new file mode 100644
index 0000000..e543265
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveComponentsRefresher.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.application.resources;
+
+import org.apache.slider.api.types.SerializedComponentInformation;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class LiveComponentsRefresher
+ implements ResourceRefresher<Map<String, SerializedComponentInformation>> {
+
+ private final StateAccessForProviders state;
+
+ public LiveComponentsRefresher(StateAccessForProviders state) {
+ this.state = state;
+ }
+
+ @Override
+ public Map<String, SerializedComponentInformation> refresh() {
+
+ Map<Integer, RoleStatus> roleStatusMap = state.getRoleStatusMap();
+ Map<String, SerializedComponentInformation> results =
+ new HashMap<String, SerializedComponentInformation>(
+ roleStatusMap.size());
+
+ for (RoleStatus status : roleStatusMap.values()) {
+ String name = status.getName();
+ SerializedComponentInformation info = status.serialize();
+ results.put(name, info);
+ }
+ return results;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c450b93/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java
new file mode 100644
index 0000000..39a543b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.application.resources;
+
+import org.apache.slider.api.types.SerializedContainerInformation;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Refresh the container list.
+ */
+public class LiveContainersRefresher implements ResourceRefresher<Map<String, SerializedContainerInformation>> {
+
+ private final StateAccessForProviders state;
+
+ public LiveContainersRefresher(StateAccessForProviders state) {
+ this.state = state;
+ }
+
+ @Override
+ public Map<String, SerializedContainerInformation> refresh() {
+ List<RoleInstance> containerList = state.cloneOwnedContainerList();
+
+ Map<String, SerializedContainerInformation> map = new HashMap<String, SerializedContainerInformation>();
+ for (RoleInstance instance : containerList) {
+ SerializedContainerInformation serialized = instance.serialize();
+ map.put(serialized.containerId, serialized);
+ }
+ return map;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c450b93/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
index 4312cf3..2ad7129 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
@@ -21,9 +21,12 @@ package org.apache.slider.agent.standalone
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.webapp.NotFoundException
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.api.StateValues
+import org.apache.slider.api.types.SerializedComponentInformation
import org.apache.slider.api.types.SerializedContainerInformation
+import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource
import static org.apache.slider.api.ResourceKeys.*
import static org.apache.slider.api.StatusKeys.*
@@ -118,21 +121,65 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
describe "Application REST ${LIVE_CONTAINERS}"
- Map<String, SerializedContainerInformation> map =
+ Map<String, SerializedContainerInformation> containers =
fetchType(HashMap, appmaster, LIVE_CONTAINERS)
- assert map.size() == 1
- log.info "${map}"
- SerializedContainerInformation info = (SerializedContainerInformation) map.values()[0]
- assert info.containerId
- assert map[info.containerId]
-
- assert info.component == COMPONENT_AM
- assert info.createTime > 0
- assert info.exitCode == null
- assert info.output == null
- assert info.released == null
- assert info.state == StateValues.STATE_LIVE
+ assert containers.size() == 1
+ log.info "${containers}"
+ SerializedContainerInformation amContainerInfo = (SerializedContainerInformation) containers.values()[0]
+ assert amContainerInfo.containerId
+
+ def amContainerId = amContainerInfo.containerId
+ assert containers[amContainerId]
+
+ assert amContainerInfo.component == COMPONENT_AM
+ assert amContainerInfo.createTime > 0
+ assert amContainerInfo.exitCode == null
+ assert amContainerInfo.output == null
+ assert amContainerInfo.released == null
+ assert amContainerInfo.state == StateValues.STATE_LIVE
+
+ describe "base entry lists"
+ def list = fetchType(ArrayList, appmaster, LIVE)
+
+ def live_entries = ApplicationResource.LIVE_ENTRIES
+ assert list.size() == live_entries.size()
+ live_entries.containsAll(list)
+ describe "containers"
+
+ SerializedContainerInformation retrievedContainerInfo =
+ fetchType(SerializedContainerInformation, appmaster,
+ LIVE_CONTAINERS +"/${amContainerId}")
+ assert retrievedContainerInfo.containerId == amContainerId
+
+ // fetch missing
+ try {
+ def result = fetchType(SerializedContainerInformation, appmaster,
+ LIVE_CONTAINERS + "/unknown")
+ fail("expected an error, got $result")
+ } catch (NotFoundException e) {
+ // expected
+ }
+
+ describe "components"
+
+ Map<String, SerializedComponentInformation> components =
+ fetchType(HashMap, appmaster, LIVE_COMPONENTS)
+ // two components
+ assert components.size() == 1
+ log.info "${components}"
+
+ SerializedComponentInformation amComponentInfo =
+ (SerializedComponentInformation)components[COMPONENT_AM]
+
+ SerializedComponentInformation amFullInfo = fetchType(
+ SerializedComponentInformation,
+ appmaster,
+ LIVE_COMPONENTS +"/${COMPONENT_AM}")
+
+ assert amFullInfo.containers.size() == 1
+ assert amFullInfo.containers[0] == amContainerId
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c450b93/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
index c188b9c..712445d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
@@ -19,10 +19,6 @@
package org.apache.slider.server.appmaster.model.appstate
import groovy.util.logging.Slf4j
-import org.apache.hadoop.ipc.ProtocolSignature
-import org.apache.hadoop.yarn.exceptions.YarnException
-import org.apache.slider.api.SliderClusterProtocol
-import org.apache.slider.api.proto.Messages
import org.apache.slider.api.types.SerializedContainerInformation
import org.apache.slider.core.persist.JsonSerDeser
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring
@@ -36,7 +32,7 @@ import org.apache.slider.server.appmaster.web.WebAppApi
import org.apache.slider.server.appmaster.web.WebAppApiImpl
import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource
import org.apache.slider.server.appmaster.web.rest.application.resources.CachedContent
-import org.apache.slider.server.appmaster.web.rest.application.resources.ContainerListRefresher
+import org.apache.slider.server.appmaster.web.rest.application.resources.LiveContainersRefresher
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache
import org.apache.slider.server.appmaster.web.rest.application.resources.ResourceRefresher
import org.junit.Test
@@ -73,7 +69,7 @@ class TestMockAppStateAppRestIntegration extends BaseMockAppStateTest implements
@Test
public void testContainerListRefresher() throws Throwable {
- ContainerListRefresher clr = new ContainerListRefresher(stateAccess)
+ LiveContainersRefresher clr = new LiveContainersRefresher(stateAccess)
def map = clr.refresh()
assert map.size() == 0
List<RoleInstance> instances = startNodes()
[13/26] incubator-slider git commit: SLIDER-151 better functional
test stop (don't force, hence doesn't appear in RM as killed)
Posted by st...@apache.org.
SLIDER-151 better functional test stop (don't force, hence doesn't appear in RM as killed)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8e18403c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8e18403c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8e18403c
Branch: refs/heads/develop
Commit: 8e18403c92ad1f11817e2175e85d304f2de7ad72
Parents: e654a0f
Author: Steve Loughran <st...@apache.org>
Authored: Tue Dec 16 20:25:07 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Dec 16 20:25:40 2014 +0000
----------------------------------------------------------------------
.../slider/funtest/framework/AgentCommandTestBase.groovy | 8 +++++---
.../apache/slider/funtest/framework/CommandTestBase.groovy | 9 +++++++++
2 files changed, 14 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8e18403c/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index 668a264..832f855 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -159,14 +159,16 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
describe "Teardown app instance " + applicationName
// forced freeze with wait
+ SliderShell shell
+ shell = stop(applicationName)
teardown(applicationName)
- SliderShell shell = slider([
+
+ shell = slider([
ACTION_DESTROY,
applicationName])
if (shell.ret != 0 && shell.ret != EXIT_UNKNOWN_INSTANCE) {
- logShell(shell)
- assert fail("Old cluster either should not exist or should get destroyed; destroy exit code = ${shell.ret}")
+ assertExitCode(shell, 0)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8e18403c/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 4757d77..143d284 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -345,6 +345,15 @@ abstract class CommandTestBase extends SliderTestUtils {
freeze(name, [ARG_FORCE, ARG_WAIT, "10000"])
}
+ /**
+ * Non-forced stop, wait some seconds
+ * @param name
+ * @return
+ */
+ static SliderShell stop(String name) {
+ freeze(name, [ARG_WAIT, "10000"])
+ }
+
static SliderShell killContainer(String name, String containerID) {
slider(0,
[
[17/26] incubator-slider git commit: SLIDER-711 verify that cache
headers propagate through minimr cluster connections
Posted by st...@apache.org.
SLIDER-711 verify that cache headers propagate through minimr cluster connections
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/515ccba5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/515ccba5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/515ccba5
Branch: refs/heads/develop
Commit: 515ccba5524106e86dadb6dbab2fad71a2acbff0
Parents: ca4686d
Author: Steve Loughran <st...@apache.org>
Authored: Wed Dec 17 17:59:01 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Dec 17 17:59:01 2014 +0000
----------------------------------------------------------------------
.../server/appmaster/web/HttpCacheHeaders.java | 35 ++++++++++++
.../standalone/TestStandaloneAgentWeb.groovy | 24 ++++++---
.../org/apache/slider/test/KeysForTests.groovy | 1 +
.../apache/slider/test/SliderTestUtils.groovy | 56 +++++++++++++++-----
.../test/YarnZKMiniClusterTestBase.groovy | 1 +
5 files changed, 96 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/515ccba5/slider-core/src/main/java/org/apache/slider/server/appmaster/web/HttpCacheHeaders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/HttpCacheHeaders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/HttpCacheHeaders.java
new file mode 100644
index 0000000..be8960d
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/HttpCacheHeaders.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web;
+
+/*
+
+
+ , );
+ long now = System.currentTimeMillis();
+ httpRes.addDateHeader ( "Expires", now );
+ httpRes.addDateHeader ( "Date", now );
+ httpRes.addHeader ( "Pragma", "no-cache" );
+ */
+public interface HttpCacheHeaders {
+ String HTTP_HEADER_CACHE_CONTROL = "Cache-Control";
+ String HTTP_HEADER_CACHE_CONTROL_NONE = "no-cache";
+ String HTTP_HEADER_PRAGMA = "Pragma";
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/515ccba5/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
index d524698..7f8ddf5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
@@ -23,6 +23,7 @@ import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.agent.AgentMiniClusterTestBase
+
import static org.apache.slider.api.ResourceKeys.*
import static org.apache.slider.api.StatusKeys.*
import org.apache.slider.client.SliderClient
@@ -58,6 +59,11 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
ApplicationReport report = waitForClusterLive(client)
def realappmaster = report.originalTrackingUrl
+
+ // set up url config to match
+ initConnectionFactory(launcher.configuration)
+
+
execHttpRequest(30000) {
GET(realappmaster)
}
@@ -78,14 +84,19 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
describe "Codahale operations"
// now switch to the Hadoop URL connection, with SPNEGO escalation
- getWebPage(conf, appmaster)
- getWebPage(conf, appmaster, SYSTEM_THREADS)
- getWebPage(conf, appmaster, SYSTEM_HEALTHCHECK)
- getWebPage(conf, appmaster, SYSTEM_METRICS_JSON)
+ getWebPage(appmaster)
+ getWebPage(appmaster, SYSTEM_THREADS)
+ getWebPage(appmaster, SYSTEM_HEALTHCHECK)
+ getWebPage(appmaster, SYSTEM_METRICS_JSON)
- log.info getWebPage(conf, realappmaster, SYSTEM_METRICS_JSON)
+ log.info getWebPage(realappmaster, SYSTEM_METRICS_JSON)
+
+ // get the root page, including some checks for connectivity
+ getWebPage(appmaster, {
+ HttpURLConnection conn ->
+ assertConnectionNotCaching(conn)
+ })
-
// now some REST gets
describe "Application REST GETs"
@@ -111,7 +122,6 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
ConfTreeSerDeser serDeser = new ConfTreeSerDeser()
def json = getWebPage(
- conf,
appmaster,
SLIDER_PATH_APPLICATION + subpath)
def ctree = serDeser.fromJson(json)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/515ccba5/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy b/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
index ddd52eb..4fae2ae 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
@@ -34,4 +34,5 @@ public interface KeysForTests extends SliderKeys, SliderXMLConfKeysForTesting {
String WAIT_TIME_ARG = WAIT_TIME.toString()
String SLIDER_TEST_XML = "slider-test.xml"
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/515ccba5/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index d3cbae0..39dfe39 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -52,10 +52,12 @@ import org.apache.slider.core.exceptions.WaitTimeoutException
import org.apache.slider.core.main.ServiceLaunchException
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.registry.docstore.PublishedConfigSet
+import org.apache.slider.server.appmaster.web.HttpCacheHeaders
import org.apache.slider.server.services.workflow.ForkedProcessService
import org.junit.Assert
import org.junit.Assume
+import javax.ws.rs.core.HttpHeaders
import java.util.concurrent.TimeoutException
import static Arguments.ARG_OPTION
@@ -531,11 +533,9 @@ class SliderTestUtils extends Assert {
* @param page
* @return body of response
*/
- public static String getWebPage(Configuration conf,
- String base,
- String path) {
+ public static String getWebPage(String base, String path) {
String s = appendToURL(base, path)
- return getWebPage(conf, s)
+ return getWebPage(s)
}
/**
@@ -561,21 +561,33 @@ class SliderTestUtils extends Assert {
throw ex;
}
+ static URLConnectionFactory connectionFactory
+
+ public static def initConnectionFactory(Configuration conf) {
+ connectionFactory = URLConnectionFactory
+ .newDefaultURLConnectionFactory(conf);
+ }
+
+
/**
* Fetches a web page asserting that the response code is between 200 and 400.
* Will error on 400 and 500 series response codes and let 200 and 300 through.
*
* if security is enabled, this uses SPNEGO to auth
- * @param page
+ * <p>
+ * Relies on {@link #initConnectionFactory(org.apache.hadoop.conf.Configuration)}
+ * to have been called.
+ *
+ * @param path path to page
+ * @param connectionChecks optional closure to run against an open connection
* @return body of response
*/
- public static String getWebPage(Configuration conf, String page) {
- assert null != page
+ public static String getWebPage(String path, Closure connectionChecks = null) {
+ assert path
+ assert null != connectionFactory
- log.info("Fetching HTTP content at " + page);
- URLConnectionFactory connectionFactory = URLConnectionFactory
- .newDefaultURLConnectionFactory(conf);
- URL url = new URL(page)
+ log.info("Fetching HTTP content at " + path);
+ URL url = new URL(path)
assert url.port != 0
HttpURLConnection conn = null;
int resultCode = 0
@@ -584,8 +596,14 @@ class SliderTestUtils extends Assert {
conn = (HttpURLConnection) connectionFactory.openConnection(url);
conn.instanceFollowRedirects = true;
conn.connect()
+
resultCode = conn.responseCode
+
+ if (connectionChecks) {
+ connectionChecks(conn)
+ }
+
InputStream stream = conn.errorStream;
if (stream == null) {
stream = conn.inputStream;
@@ -597,11 +615,21 @@ class SliderTestUtils extends Assert {
} finally {
conn?.disconnect()
}
- uprateFaults(page, resultCode, body)
+ uprateFaults(path, resultCode, body)
return body;
}
-
-
+
+ /**
+ * Assert that a connection is not caching by looking at the headers
+ * @param conn connection to examine
+ */
+ public static void assertConnectionNotCaching(HttpURLConnection conn) {
+ assert conn.expiration <= conn.date
+ assert conn.getHeaderField(HttpHeaders.CACHE_CONTROL) ==
+ HttpCacheHeaders.HTTP_HEADER_CACHE_CONTROL_NONE
+ assert conn.getHeaderField(HttpCacheHeaders.HTTP_HEADER_PRAGMA) ==
+ HttpCacheHeaders.HTTP_HEADER_CACHE_CONTROL_NONE
+ }
/**
* Assert that a service operation succeeded
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/515ccba5/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
index f6b13a4..e907209 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
@@ -103,6 +103,7 @@ public abstract class YarnZKMiniClusterTestBase extends YarnMiniClusterTestBase
createMicroZKCluster("-${name?:methodName.methodName}", conf)
conf.setBoolean(RegistryConstants.KEY_REGISTRY_ENABLED, true)
conf.set(RegistryConstants.KEY_REGISTRY_ZK_QUORUM, ZKBinding)
+ //now create the cluster
name = super.createMiniCluster(name, conf, noOfNodeManagers, numLocalDirs, numLogDirs,
startHDFS)
[04/26] incubator-slider git commit: SLIDER-151 moving rest paths
slightly, making more constants,
and adding a shared base class AbstractSliderResource for all resources that
need at the slider web api
Posted by st...@apache.org.
SLIDER-151 moving rest paths slightly, making more constants, and adding a shared base class AbstractSliderResource for all resources that need at the slider web api
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/aced331b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/aced331b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/aced331b
Branch: refs/heads/develop
Commit: aced331b43e5f36d1c268111ae4daa6834b76378
Parents: db83a7c
Author: Steve Loughran <st...@apache.org>
Authored: Thu Jun 19 14:54:44 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Dec 16 20:25:39 2014 +0000
----------------------------------------------------------------------
.../web/rest/AMWadlGeneratorConfig.java | 4 +-
.../web/rest/AbstractSliderResource.java | 34 +++++++++++++
.../server/appmaster/web/rest/RestPaths.java | 3 +-
.../web/rest/SliderJacksonJaxbJsonProvider.java | 1 -
.../appmaster/web/rest/agent/AgentResource.java | 6 +--
.../web/rest/management/ManagementResource.java | 15 +++---
.../management/resources/ActionsResource.java | 22 +++++++++
.../resources/AggregateConfResource.java | 4 +-
.../management/resources/ConfTreeResource.java | 3 +-
.../web/rest/publisher/PublisherResource.java | 52 +++++++++++---------
10 files changed, 103 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aced331b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWadlGeneratorConfig.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWadlGeneratorConfig.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWadlGeneratorConfig.java
index 4ae7490..ea9f22b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWadlGeneratorConfig.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWadlGeneratorConfig.java
@@ -22,9 +22,9 @@ import com.sun.jersey.api.wadl.config.WadlGeneratorDescription;
import java.util.List;
/**
- *
+ * App master's WADL generation support
*/
-public class AMWadlGeneratorConfig extends WadlGeneratorConfig{
+public class AMWadlGeneratorConfig extends WadlGeneratorConfig {
public static final String CLASSNAME = "org.apache.slider.server.appmaster.web.rest.AMWadlGeneratorConfig";
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aced331b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
new file mode 100644
index 0000000..4f41815
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest;
+
+import org.apache.slider.server.appmaster.web.WebAppApi;
+
+/**
+ * Abstract resource base class for REST resources
+ * that use the slider WebAppApi
+ */
+public abstract class AbstractSliderResource {
+
+ protected final WebAppApi slider;
+
+ public AbstractSliderResource(WebAppApi slider) {
+ this.slider = slider;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aced331b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
index be18914..c3c81fc 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
@@ -29,7 +29,8 @@ public class RestPaths {
public static final String WS_AGENT_CONTEXT_ROOT = "/" + AGENT_WS_CONTEXT;
public static final String SLIDER_CONTEXT_ROOT = WS_CONTEXT_ROOT +"/v1/slider";
public static final String SLIDER_AGENT_CONTEXT_ROOT = WS_AGENT_CONTEXT_ROOT +"/v1/slider";
- public static final String SLIDER_SUBPATH_MANAGEMENT = "/mgmt";
+ public static final String MANAGEMENT = "mgmt";
+ public static final String SLIDER_SUBPATH_MANAGEMENT = "/" + MANAGEMENT;
public static final String SLIDER_SUBPATH_AGENTS = "/agents";
public static final String SLIDER_SUBPATH_PUBLISHER = "/publisher";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aced331b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/SliderJacksonJaxbJsonProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/SliderJacksonJaxbJsonProvider.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/SliderJacksonJaxbJsonProvider.java
index 95b0aa1..86d68a8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/SliderJacksonJaxbJsonProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/SliderJacksonJaxbJsonProvider.java
@@ -42,7 +42,6 @@ import javax.ws.rs.ext.Provider;
public class SliderJacksonJaxbJsonProvider extends JacksonJaxbJsonProvider {
public SliderJacksonJaxbJsonProvider() {
- super();
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aced331b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java
index 9d1e840..f1e105a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java
@@ -19,6 +19,7 @@ package org.apache.slider.server.appmaster.web.rest.agent;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.services.security.SignCertResponse;
import org.apache.slider.server.services.security.SignMessage;
+import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
@@ -39,13 +40,12 @@ import javax.ws.rs.core.Response;
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class AgentResource {
+public class AgentResource extends AbstractSliderResource {
- private final WebAppApi slider;
private String agent_name;
public AgentResource(WebAppApi slider) {
- this.slider = slider;
+ super(slider);
}
private void init(HttpServletResponse res) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aced331b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java
index 94db409..02199cf 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java
@@ -18,6 +18,7 @@ package org.apache.slider.server.appmaster.web.rest.management;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
import org.apache.slider.server.appmaster.web.rest.management.resources.AggregateConfResource;
import org.apache.slider.server.appmaster.web.rest.management.resources.ConfTreeResource;
@@ -36,18 +37,18 @@ import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriInfo;
+import java.net.URI;
import java.net.URL;
/**
*
*/
-public class ManagementResource {
+public class ManagementResource extends AbstractSliderResource {
protected static final Logger log =
LoggerFactory.getLogger(ManagementResource.class);
- private final WebAppApi slider;
public ManagementResource(WebAppApi slider) {
- this.slider = slider;
+ super(slider);
}
private void init(HttpServletResponse res) {
@@ -57,7 +58,7 @@ public class ManagementResource {
@GET
public Response getWadl (@Context HttpServletRequest request) {
try {
- java.net.URI location = new URL(request.getScheme(),
+ URI location = new URL(request.getScheme(),
request.getServerName(),
request.getServerPort(),
"/application.wadl").toURI();
@@ -87,9 +88,9 @@ public class ManagementResource {
init(res);
AggregateConfResource aggregateConf =
ResourceFactory.createAggregateConfResource(getAggregateConf(),
- uriInfo.getBaseUriBuilder()
- .path(RestPaths.SLIDER_CONTEXT_ROOT).path(
- "mgmt/app"));
+ uriInfo.getBaseUriBuilder()
+ .path(RestPaths.SLIDER_CONTEXT_ROOT)
+ .path(RestPaths.MANAGEMENT + "/app"));
return aggregateConf.getConfTree(config);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aced331b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ActionsResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ActionsResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ActionsResource.java
new file mode 100644
index 0000000..9b340fa
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ActionsResource.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.management.resources;
+
+public class ActionsResource {
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aced331b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
index 9df692a..dc26699 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
@@ -38,7 +38,7 @@ public class AggregateConfResource {
public AggregateConfResource(AggregateConf conf, UriBuilder uriBuilder) {
if (uriBuilder != null) {
this.href =
- uriBuilder.build(null).toASCIIString();
+ uriBuilder.build().toASCIIString();
resources =
ResourceFactory.createConfTreeResource(conf.getAppConf(),
uriBuilder.clone().path(
@@ -63,7 +63,7 @@ public class AggregateConfResource {
}
private void initConfMap() {
- confMap = new HashMap<String, ConfTreeResource>();
+ confMap = new HashMap<>();
confMap.put("internal", internal);
confMap.put("resources", resources);
confMap.put("appConf", appConf);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aced331b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ConfTreeResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ConfTreeResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ConfTreeResource.java
index 79f5399..407bab6 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ConfTreeResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ConfTreeResource.java
@@ -42,8 +42,7 @@ public class ConfTreeResource {
metadata = confTree.metadata;
global = confTree.global;
components = confTree.components;
- uriBuilder = uriBuilder.clone();
- this.href = uriBuilder.build(null).toASCIIString();
+ this.href = uriBuilder.build().toASCIIString();
} else {
this.href = null;
this.metadata = null;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aced331b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
index d2ad09e..df19687 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
@@ -28,6 +28,7 @@ import org.apache.slider.core.registry.docstore.PublishedExportsSet;
import org.apache.slider.core.registry.docstore.UriMap;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,21 +54,26 @@ import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
/**
* This publishes configuration sets
*/
-public class PublisherResource {
+public class PublisherResource extends AbstractSliderResource {
protected static final Logger log =
LoggerFactory.getLogger(PublisherResource.class);
- private final WebAppApi slider;
public static final String EXPORTS_NAME = "exports";
public static final String EXPORTS_RESOURCES_PATH = "/" + EXPORTS_NAME;
public static final String EXPORT_RESOURCE_PATH = EXPORTS_RESOURCES_PATH + "/{exportname}" ;
public static final String SET_NAME =
"{setname: " + PUBLISHED_CONFIGURATION_SET_REGEXP + "}";
- private static final String CONFIG =
- SET_NAME + "/{config: " + PUBLISHED_CONFIGURATION_REGEXP + "}";
+ public static final String SETNAME = "setname";
+ public static final String CLASSPATH = "/classpath";
+ public static final String CONFIG = "config";
+
+ public static final String SETNAME_PATTERN =
+ "{"+ SETNAME+": " + PUBLISHED_CONFIGURATION_SET_REGEXP + "}";
+ private static final String CONFIG_PATTERN =
+ SETNAME_PATTERN + "/{"+ CONFIG +": " + PUBLISHED_CONFIGURATION_REGEXP + "}";
private final StateAccessForProviders appState;
public PublisherResource(WebAppApi slider) {
- this.slider = slider;
+ super(slider);
appState = slider.getAppState();
}
@@ -112,7 +118,7 @@ public class PublisherResource {
}
@GET
- @Path("/classpath")
+ @Path(CLASSPATH)
@Produces({MediaType.APPLICATION_JSON})
public Set<URL> getAMClassPath() {
URL[] urls = ((URLClassLoader) getClass().getClassLoader()).getURLs();
@@ -140,10 +146,10 @@ public class PublisherResource {
}
@GET
- @Path("/"+ SET_NAME)
+ @Path("/"+ SETNAME_PATTERN)
@Produces({MediaType.APPLICATION_JSON})
public PublishedConfigSet getPublishedConfiguration(
- @PathParam("setname") String setname,
+ @PathParam(SETNAME) String setname,
@Context UriInfo uriInfo,
@Context HttpServletResponse res) {
init(res, uriInfo);
@@ -159,11 +165,11 @@ public class PublisherResource {
}
@GET
- @Path("/" + CONFIG)
+ @Path("/" + CONFIG_PATTERN)
@Produces({MediaType.APPLICATION_JSON})
public PublishedConfiguration getConfigurationInstance(
- @PathParam("setname") String setname,
- @PathParam("config") String config,
+ @PathParam(SETNAME) String setname,
+ @PathParam(CONFIG) String config,
@Context UriInfo uriInfo,
@Context HttpServletResponse res) {
init(res, uriInfo);
@@ -190,12 +196,12 @@ public class PublisherResource {
}
@GET
- @Path("/" + CONFIG + ".json")
+ @Path("/" + CONFIG_PATTERN + ".json")
@Produces({MediaType.APPLICATION_JSON})
public String getConfigurationContentJson(
- @PathParam("setname") String setname,
+ @PathParam(SETNAME) String setname,
- @PathParam("config") String config,
+ @PathParam(CONFIG) String config,
@Context UriInfo uriInfo,
@Context HttpServletResponse res) throws IOException {
return getStringRepresentation(setname, config, uriInfo, res,
@@ -203,11 +209,11 @@ public class PublisherResource {
}
@GET
- @Path("/" + CONFIG + ".xml")
+ @Path("/" + CONFIG_PATTERN + ".xml")
@Produces({MediaType.APPLICATION_XML})
public String getConfigurationContentXML(
- @PathParam("setname") String setname,
- @PathParam("config") String config,
+ @PathParam(SETNAME) String setname,
+ @PathParam(CONFIG) String config,
@Context UriInfo uriInfo,
@Context HttpServletResponse res) throws IOException {
return getStringRepresentation(setname, config, uriInfo, res,
@@ -215,12 +221,12 @@ public class PublisherResource {
}
@GET
- @Path("/" + CONFIG + ".properties")
+ @Path("/" + CONFIG_PATTERN + ".properties")
@Produces({MediaType.APPLICATION_XML})
public String getConfigurationContentProperties(
- @PathParam("setname") String setname,
+ @PathParam(SETNAME) String setname,
- @PathParam("config") String config,
+ @PathParam(CONFIG) String config,
@Context UriInfo uriInfo,
@Context HttpServletResponse res) throws IOException {
@@ -241,11 +247,11 @@ public class PublisherResource {
}
@GET
- @Path("/" + CONFIG +"/{propertyName}")
+ @Path("/" + CONFIG_PATTERN +"/{propertyName}")
@Produces({MediaType.APPLICATION_JSON})
public Map<String,String> getConfigurationProperty(
- @PathParam("setname") String setname,
- @PathParam("config") String config,
+ @PathParam(SETNAME) String setname,
+ @PathParam(CONFIG) String config,
@PathParam("propertyName") String propertyName,
@Context UriInfo uriInfo,
@Context HttpServletResponse res) {
[14/26] incubator-slider git commit: SLIDER-711 Implement initial
read-only view of REST resources. Core structure and live/resources.json
Posted by st...@apache.org.
SLIDER-711 Implement initial read-only view of REST resources. Core structure and live/resources.json
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fad1b978
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fad1b978
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fad1b978
Branch: refs/heads/develop
Commit: fad1b9784bc0576b6cdc187f29fb4ba7c0bde289
Parents: 5b81968
Author: Steve Loughran <st...@apache.org>
Authored: Tue Dec 16 13:31:06 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Dec 16 20:25:40 2014 +0000
----------------------------------------------------------------------
.../org/apache/slider/api/ResourceKeys.java | 2 +-
.../java/org/apache/slider/api/StatusKeys.java | 41 ++++++-
.../slider/core/conf/ConfTreeOperations.java | 3 +-
.../server/appmaster/web/SliderAMWebApp.java | 1 -
.../appmaster/web/rest/AMWebServices.java | 24 ++++-
.../web/rest/AbstractSliderResource.java | 31 +++++-
.../server/appmaster/web/rest/RestPaths.java | 19 ++++
.../rest/application/ApplicationResource.java | 108 +++++++++++++++++++
.../web/rest/application/package-info.java | 24 +++++
.../application/resources/CachedContent.java | 92 ++++++++++++++++
.../application/resources/ContentCache.java | 26 +++++
.../resources/LiveResourcesRefresher.java | 69 ++++++++++++
.../resources/ResourceRefresher.java | 31 ++++++
.../web/rest/management/ManagementResource.java | 46 +-------
.../web/rest/publisher/PublisherResource.java | 4 +-
.../TestMockContainerResourceAllocations.groovy | 8 +-
16 files changed, 468 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fad1b978/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
index f3eb963..52633f4 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
@@ -44,7 +44,7 @@ public interface ResourceKeys {
/**
- * #of instances of a component
+ * #of instances of a component: {@value}
*
*/
String COMPONENT_INSTANCES = "yarn.component.instances";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fad1b978/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java b/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java
index 4e46605..92cc8aa 100644
--- a/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java
@@ -17,7 +17,7 @@
*/
package org.apache.slider.api;
-
+import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES;
/**
* Contains status and statistics keys
*/
@@ -71,4 +71,43 @@ public interface StatusKeys {
String INFO_AM_AGENT_OPS_PORT = "info.am.agent.ops.port";
String INFO_AM_AGENT_OPS_URL = "info.am.agent.ops.url";
String INFO_AM_AGENT_STATUS_URL = "info.am.agent.status.url";
+
+ /**
+ * info: #of instances of a component requested: {@value}
+ *
+ */
+ String COMPONENT_INSTANCES_ACTUAL = COMPONENT_INSTANCES + ".actual";
+
+ /**
+ * info: #of instances of a component requested: {@value}
+ *
+ */
+ String COMPONENT_INSTANCES_REQUESTED = COMPONENT_INSTANCES + ".requested";
+
+ /**
+ * info: #of instances of a component being released: {@value}
+ *
+ */
+ String COMPONENT_INSTANCES_RELEASING = COMPONENT_INSTANCES + ".releasing";
+
+ /**
+ * info: #of instances of a component failed: {@value}
+ *
+ */
+ String COMPONENT_INSTANCES_FAILED = COMPONENT_INSTANCES + ".failed";
+
+ /**
+ * info: #of instances of a component started: {@value}
+ *
+ */
+ String COMPONENT_INSTANCES_STARTED = COMPONENT_INSTANCES + ".started";
+
+
+ /**
+ * info: #of instances of a component completed: {@value}
+ *
+ */
+ String COMPONENT_INSTANCES_COMPLETED = COMPONENT_INSTANCES + ".completed";
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fad1b978/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
index e946e43..dca58aa 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
@@ -429,9 +429,8 @@ public class ConfTreeOperations {
* @param option option name
* @param val integer value
*/
- public void setRoleOpt(String role, String option, int val) {
+ public void setComponentOpt(String role, String option, int val) {
setComponentOpt(role, option, Integer.toString(val));
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fad1b978/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
index f2fc903..84f0eba 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
@@ -16,7 +16,6 @@
*/
package org.apache.slider.server.appmaster.web;
-import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.servlets.HealthCheckServlet;
import com.codahale.metrics.servlets.MetricsServlet;
import com.codahale.metrics.servlets.PingServlet;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fad1b978/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
index 30db98e..03bf703 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
@@ -20,39 +20,53 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.rest.agent.AgentResource;
+import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource;
import org.apache.slider.server.appmaster.web.rest.management.ManagementResource;
import org.apache.slider.server.appmaster.web.rest.publisher.PublisherResource;
import org.apache.slider.server.appmaster.web.rest.registry.RegistryResource;
import javax.ws.rs.Path;
-/** The available REST services exposed by a slider AM. */
+/**
+ * The available REST services exposed by a slider AM.
+ */
@Singleton
@Path(RestPaths.SLIDER_CONTEXT_ROOT)
public class AMWebServices {
/** AM/WebApp info object */
private WebAppApi slider;
+ private final ManagementResource managementResource;
+ private final PublisherResource publisherResource;
+ private final RegistryResource registryResource;
+ private final ApplicationResource applicationResource;
@Inject
public AMWebServices(WebAppApi slider) {
this.slider = slider;
+ managementResource = new ManagementResource(slider);
+ publisherResource = new PublisherResource(slider);
+ registryResource = new RegistryResource(slider);
+ applicationResource = new ApplicationResource(slider);
}
@Path(RestPaths.SLIDER_SUBPATH_MANAGEMENT)
public ManagementResource getManagementResource() {
- return new ManagementResource(slider);
+ return managementResource;
}
@Path(RestPaths.SLIDER_SUBPATH_PUBLISHER)
public PublisherResource getPublisherResource() {
- return new PublisherResource(slider);
+ return publisherResource;
}
@Path(RestPaths.SLIDER_SUBPATH_REGISTRY)
public RegistryResource getRegistryResource() {
- return new RegistryResource(slider);
+ return registryResource;
}
-
+ @Path(RestPaths.SLIDER_SUBPATH_APPLICATION)
+ public ApplicationResource getApplicationResource() {
+ return applicationResource;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fad1b978/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
index 4f41815..90fe6fe 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
@@ -19,16 +19,45 @@
package org.apache.slider.server.appmaster.web.rest;
import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Response;
+import java.net.URI;
+import java.net.URL;
/**
* Abstract resource base class for REST resources
* that use the slider WebAppApi
*/
public abstract class AbstractSliderResource {
-
+ private static final Logger log =
+ LoggerFactory.getLogger(AbstractSliderResource.class);
protected final WebAppApi slider;
public AbstractSliderResource(WebAppApi slider) {
this.slider = slider;
}
+
+
+ /**
+ * Generate a redirect to the WASL
+ * @param request to base the URL on
+ * @return a 302 response
+ */
+ protected Response redirectToAppWadl(HttpServletRequest request) {
+ try {
+ URI location = new URL(request.getScheme(),
+ request.getServerName(),
+ request.getServerPort(),
+ RestPaths.APPLICATION_WADL).toURI();
+ return Response.temporaryRedirect(location).build();
+ } catch (Exception e) {
+ log.error("Error during redirect to WADL", e);
+ throw new WebApplicationException(Response.serverError().build());
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fad1b978/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
index 0df617f..a264f77 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
@@ -61,6 +61,9 @@ public class RestPaths {
public static final String SLIDER_PATH_PUBLISHER = SLIDER_CONTEXT_ROOT
+ SLIDER_SUBPATH_PUBLISHER;
+ /**
+ * Registry subpath: {@value}
+ */
public static final String SLIDER_SUBPATH_REGISTRY = "/registry";
/**
@@ -112,4 +115,20 @@ public class RestPaths {
* Codahale Metrics - thread dump: {@value}
*/
public static final String SYSTEM_THREADS = SYSTEM + "/threads";
+
+
+ /**
+ * application subpath
+ */
+ public static final String SLIDER_SUBPATH_APPLICATION = "/application";
+
+
+ /**
+ * management path: {@value}
+ */
+ public static final String SLIDER_PATH_APPLICATION =
+ SLIDER_CONTEXT_ROOT + SLIDER_SUBPATH_APPLICATION;
+
+
+ public static final String APPLICATION_WADL = "/application.wadl";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fad1b978/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
new file mode 100644
index 0000000..2d63269
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.application;
+
+import com.google.common.collect.Lists;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource;
+import org.apache.slider.server.appmaster.web.rest.application.resources.CachedContent;
+import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
+import org.apache.slider.server.appmaster.web.rest.application.resources.LiveResourcesRefresher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import java.util.List;
+
+public class ApplicationResource extends AbstractSliderResource {
+ private static final Logger log =
+ LoggerFactory.getLogger(ApplicationResource.class);
+ public static final String LIVE_RESOURCES_JSON = "/live/resources.json";
+
+ ContentCache cache = new ContentCache();
+ public static final int LIFESPAN = 1000;
+
+ public ApplicationResource(WebAppApi slider) {
+ super(slider);
+ cache.put(LIVE_RESOURCES_JSON,
+ new CachedContent<ConfTree>(LIFESPAN,
+ new LiveResourcesRefresher(slider.getAppState())));
+ }
+
+ /**
+ * Build a new JSON-marshallable list of string elements
+ * @param elements elements
+ * @return something that can be returned
+ */
+ private List<String> toJsonList(String... elements) {
+ return Lists.newArrayList(elements);
+ }
+
+ @GET
+ @Path("/")
+ @Produces({MediaType.APPLICATION_JSON})
+ public List<String> getRoot() {
+ return toJsonList("model", "live", "actions");
+ }
+
+ @GET
+ @Path("/model")
+ @Produces({MediaType.APPLICATION_JSON})
+ public List<String> getModel() {
+ return toJsonList("desired", "resolved");
+ }
+
+ @GET
+ @Path("/model/")
+ @Produces({MediaType.APPLICATION_JSON})
+ public List<String> getModelSlash() {
+ return getModel();
+ }
+
+ @GET
+ @Path("/live")
+ @Produces({MediaType.APPLICATION_JSON})
+ public List<String> getLive() {
+ return toJsonList("resources",
+ "containers",
+ "components",
+ "nodes",
+ "statistics",
+ "internal");
+ }
+
+ @GET
+ @Path("/live/")
+ @Produces({MediaType.APPLICATION_JSON})
+ public List<String> getLiveSlash() {
+ return getLive();
+ }
+
+ @GET
+ @Path(LIVE_RESOURCES_JSON)
+ @Produces({MediaType.APPLICATION_JSON})
+ public Object getLiveResources() {
+ return cache.get(LIVE_RESOURCES_JSON).get();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fad1b978/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/package-info.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/package-info.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/package-info.java
new file mode 100644
index 0000000..34eb82c
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ * This package contains resources related to the application
+ * REST API
+ */
+package org.apache.slider.server.appmaster.web.rest.application;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fad1b978/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java
new file mode 100644
index 0000000..c2edd61
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.application.resources;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Time;
+
+/**
+ * A cached resource is one that can be stored and served up, with a refresh
+ * only taking place when the expiry happens
+ * @param <T>
+ */
+public class CachedContent<T> {
+
+ private T cachedValue;
+ private long expires;
+ private final long lifespan;
+ private final ResourceRefresher<T> refresh;
+
+ public CachedContent(long lifespan,
+ ResourceRefresher<T> refresh) {
+ this.lifespan = lifespan;
+ this.refresh = refresh;
+ }
+
+ /**
+ * Get the value, triggering a refresh if needed
+ * @return the cached or latest value.
+ */
+ public T get() {
+ maybeRefresh();
+ return cachedValue;
+ }
+
+ /**
+ * Get the cached value without any expiry check
+ * @return the last value set. May be null.
+ */
+ public T getCachedValue() {
+ return cachedValue;
+ }
+
+ /**
+ * Get the lifespan in millis of the cached value
+ * @return the lifespan
+ */
+ public long getLifespan() {
+ return lifespan;
+ }
+
+ /**
+ * Maybe refresh the content
+ * @return true if a refresh took place.
+ */
+ public boolean maybeRefresh() {
+ long now = Time.monotonicNow();
+ if (now > expires) {
+ forceRefresh();
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Force a refresh and reset the expiry counter
+ * @return the new value
+ */
+ public T forceRefresh() {
+ T updated = refresh.refresh();
+ Preconditions.checkNotNull(updated);
+ cachedValue = updated;
+ expires = Time.monotonicNow() + lifespan;
+ return updated;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fad1b978/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java
new file mode 100644
index 0000000..0cc9eb4
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.application.resources;
+
+import java.util.HashMap;
+
+public class ContentCache extends HashMap<String, CachedContent> {
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fad1b978/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java
new file mode 100644
index 0000000..5c46a64
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.application.resources;
+
+import org.apache.slider.api.StatusKeys;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+
+import java.util.Map;
+
+public class LiveResourcesRefresher implements ResourceRefresher<ConfTree> {
+
+ private final StateAccessForProviders state;
+
+ public LiveResourcesRefresher(StateAccessForProviders state) {
+ this.state = state;
+ }
+
+ @Override
+ public ConfTree refresh() {
+ ConfTree resources = new ConfTree();
+ ConfTreeOperations ops = new ConfTreeOperations(resources);
+ // snapshot resources
+ state.getResourcesSnapshot();
+ // then add actual values
+ Map<Integer, RoleStatus> roleStatusMap = state.getRoleStatusMap();
+
+ for (RoleStatus status : roleStatusMap.values()) {
+ String name = status.getName();
+ ops.setComponentOpt(name,
+ StatusKeys.COMPONENT_INSTANCES_REQUESTED,
+ status.getRequested());
+ ops.setComponentOpt(name,
+ StatusKeys.COMPONENT_INSTANCES_ACTUAL,
+ status.getActual());
+ ops.setComponentOpt(name,
+ StatusKeys.COMPONENT_INSTANCES_RELEASING,
+ status.getReleasing());
+ ops.setComponentOpt(name,
+ StatusKeys.COMPONENT_INSTANCES_FAILED,
+ status.getFailed());
+ ops.setComponentOpt(name,
+ StatusKeys.COMPONENT_INSTANCES_COMPLETED,
+ status.getCompleted());
+ ops.setComponentOpt(name,
+ StatusKeys.COMPONENT_INSTANCES_STARTED,
+ status.getStarted());
+ }
+ return resources;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fad1b978/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceRefresher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceRefresher.java
new file mode 100644
index 0000000..9822fbc
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceRefresher.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.application.resources;
+
+/**
+ * Interface which must be implemented to act as a source for cached content.
+ * @param <T> type to return
+ */
+public interface ResourceRefresher<T> {
+ /**
+ * Build an up to date version of the data
+ * @return a new instance of the (JSON serializable) data
+ */
+ T refresh();
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fad1b978/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java
index 8a6b928..f27711a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java
@@ -29,7 +29,6 @@ import org.slf4j.LoggerFactory;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.GET;
-import javax.ws.rs.*;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
@@ -60,16 +59,7 @@ public class ManagementResource extends AbstractSliderResource {
@GET
public Response getWadl (@Context HttpServletRequest request) {
- try {
- URI location = new URL(request.getScheme(),
- request.getServerName(),
- request.getServerPort(),
- "/application.wadl").toURI();
- return Response.temporaryRedirect(location).build();
- } catch (Exception e) {
- log.error("Error during redirect to WADL", e);
- throw new WebApplicationException(Response.serverError().build());
- }
+ return redirectToAppWadl(request);
}
@GET
@@ -100,38 +90,4 @@ public class ManagementResource extends AbstractSliderResource {
protected AggregateConf getAggregateConf() {
return slider.getAppState().getInstanceDefinitionSnapshot();
}
-
- @POST
- @Path("actions/flex")
- public void actionFlex() {
- }
-
- @POST
- @Path("actions/stop")
- public void actionStop() {
- }
-
-
- @POST
- @Path("actions/test/suicide")
- public void actionSuicide() {
- }
-
- @POST
- @Path("actions/test/kill-container")
- public void actionKillContainer() {
- }
-
-
- @GET
- @Path("containers"+"/components" +"/{name}")
- public void actionListContainers() {
- }
-
- @GET
- @Path("containers"+"/components" +"/{name}")
- public void actionListContainersbyComponent() {
- }
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fad1b978/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
index df19687..431d50f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
@@ -32,6 +32,7 @@ import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
@@ -39,6 +40,7 @@ import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriInfo;
import java.io.IOException;
import java.net.URL;
@@ -81,7 +83,7 @@ public class PublisherResource extends AbstractSliderResource {
res.setContentType(null);
log.debug(uriInfo.getRequestUri().toString());
}
-
+
/**
* Get a named config set
* @param setname name of the config set
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fad1b978/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
index 73d40ee..4ba0afd 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
@@ -47,8 +47,8 @@ class TestMockContainerResourceAllocations extends BaseMockAppStateTest {
ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
ConfTreeOperations cto = new ConfTreeOperations(clusterSpec)
- cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY, 512)
- cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES, 2)
+ cto.setComponentOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY, 512)
+ cto.setComponentOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES, 2)
appState.updateResourceDefinitions(clusterSpec)
List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
assert ops.size() == 1
@@ -65,7 +65,7 @@ class TestMockContainerResourceAllocations extends BaseMockAppStateTest {
cto.setComponentOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY,
ResourceKeys.YARN_RESOURCE_MAX)
- cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES, 2)
+ cto.setComponentOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES, 2)
appState.updateResourceDefinitions(clusterSpec)
List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
assert ops.size() == 1
@@ -79,7 +79,7 @@ class TestMockContainerResourceAllocations extends BaseMockAppStateTest {
public void testMaxCoreAllocations() throws Throwable {
ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
ConfTreeOperations cto = new ConfTreeOperations(clusterSpec)
- cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY,
+ cto.setComponentOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY,
512)
cto.setComponentOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES,
ResourceKeys.YARN_RESOURCE_MAX)
[19/26] incubator-slider git commit: SLIDER-711 container
serialization/refresh testing
Posted by st...@apache.org.
SLIDER-711 container serialization/refresh testing
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8ad2bfbf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8ad2bfbf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8ad2bfbf
Branch: refs/heads/develop
Commit: 8ad2bfbfda1aff91efd630671acfdff6accfa46f
Parents: 7454e0b
Author: Steve Loughran <st...@apache.org>
Authored: Thu Dec 18 12:13:47 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Dec 18 12:13:47 2014 +0000
----------------------------------------------------------------------
.../types/SerializedContainerInformation.java | 56 ++++++++
.../server/appmaster/state/RoleInstance.java | 12 +-
.../server/appmaster/web/rest/RestPaths.java | 1 +
.../rest/application/ApplicationResource.java | 10 ++
.../application/resources/CachedContent.java | 45 +++++--
.../resources/ContainerListRefresher.java | 49 +++++++
.../application/resources/ContentCache.java | 7 +-
.../standalone/TestStandaloneAgentWeb.groovy | 2 +-
...estMockAppStateAppResourceIntegration.groovy | 72 ----------
.../TestMockAppStateAppRestIntegration.groovy | 132 +++++++++++++++++++
10 files changed, 298 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/main/java/org/apache/slider/api/types/SerializedContainerInformation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/SerializedContainerInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/SerializedContainerInformation.java
new file mode 100644
index 0000000..dfb70f5
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/types/SerializedContainerInformation.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.types;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+
+/**
+ * Serializable version of role instance data
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class SerializedContainerInformation {
+
+ public String containerId;
+ public String component;
+ public Boolean released;
+ public int state;
+ public Integer exitCode;
+ public String diagnostics;
+ public long createTime;
+ public long startTime;
+
+ /**
+ * What is the tail output from the executed process (or [] if not started
+ * or the log cannot be picked up
+ */
+ public String[] output;
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("SerializedContainerInformation{");
+ sb.append("containerId='").append(containerId).append('\'');
+ sb.append(", component='").append(component).append('\'');
+ sb.append('}');
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
index c8ddc6f..1488fb7 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ProtocolTypes;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.proto.Messages;
+import org.apache.slider.api.types.SerializedContainerInformation;
import org.apache.slider.common.tools.SliderUtils;
import java.util.ArrayList;
@@ -40,8 +41,7 @@ public final class RoleInstance implements Cloneable {
public Container container;
/**
- * UUID of container used in Slider RPC to refer to instances.
- * The string value of the container ID is used here.
+ * Container ID
*/
public final String id;
public long createTime;
@@ -233,5 +233,11 @@ public final class RoleInstance implements Cloneable {
ProtocolTypes.PROTOCOL_TCP, host, port);
addEndpoint(epr);
}
-
+
+ public SerializedContainerInformation serialize() {
+ SerializedContainerInformation info = new SerializedContainerInformation();
+ info.containerId = id;
+
+ return info;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
index 5dbc090..28c0fab 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
@@ -131,6 +131,7 @@ public class RestPaths {
public static final String APPLICATION_WADL = "/application.wadl";
public static final String LIVE_RESOURCES = "/live/resources";
+ public static final String LIVE_CONTAINERS = "/live/containers";
public static final String MODEL_DESIRED = "/model/desired";
public static final String MODEL_DESIRED_APPCONF = MODEL_DESIRED +"/appconf";
public static final String MODEL_DESIRED_RESOURCES = MODEL_DESIRED +"/resources";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
index b84717d..374e8de 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
@@ -96,5 +96,15 @@ public class ApplicationResource extends AbstractSliderResource {
throw buildException(RestPaths.LIVE_RESOURCES, e);
}
}
+ @GET
+ @Path(RestPaths.LIVE_CONTAINERS)
+ @Produces({MediaType.APPLICATION_JSON})
+ public Object getLiveContainers() {
+ try {
+ return cache.get(RestPaths.LIVE_RESOURCES).get();
+ } catch (Exception e) {
+ throw buildException(RestPaths.LIVE_RESOURCES, e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java
index c11a3ee..78e65e8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java
@@ -20,18 +20,24 @@ package org.apache.slider.server.appmaster.web.rest.application.resources;
import com.google.common.base.Preconditions;
import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A cached resource is one that can be stored and served up, with a refresh
- * only taking place when the expiry happens
- * @param <T>
+ * only taking place when the expiry happens.
+ *
+ * The refresh check/refresh is synchronized.
+ * @param <T> type to return
*/
public class CachedContent<T> {
-
+ private static final Logger log =
+ LoggerFactory.getLogger(CachedContent.class);
private T cachedValue;
private long expires;
private final long lifespan;
private final ResourceRefresher<T> refresh;
+ private int refreshCounter;
public CachedContent(long lifespan,
ResourceRefresher<T> refresh) {
@@ -45,17 +51,21 @@ public class CachedContent<T> {
*/
public T get() {
maybeRefresh();
- return cachedValue;
+ return getCachedValue();
}
/**
* Get the cached value without any expiry check
* @return the last value set. May be null.
*/
- public T getCachedValue() {
+ public synchronized T getCachedValue() {
return cachedValue;
}
+ public synchronized int getRefreshCounter() {
+ return refreshCounter;
+ }
+
/**
* Get the lifespan in millis of the cached value
* @return the lifespan
@@ -68,10 +78,12 @@ public class CachedContent<T> {
* Maybe refresh the content
* @return true if a refresh took place.
*/
- public boolean maybeRefresh() {
+ public synchronized boolean maybeRefresh() {
long now = now();
- if (cachedValue == null || now > expires) {
+ if (cachedValue == null || now >= expires) {
+ log.debug("Refreshing at time {}", now);
forceRefresh();
+ log.debug("Refreshed value now {}", cachedValue);
return true;
}
return false;
@@ -85,12 +97,25 @@ public class CachedContent<T> {
* Force a refresh and reset the expiry counter
* @return the new value
*/
- public T forceRefresh() {
+ protected synchronized T forceRefresh() {
+ refreshCounter ++;
T updated = refresh.refresh();
Preconditions.checkNotNull(updated);
cachedValue = updated;
expires = now() + lifespan;
- return updated;
+ return cachedValue;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("CachedContent{");
+ sb.append(" expires=").append(expires);
+ sb.append(", lifespan=").append(lifespan);
+ sb.append(", refresh=").append(refresh);
+ sb.append(", refreshCounter=").append(refreshCounter);
+ sb.append(", cached=").append(cachedValue);
+ sb.append('}');
+ return sb.toString();
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java
new file mode 100644
index 0000000..7a48509
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.application.resources;
+
+import org.apache.slider.api.types.SerializedContainerInformation;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ContainerListRefresher implements ResourceRefresher<Map<String, SerializedContainerInformation>> {
+
+ private final StateAccessForProviders state;
+
+ public ContainerListRefresher(StateAccessForProviders state) {
+ this.state = state;
+ }
+
+ @Override
+ public Map<String, SerializedContainerInformation> refresh() {
+ List<RoleInstance> containerList = state.cloneOwnedContainerList();
+
+ Map<String, SerializedContainerInformation> map = new HashMap<String, SerializedContainerInformation>();
+ for (RoleInstance instance : containerList) {
+ SerializedContainerInformation serialized = instance.serialize();
+ map.put(serialized.containerId, serialized);
+ }
+ return map;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java
index 169eaa3..f309570 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java
@@ -18,9 +18,12 @@
package org.apache.slider.server.appmaster.web.rest.application.resources;
-import java.util.HashMap;
+import java.util.concurrent.ConcurrentHashMap;
-public class ContentCache extends HashMap<String, CachedContent> {
+/**
+ * Cache of content
+ */
+public class ContentCache extends ConcurrentHashMap<String, CachedContent> {
public ContentCache(int initialCapacity) {
super(initialCapacity);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
index 7f8ddf5..cbb340f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
@@ -91,7 +91,7 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
log.info getWebPage(realappmaster, SYSTEM_METRICS_JSON)
- // get the root page, including some checks for connectivity
+ // get the root page, including some checks for cache disabled
getWebPage(appmaster, {
HttpURLConnection conn ->
assertConnectionNotCaching(conn)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppResourceIntegration.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppResourceIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppResourceIntegration.groovy
deleted file mode 100644
index 2d0b8bf..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppResourceIntegration.groovy
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate
-
-import groovy.util.logging.Slf4j
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.web.rest.application.resources.CachedContent
-import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache
-import org.apache.slider.server.appmaster.web.rest.application.resources.ResourceRefresher
-import org.junit.Test
-
-@Slf4j
-class TestMockAppStateAppResourceIntegration extends BaseMockAppStateTest implements MockRoles {
-
- @Override
- String getTestName() {
- return "TestMockAppStateAppResourceIntegration"
- }
-
- @Test
- public void testCachedDocument() throws Throwable {
- ContentCache cache = new ContentCache()
-
- def content = new CachedContentManagedTimer(new IntRefresher())
- cache.put("/int", content)
- def content1 = cache.get("/int")
- assert content.equals(content1)
-
- assert 0 == content.get()
- assert 0 == content.getCachedValue()
-
- }
-
- class IntRefresher implements ResourceRefresher<Integer> {
- int count ;
- @Override
- Integer refresh() {
- return count++;
- }
- }
-
- class CachedContentManagedTimer extends CachedContent {
- int time = 0;
-
- @Override
- protected long now() {
- return time++;
- }
-
- CachedContentManagedTimer(ResourceRefresher refresh) {
- super(1, refresh)
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
new file mode 100644
index 0000000..aefeecf
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.util.logging.Slf4j
+import org.apache.slider.api.types.SerializedContainerInformation
+import org.apache.slider.core.persist.JsonSerDeser
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.state.ProviderAppState
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.apache.slider.server.appmaster.state.StateAccessForProviders
+import org.apache.slider.server.appmaster.web.rest.application.resources.CachedContent
+import org.apache.slider.server.appmaster.web.rest.application.resources.ContainerListRefresher
+import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache
+import org.apache.slider.server.appmaster.web.rest.application.resources.ResourceRefresher
+import org.junit.Test
+
+@Slf4j
+class TestMockAppStateAppRestIntegration extends BaseMockAppStateTest implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestMockAppStateAppRestIntegration"
+ }
+
+ @Test
+ public void testCachedIntDocument() throws Throwable {
+ ContentCache cache = new ContentCache()
+
+
+ def refresher = new IntRefresher()
+ assert 0 == refresher.count
+ def entry = new CachedContentManagedTimer(refresher)
+ cache.put("/int", entry)
+ def content1 = cache.get("/int")
+ assert entry.equals(content1)
+
+ assert 0 == entry.get()
+ assert 1 == refresher.count
+ assert 0 == entry.cachedValue
+ assert entry.refreshCounter == 1
+
+ def got = entry.get()
+ assert entry.refreshCounter == 2
+ assert 1 == got;
+ }
+
+ @Test
+ public void testContainerListRefresher() throws Throwable {
+ int r0 = 1
+ int r1 = 2
+ int r2 = 3
+ role0Status.desired = r0
+ role1Status.desired = r1
+ role2Status.desired = r2
+ ContainerListRefresher clr = new ContainerListRefresher(stateAccess)
+ def map = clr.refresh()
+ assert map.size() == 0
+ List<RoleInstance> instances = createAndStartNodes()
+ map = clr.refresh()
+ assert instances.size() == r0 + r1 + r2
+ assert map.size() == instances.size()
+ log.info("$map")
+ JsonSerDeser<SerializedContainerInformation> serDeser =
+ new JsonSerDeser<>(SerializedContainerInformation)
+ map.each { key, value ->
+ log.info("$key -> ${serDeser.toJson(value)}")
+ }
+ }
+
+ public ProviderAppState getStateAccess() {
+ StateAccessForProviders state = new ProviderAppState("name", appState)
+ return state
+ }
+
+ /**
+ * Little class to do integer refreshing & so test refresh logic
+ */
+ class IntRefresher implements ResourceRefresher<Integer> {
+ int count ;
+ @Override
+ Integer refresh() {
+ log.info("Refresh at $count")
+ def result = count
+ count += 1;
+ return result;
+ }
+
+ @Override
+ String toString() {
+ return "IntRefresher at " + count;
+ }
+
+ }
+
+ class CachedContentManagedTimer extends CachedContent {
+ int time = 0;
+
+ @Override
+ protected long now() {
+ return time++;
+ }
+
+ CachedContentManagedTimer(ResourceRefresher refresh) {
+ super(1, refresh)
+ }
+
+ @Override
+ String toString() {
+ return "CachedContentManagedTimer at " + time + "; " + super.toString();
+ }
+ }
+
+
+}
[10/26] incubator-slider git commit: SLIDER-711 Implement initial
read-only view of REST resources.
Posted by st...@apache.org.
SLIDER-711 Implement initial read-only view of REST resources.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/47ca9e65
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/47ca9e65
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/47ca9e65
Branch: refs/heads/develop
Commit: 47ca9e656b186d054b2f2ef2506f6bfd4b564d0d
Parents: fad1b97
Author: Steve Loughran <st...@apache.org>
Authored: Tue Dec 16 20:23:03 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Dec 16 20:25:40 2014 +0000
----------------------------------------------------------------------
.../server/appmaster/web/rest/RestPaths.java | 8 ++++
.../rest/application/ApplicationResource.java | 10 ++---
.../resources/LiveResourcesRefresher.java | 19 +++++-----
.../resources/ResourceSnapshotRefresher.java | 40 ++++++++++++++++++++
4 files changed, 62 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47ca9e65/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
index a264f77..bd569ef 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
@@ -131,4 +131,12 @@ public class RestPaths {
public static final String APPLICATION_WADL = "/application.wadl";
+ public static final String LIVE_RESOURCES = "/live/resources";
+ public static final String MODEL_DESIRED = "/model/desired";
+ public static final String MODEL_DESIRED_APPCONF = MODEL_DESIRED +"/appconf";
+ public static final String MODEL_DESIRED_RESOURCES = MODEL_DESIRED +"/resources";
+ public static final String MODEL_RESOLVED = "/model/desired";
+ public static final String MODEL_RESOLVED_APPCONF = MODEL_RESOLVED +"/appconf";
+ public static final String MODEL_RESOLVED_RESOURCES = MODEL_RESOLVED +"/resources";
+ public static final String MODEL_INTERNAL = "/model/internal";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47ca9e65/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
index 2d63269..2477464 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
@@ -22,6 +22,7 @@ import com.google.common.collect.Lists;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource;
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
import org.apache.slider.server.appmaster.web.rest.application.resources.CachedContent;
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
import org.apache.slider.server.appmaster.web.rest.application.resources.LiveResourcesRefresher;
@@ -37,14 +38,13 @@ import java.util.List;
public class ApplicationResource extends AbstractSliderResource {
private static final Logger log =
LoggerFactory.getLogger(ApplicationResource.class);
- public static final String LIVE_RESOURCES_JSON = "/live/resources.json";
- ContentCache cache = new ContentCache();
public static final int LIFESPAN = 1000;
+ private final ContentCache cache = new ContentCache();
public ApplicationResource(WebAppApi slider) {
super(slider);
- cache.put(LIVE_RESOURCES_JSON,
+ cache.put(RestPaths.LIVE_RESOURCES,
new CachedContent<ConfTree>(LIFESPAN,
new LiveResourcesRefresher(slider.getAppState())));
}
@@ -99,10 +99,10 @@ public class ApplicationResource extends AbstractSliderResource {
}
@GET
- @Path(LIVE_RESOURCES_JSON)
+ @Path(RestPaths.LIVE_RESOURCES)
@Produces({MediaType.APPLICATION_JSON})
public Object getLiveResources() {
- return cache.get(LIVE_RESOURCES_JSON).get();
+ return cache.get(RestPaths.LIVE_RESOURCES).get();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47ca9e65/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java
index 5c46a64..fd52528 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java
@@ -36,34 +36,33 @@ public class LiveResourcesRefresher implements ResourceRefresher<ConfTree> {
@Override
public ConfTree refresh() {
- ConfTree resources = new ConfTree();
- ConfTreeOperations ops = new ConfTreeOperations(resources);
+
// snapshot resources
- state.getResourcesSnapshot();
+ ConfTreeOperations resources = state.getResourcesSnapshot();
// then add actual values
Map<Integer, RoleStatus> roleStatusMap = state.getRoleStatusMap();
for (RoleStatus status : roleStatusMap.values()) {
String name = status.getName();
- ops.setComponentOpt(name,
+ resources.setComponentOpt(name,
StatusKeys.COMPONENT_INSTANCES_REQUESTED,
status.getRequested());
- ops.setComponentOpt(name,
+ resources.setComponentOpt(name,
StatusKeys.COMPONENT_INSTANCES_ACTUAL,
status.getActual());
- ops.setComponentOpt(name,
+ resources.setComponentOpt(name,
StatusKeys.COMPONENT_INSTANCES_RELEASING,
status.getReleasing());
- ops.setComponentOpt(name,
+ resources.setComponentOpt(name,
StatusKeys.COMPONENT_INSTANCES_FAILED,
status.getFailed());
- ops.setComponentOpt(name,
+ resources.setComponentOpt(name,
StatusKeys.COMPONENT_INSTANCES_COMPLETED,
status.getCompleted());
- ops.setComponentOpt(name,
+ resources.setComponentOpt(name,
StatusKeys.COMPONENT_INSTANCES_STARTED,
status.getStarted());
}
- return resources;
+ return resources.getConfTree();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47ca9e65/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceSnapshotRefresher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceSnapshotRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceSnapshotRefresher.java
new file mode 100644
index 0000000..0034d57
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceSnapshotRefresher.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.application.resources;
+
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+
+public class ResourceSnapshotRefresher implements ResourceRefresher<ConfTree> {
+
+ private final StateAccessForProviders state;
+
+ public ResourceSnapshotRefresher(StateAccessForProviders state) {
+ this.state = state;
+ }
+
+ @Override
+ public ConfTree refresh() {
+
+ // snapshot resources
+ ConfTreeOperations resources = state.getResourcesSnapshot();
+ return resources.getConfTree();
+ }
+}
[18/26] incubator-slider git commit: SLIDER-689 AgentWebPagesIT fails
to run against a kerberos cluster
Posted by st...@apache.org.
SLIDER-689 AgentWebPagesIT fails to run against a kerberos cluster
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7454e0b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7454e0b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7454e0b1
Branch: refs/heads/develop
Commit: 7454e0b145f4a5e852e91cacf1df3411d2c42169
Parents: 515ccba
Author: Steve Loughran <st...@apache.org>
Authored: Wed Dec 17 18:01:40 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Dec 17 18:01:40 2014 +0000
----------------------------------------------------------------------
.../funtest/framework/CommandTestBase.groovy | 8 ++++----
.../funtest/lifecycle/AgentWebPagesIT.groovy | 21 ++++++++------------
2 files changed, 12 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7454e0b1/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 143d284..00b4062 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -1280,6 +1280,7 @@ abstract class CommandTestBase extends SliderTestUtils {
*/
protected static Outcome isRootWebPageUp(
Map<String, String> args) {
+
assert args['applicationId'] != null
String applicationId = args['applicationId'];
def sar = lookupApplication(applicationId)
@@ -1288,7 +1289,7 @@ abstract class CommandTestBase extends SliderTestUtils {
return Outcome.Retry;
}
try {
- GET(sar.url)
+ getWebPage(sar.url)
return Outcome.Success
} catch (Exception e) {
return Outcome.Retry;
@@ -1301,8 +1302,7 @@ abstract class CommandTestBase extends SliderTestUtils {
* @param launch_timeout launch timeout
*/
void expectRootWebPageUp(
- String applicationId,
- int launch_timeout) {
+ String applicationId, int launch_timeout) {
repeatUntilSuccess(
"await root web page",
@@ -1319,7 +1319,7 @@ abstract class CommandTestBase extends SliderTestUtils {
assert sar != null;
assert sar.url
// this is the final failure cause
- GET(sar.url)
+ getWebPage(sar.url)
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7454e0b1/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
index d29fe63..f5c35f3 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
@@ -20,25 +20,17 @@ package org.apache.slider.funtest.lifecycle
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.registry.client.binding.RegistryUtils
-import org.apache.hadoop.registry.client.types.Endpoint
-import org.apache.hadoop.registry.client.types.ServiceRecord
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
import org.apache.slider.server.appmaster.web.rest.RestPaths
-import org.apache.slider.test.Outcome
import org.junit.After
import org.junit.Before
import org.junit.Test
-import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
-
@CompileStatic
@Slf4j
public class AgentWebPagesIT extends AgentCommandTestBase
@@ -73,6 +65,10 @@ public class AgentWebPagesIT extends AgentCommandTestBase
logShell(shell)
+ def conf = SLIDER_CONFIG
+
+ initConnectionFactory(conf)
+
def appId = ensureYarnApplicationIsUp(launchReportFile)
assert appId
expectRootWebPageUp(appId, instanceLaunchTime)
@@ -82,12 +78,11 @@ public class AgentWebPagesIT extends AgentCommandTestBase
def report = loadAppReport(liveReportFile)
assert report.url
- def conf = SLIDER_CONFIG
def root = report.url
- log.info getWebPage (conf, root, RestPaths.SYSTEM_METRICS)
- log.info getWebPage(conf, root, RestPaths.SYSTEM_THREADS)
- log.info getWebPage(conf, root, RestPaths.SYSTEM_HEALTHCHECK)
- log.info getWebPage(conf, root, RestPaths.SYSTEM_PING)
+ log.info getWebPage(root, RestPaths.SYSTEM_METRICS)
+ log.info getWebPage(root, RestPaths.SYSTEM_THREADS)
+ log.info getWebPage(root, RestPaths.SYSTEM_HEALTHCHECK)
+ log.info getWebPage(root, RestPaths.SYSTEM_PING)
}
}
[16/26] incubator-slider git commit: SLIDER-711 live/resources fetch
working, with test
Posted by st...@apache.org.
SLIDER-711 live/resources fetch working, with test
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ca4686de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ca4686de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ca4686de
Branch: refs/heads/develop
Commit: ca4686de3c21ee0565c2fbab58943838e7654c97
Parents: 8e18403
Author: Steve Loughran <st...@apache.org>
Authored: Wed Dec 17 13:03:21 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Dec 17 13:03:21 2014 +0000
----------------------------------------------------------------------
.../java/org/apache/slider/api/StatusKeys.java | 2 +-
.../org/apache/slider/client/SliderClient.java | 3 +-
.../org/apache/slider/common/SliderKeys.java | 6 +-
.../apache/slider/core/build/InstanceIO.java | 14 ++--
.../server/appmaster/SliderAppMaster.java | 1 +
.../web/rest/AbstractSliderResource.java | 30 ++++++++
.../server/appmaster/web/rest/RestPaths.java | 3 +-
.../rest/application/ApplicationResource.java | 22 ++----
.../application/resources/CachedContent.java | 10 ++-
.../application/resources/ContentCache.java | 9 ++-
.../resources/LiveResourcesRefresher.java | 2 +-
.../web/rest/registry/RegistryResource.java | 18 ++---
.../standalone/TestStandaloneAgentWeb.groovy | 46 +++++++++++--
...estMockAppStateAppResourceIntegration.groovy | 72 ++++++++++++++++++++
14 files changed, 185 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ca4686de/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java b/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java
index 92cc8aa..ef68aad 100644
--- a/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java
@@ -82,7 +82,7 @@ public interface StatusKeys {
* info: #of instances of a component requested: {@value}
*
*/
- String COMPONENT_INSTANCES_REQUESTED = COMPONENT_INSTANCES + ".requested";
+ String COMPONENT_INSTANCES_REQUESTING = COMPONENT_INSTANCES + ".requesting";
/**
* info: #of instances of a component being released: {@value}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ca4686de/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 813ae0f..f9aae6f 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2296,7 +2296,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
int exitCode = EXIT_FALSE;
// save the specification
try {
- InstanceIO.updateInstanceDefinition(sliderFileSystem, clusterDirectory, instanceDefinition);
+ InstanceIO.saveInstanceDefinition(sliderFileSystem, clusterDirectory,
+ instanceDefinition);
} catch (LockAcquireFailedException e) {
// lock failure
log.debug("Failed to lock dir {}", clusterDirectory, e);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ca4686de/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index b9dc8e1..701ef14 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -24,11 +24,13 @@ package org.apache.slider.common;
*/
public interface SliderKeys extends SliderXmlConfKeys {
-
+ /**
+ * This is the name of the slider appmaster in configurations :{@value}
+ */
String COMPONENT_AM = "slider-appmaster";
/**
- * Slider role is "special"
+ * Slider role is "special":{@value}
*/
int ROLE_AM_PRIORITY_INDEX = 0;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ca4686de/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java
index 25bb4ab..3a8f805 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java
@@ -64,17 +64,17 @@ public class InstanceIO {
/**
- * Update an instance definition
- * @param coreFS
- * @param dir
- * @param instanceDefinition
+ * Update a persisted instance definition
+ * @param coreFS filesystem
+ * @param dir directory to load from
+ * @param instanceDefinition instance definition to save do
* @throws SliderException
* @throws IOException
* @throws LockAcquireFailedException
*/
- public static void updateInstanceDefinition(CoreFileSystem coreFS,
- Path dir,
- AggregateConf instanceDefinition)
+ public static void saveInstanceDefinition(CoreFileSystem coreFS,
+ Path dir,
+ AggregateConf instanceDefinition)
throws SliderException, IOException, LockAcquireFailedException {
ConfPersister persister =
new ConfPersister(coreFS, dir);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ca4686de/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 67e050a..c8764c9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -933,6 +933,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
RestPaths.WS_CONTEXT)
.withHttpPolicy(serviceConf, HttpConfig.Policy.HTTP_ONLY)
.at(port)
+ .inDevMode()
.start(webApp);
WebAppService<SliderAMWebApp> webAppService =
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ca4686de/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
index 90fe6fe..ede46f0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
@@ -18,6 +18,11 @@
package org.apache.slider.server.appmaster.web.rest;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.exceptions.AuthenticationFailedException;
+import org.apache.hadoop.registry.client.exceptions.NoPathPermissionsException;
+import org.apache.hadoop.yarn.webapp.ForbiddenException;
+import org.apache.hadoop.yarn.webapp.NotFoundException;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,4 +65,29 @@ public abstract class AbstractSliderResource {
}
}
+ /**
+ * Convert any exception caught into a web application
+ * exception for rethrowing
+ * @param path path of request
+ * @param ex exception
+ * @return an exception to throw
+ */
+ public WebApplicationException buildException(String path,
+ Exception ex) {
+ try {
+ throw ex;
+ } catch (WebApplicationException e) {
+ // rethrow direct
+ throw e;
+ } catch (PathNotFoundException e) {
+ return new NotFoundException("Not found: " + path);
+ } catch (AuthenticationFailedException e) {
+ return new ForbiddenException(path);
+ } catch (NoPathPermissionsException e) {
+ return new ForbiddenException(path);
+ } catch (Exception e) {
+ log.error("Error during generation of response: {}", e, e);
+ return new WebApplicationException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ca4686de/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
index bd569ef..5dbc090 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
@@ -121,8 +121,7 @@ public class RestPaths {
* application subpath
*/
public static final String SLIDER_SUBPATH_APPLICATION = "/application";
-
-
+
/**
* management path: {@value}
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ca4686de/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
index 2477464..b84717d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
@@ -29,12 +29,14 @@ import org.apache.slider.server.appmaster.web.rest.application.resources.LiveRes
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.inject.Singleton;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import java.util.List;
+@Singleton
public class ApplicationResource extends AbstractSliderResource {
private static final Logger log =
LoggerFactory.getLogger(ApplicationResource.class);
@@ -73,13 +75,6 @@ public class ApplicationResource extends AbstractSliderResource {
}
@GET
- @Path("/model/")
- @Produces({MediaType.APPLICATION_JSON})
- public List<String> getModelSlash() {
- return getModel();
- }
-
- @GET
@Path("/live")
@Produces({MediaType.APPLICATION_JSON})
public List<String> getLive() {
@@ -92,17 +87,14 @@ public class ApplicationResource extends AbstractSliderResource {
}
@GET
- @Path("/live/")
- @Produces({MediaType.APPLICATION_JSON})
- public List<String> getLiveSlash() {
- return getLive();
- }
-
- @GET
@Path(RestPaths.LIVE_RESOURCES)
@Produces({MediaType.APPLICATION_JSON})
public Object getLiveResources() {
- return cache.get(RestPaths.LIVE_RESOURCES).get();
+ try {
+ return cache.get(RestPaths.LIVE_RESOURCES).get();
+ } catch (Exception e) {
+ throw buildException(RestPaths.LIVE_RESOURCES, e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ca4686de/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java
index c2edd61..c11a3ee 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java
@@ -69,14 +69,18 @@ public class CachedContent<T> {
* @return true if a refresh took place.
*/
public boolean maybeRefresh() {
- long now = Time.monotonicNow();
- if (now > expires) {
+ long now = now();
+ if (cachedValue == null || now > expires) {
forceRefresh();
return true;
}
return false;
}
+ protected long now() {
+ return Time.monotonicNow();
+ }
+
/**
* Force a refresh and reset the expiry counter
* @return the new value
@@ -85,7 +89,7 @@ public class CachedContent<T> {
T updated = refresh.refresh();
Preconditions.checkNotNull(updated);
cachedValue = updated;
- expires = Time.monotonicNow() + lifespan;
+ expires = now() + lifespan;
return updated;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ca4686de/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java
index 0cc9eb4..169eaa3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java
@@ -21,6 +21,11 @@ package org.apache.slider.server.appmaster.web.rest.application.resources;
import java.util.HashMap;
public class ContentCache extends HashMap<String, CachedContent> {
-
-
+
+ public ContentCache(int initialCapacity) {
+ super(initialCapacity);
+ }
+
+ public ContentCache() {
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ca4686de/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java
index fd52528..dd845d8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java
@@ -45,7 +45,7 @@ public class LiveResourcesRefresher implements ResourceRefresher<ConfTree> {
for (RoleStatus status : roleStatusMap.values()) {
String name = status.getName();
resources.setComponentOpt(name,
- StatusKeys.COMPONENT_INSTANCES_REQUESTED,
+ StatusKeys.COMPONENT_INSTANCES_REQUESTING,
status.getRequested());
resources.setComponentOpt(name,
StatusKeys.COMPONENT_INSTANCES_ACTUAL,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ca4686de/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
index 70c0826..c824848 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.registry.client.exceptions.NoRecordException;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,7 +54,7 @@ import java.io.IOException;
*
*/
@Singleton
-public class RegistryResource {
+public class RegistryResource extends AbstractSliderResource {
protected static final Logger log =
LoggerFactory.getLogger(RegistryResource.class);
public static final String SERVICE_PATH =
@@ -66,9 +67,11 @@ public class RegistryResource {
* @param slider slider API
*/
public RegistryResource(WebAppApi slider) {
+ super(slider);
this.registry = slider.getRegistryOperations();
}
+
/**
* Internal init code, per request
* @param request incoming request
@@ -114,18 +117,9 @@ public class RegistryResource {
log.debug("Resolved:\n{}", pathEntry);
}
return pathEntry;
- } catch (WebApplicationException e) {
- // rethrow direct
- throw e;
- } catch (PathNotFoundException e) {
- throw new NotFoundException("Not found: " + path);
- } catch (AuthenticationFailedException e) {
- throw new ForbiddenException(path);
- } catch (NoPathPermissionsException e) {
- throw new ForbiddenException(path);
+
} catch (Exception e) {
- log.error("Error during generation of response: {}", e, e);
- throw new WebApplicationException(e);
+ throw buildException(path, e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ca4686de/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
index 577ad73..d524698 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
@@ -21,10 +21,16 @@ package org.apache.slider.agent.standalone
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.agent.AgentMiniClusterTestBase
+import static org.apache.slider.api.ResourceKeys.*
+import static org.apache.slider.api.StatusKeys.*
import org.apache.slider.client.SliderClient
+import static org.apache.slider.common.SliderKeys.*;
+import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.server.appmaster.web.rest.RestPaths
+import org.apache.slider.core.persist.ConfTreeSerDeser
+
import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
import org.junit.Test
@@ -33,8 +39,7 @@ import static org.apache.slider.server.appmaster.management.MetricsKeys.*
@CompileStatic
@Slf4j
class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
-
-
+
@Test
public void testStandaloneAgentWeb() throws Throwable {
@@ -55,9 +60,11 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
def realappmaster = report.originalTrackingUrl
execHttpRequest(30000) {
GET(realappmaster)
- }
- def metrics = GET(realappmaster, SYSTEM_METRICS)
- log.info metrics
+ }
+ execHttpRequest(30000) {
+ def metrics = GET(realappmaster, SYSTEM_METRICS)
+ log.info metrics
+ }
sleep(5000)
def appmaster = report.trackingUrl
@@ -82,9 +89,34 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
// now some REST gets
describe "Application REST GETs"
- getWebPage(conf, appmaster, SLIDER_PATH_APPLICATION + LIVE_RESOURCES)
+ ConfTreeOperations tree = fetchConfigTree(conf, appmaster, LIVE_RESOURCES)
+ log.info tree.toString()
+ def liveAM = tree.getComponent(COMPONENT_AM)
+ def desiredInstances = liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES);
+ assert desiredInstances == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_ACTUAL)
+
+ assert 1 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_STARTED)
+ assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_REQUESTING)
+ assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_FAILED)
+ assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_COMPLETED)
+ assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_RELEASING)
+
+ }
+ public ConfTreeOperations fetchConfigTree(
+ YarnConfiguration conf,
+ String appmaster,
+ String subpath) {
+ ConfTreeSerDeser serDeser = new ConfTreeSerDeser()
+
+ def json = getWebPage(
+ conf,
+ appmaster,
+ SLIDER_PATH_APPLICATION + subpath)
+ def ctree = serDeser.fromJson(json)
+ ConfTreeOperations tree = new ConfTreeOperations(ctree)
+ return tree
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ca4686de/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppResourceIntegration.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppResourceIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppResourceIntegration.groovy
new file mode 100644
index 0000000..2d0b8bf
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppResourceIntegration.groovy
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.util.logging.Slf4j
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.web.rest.application.resources.CachedContent
+import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache
+import org.apache.slider.server.appmaster.web.rest.application.resources.ResourceRefresher
+import org.junit.Test
+
+@Slf4j
+class TestMockAppStateAppResourceIntegration extends BaseMockAppStateTest implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestMockAppStateAppResourceIntegration"
+ }
+
+ @Test
+ public void testCachedDocument() throws Throwable {
+ ContentCache cache = new ContentCache()
+
+ def content = new CachedContentManagedTimer(new IntRefresher())
+ cache.put("/int", content)
+ def content1 = cache.get("/int")
+ assert content.equals(content1)
+
+ assert 0 == content.get()
+ assert 0 == content.getCachedValue()
+
+ }
+
+ class IntRefresher implements ResourceRefresher<Integer> {
+ int count ;
+ @Override
+ Integer refresh() {
+ return count++;
+ }
+ }
+
+ class CachedContentManagedTimer extends CachedContent {
+ int time = 0;
+
+ @Override
+ protected long now() {
+ return time++;
+ }
+
+ CachedContentManagedTimer(ResourceRefresher refresh) {
+ super(1, refresh)
+ }
+
+ }
+}
[08/26] incubator-slider git commit: SLIDER-151 java 6 compilation
Posted by st...@apache.org.
SLIDER-151 java 6 compilation
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a3420660
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a3420660
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a3420660
Branch: refs/heads/develop
Commit: a3420660e03b30174182bef02df4ff43f850e9b3
Parents: 7d52f8a
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 22 10:41:59 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Dec 16 20:25:39 2014 +0000
----------------------------------------------------------------------
.../web/rest/management/resources/AggregateConfResource.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3420660/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
index dc26699..75d417b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
@@ -63,7 +63,7 @@ public class AggregateConfResource {
}
private void initConfMap() {
- confMap = new HashMap<>();
+ confMap = new HashMap<String, ConfTreeResource>();
confMap.put("internal", internal);
confMap.put("resources", resources);
confMap.put("appConf", appConf);
[21/26] incubator-slider git commit: SLIDER-711 live/containers
resource
Posted by st...@apache.org.
SLIDER-711 live/containers resource
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9dffb405
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9dffb405
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9dffb405
Branch: refs/heads/develop
Commit: 9dffb40590a96bac96fd411564fe09f9ae77e134
Parents: a5d1f5e
Author: Steve Loughran <st...@apache.org>
Authored: Thu Dec 18 13:48:15 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Dec 18 13:48:15 2014 +0000
----------------------------------------------------------------------
.../server/appmaster/web/WebAppApiImpl.java | 4 +-
.../rest/application/ApplicationResource.java | 19 +++-
.../application/resources/ContentCache.java | 9 ++
.../appstate/StubSliderClusterProtocol.groovy | 108 +++++++++++++++++++
.../TestMockAppStateAppRestIntegration.groovy | 54 ++++++++--
5 files changed, 178 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9dffb405/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
index 36d4a8f..d5532ea 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
@@ -52,11 +52,11 @@ public class WebAppApiImpl implements WebAppApi {
CertificateManager certificateManager,
RegistryOperations registryOperations,
MetricsAndMonitoring metricsAndMonitoring) {
- this.registryOperations = registryOperations;
checkNotNull(clusterProto);
checkNotNull(appState);
checkNotNull(provider);
-
+
+ this.registryOperations = registryOperations;
this.clusterProto = clusterProto;
this.appState = appState;
this.provider = provider;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9dffb405/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
index 374e8de..e4be96f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
@@ -19,11 +19,14 @@
package org.apache.slider.server.appmaster.web.rest.application;
import com.google.common.collect.Lists;
+import org.apache.slider.api.types.SerializedContainerInformation;
import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
import org.apache.slider.server.appmaster.web.rest.application.resources.CachedContent;
+import org.apache.slider.server.appmaster.web.rest.application.resources.ContainerListRefresher;
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
import org.apache.slider.server.appmaster.web.rest.application.resources.LiveResourcesRefresher;
import org.slf4j.Logger;
@@ -35,6 +38,7 @@ import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import java.util.List;
+import java.util.Map;
@Singleton
public class ApplicationResource extends AbstractSliderResource {
@@ -46,9 +50,13 @@ public class ApplicationResource extends AbstractSliderResource {
public ApplicationResource(WebAppApi slider) {
super(slider);
+ StateAccessForProviders state = slider.getAppState();
cache.put(RestPaths.LIVE_RESOURCES,
new CachedContent<ConfTree>(LIFESPAN,
- new LiveResourcesRefresher(slider.getAppState())));
+ new LiveResourcesRefresher(state)));
+ cache.put(RestPaths.LIVE_CONTAINERS,
+ new CachedContent<Map<String, SerializedContainerInformation>>(LIFESPAN,
+ new ContainerListRefresher(state)));
}
/**
@@ -91,7 +99,7 @@ public class ApplicationResource extends AbstractSliderResource {
@Produces({MediaType.APPLICATION_JSON})
public Object getLiveResources() {
try {
- return cache.get(RestPaths.LIVE_RESOURCES).get();
+ return cache.lookup(RestPaths.LIVE_RESOURCES);
} catch (Exception e) {
throw buildException(RestPaths.LIVE_RESOURCES, e);
}
@@ -99,11 +107,12 @@ public class ApplicationResource extends AbstractSliderResource {
@GET
@Path(RestPaths.LIVE_CONTAINERS)
@Produces({MediaType.APPLICATION_JSON})
- public Object getLiveContainers() {
+ public Map<String, SerializedContainerInformation> getLiveContainers() {
try {
- return cache.get(RestPaths.LIVE_RESOURCES).get();
+ return (Map<String, SerializedContainerInformation>)cache.lookup(
+ RestPaths.LIVE_CONTAINERS);
} catch (Exception e) {
- throw buildException(RestPaths.LIVE_RESOURCES, e);
+ throw buildException(RestPaths.LIVE_CONTAINERS, e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9dffb405/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java
index f309570..6ee9604 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java
@@ -18,6 +18,9 @@
package org.apache.slider.server.appmaster.web.rest.application.resources;
+import com.google.common.base.Preconditions;
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
+
import java.util.concurrent.ConcurrentHashMap;
/**
@@ -31,4 +34,10 @@ public class ContentCache extends ConcurrentHashMap<String, CachedContent> {
public ContentCache() {
}
+
+ public Object lookup(String key) {
+ CachedContent content = get(key);
+ Preconditions.checkNotNull(content, "no content for path " + key);
+ return content.get();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9dffb405/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/StubSliderClusterProtocol.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/StubSliderClusterProtocol.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/StubSliderClusterProtocol.groovy
new file mode 100644
index 0000000..0836dcb
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/StubSliderClusterProtocol.groovy
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import org.apache.hadoop.ipc.ProtocolSignature
+import org.apache.hadoop.yarn.exceptions.YarnException
+import org.apache.slider.api.SliderClusterProtocol
+import org.apache.slider.api.proto.Messages
+
+class StubSliderClusterProtocol implements SliderClusterProtocol {
+ @Override
+ Messages.StopClusterResponseProto stopCluster(
+ Messages.StopClusterRequestProto request)
+ throws IOException, YarnException {
+ throw new UnsupportedOperationException()
+ }
+
+ @Override
+ Messages.FlexClusterResponseProto flexCluster(
+ Messages.FlexClusterRequestProto request)
+ throws IOException, YarnException {
+ throw new UnsupportedOperationException()
+ }
+
+ @Override
+ Messages.GetJSONClusterStatusResponseProto getJSONClusterStatus(
+ Messages.GetJSONClusterStatusRequestProto request)
+ throws IOException, YarnException {
+ throw new UnsupportedOperationException()
+ }
+
+ @Override
+ Messages.ListNodeUUIDsByRoleResponseProto listNodeUUIDsByRole(
+ Messages.ListNodeUUIDsByRoleRequestProto request)
+ throws IOException, YarnException {
+ throw new UnsupportedOperationException()
+ }
+
+ @Override
+ Messages.GetNodeResponseProto getNode(Messages.GetNodeRequestProto request)
+ throws IOException, YarnException {
+ throw new UnsupportedOperationException()
+ }
+
+ @Override
+ Messages.GetClusterNodesResponseProto getClusterNodes(
+ Messages.GetClusterNodesRequestProto request)
+ throws IOException, YarnException {
+ throw new UnsupportedOperationException()
+ }
+
+ @Override
+ Messages.EchoResponseProto echo(Messages.EchoRequestProto request)
+ throws IOException, YarnException {
+ throw new UnsupportedOperationException()
+ }
+
+ @Override
+ Messages.KillContainerResponseProto killContainer(
+ Messages.KillContainerRequestProto request)
+ throws IOException, YarnException {
+ throw new UnsupportedOperationException()
+ }
+
+ @Override
+ Messages.AMSuicideResponseProto amSuicide(
+ Messages.AMSuicideRequestProto request)
+ throws IOException, YarnException {
+ throw new UnsupportedOperationException()
+ }
+
+ @Override
+ Messages.GetInstanceDefinitionResponseProto getInstanceDefinition(
+ Messages.GetInstanceDefinitionRequestProto request)
+ throws IOException, YarnException {
+ throw new UnsupportedOperationException()
+ }
+
+ @Override
+ long getProtocolVersion(String protocol, long clientVersion)
+ throws IOException {
+ return 0
+ }
+
+ @Override
+ ProtocolSignature getProtocolSignature(
+ String protocol,
+ long clientVersion,
+ int clientMethodsHash) throws IOException {
+ throw new UnsupportedOperationException()
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9dffb405/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
index aefeecf..c188b9c 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
@@ -19,13 +19,22 @@
package org.apache.slider.server.appmaster.model.appstate
import groovy.util.logging.Slf4j
+import org.apache.hadoop.ipc.ProtocolSignature
+import org.apache.hadoop.yarn.exceptions.YarnException
+import org.apache.slider.api.SliderClusterProtocol
+import org.apache.slider.api.proto.Messages
import org.apache.slider.api.types.SerializedContainerInformation
import org.apache.slider.core.persist.JsonSerDeser
+import org.apache.slider.server.appmaster.management.MetricsAndMonitoring
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockProviderService
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.state.ProviderAppState
import org.apache.slider.server.appmaster.state.RoleInstance
import org.apache.slider.server.appmaster.state.StateAccessForProviders
+import org.apache.slider.server.appmaster.web.WebAppApi
+import org.apache.slider.server.appmaster.web.WebAppApiImpl
+import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource
import org.apache.slider.server.appmaster.web.rest.application.resources.CachedContent
import org.apache.slider.server.appmaster.web.rest.application.resources.ContainerListRefresher
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache
@@ -64,18 +73,11 @@ class TestMockAppStateAppRestIntegration extends BaseMockAppStateTest implements
@Test
public void testContainerListRefresher() throws Throwable {
- int r0 = 1
- int r1 = 2
- int r2 = 3
- role0Status.desired = r0
- role1Status.desired = r1
- role2Status.desired = r2
ContainerListRefresher clr = new ContainerListRefresher(stateAccess)
def map = clr.refresh()
assert map.size() == 0
- List<RoleInstance> instances = createAndStartNodes()
+ List<RoleInstance> instances = startNodes()
map = clr.refresh()
- assert instances.size() == r0 + r1 + r2
assert map.size() == instances.size()
log.info("$map")
JsonSerDeser<SerializedContainerInformation> serDeser =
@@ -85,11 +87,45 @@ class TestMockAppStateAppRestIntegration extends BaseMockAppStateTest implements
}
}
- public ProviderAppState getStateAccess() {
+ public List<RoleInstance> startNodes() {
+ int r0 = 1
+ int r1 = 2
+ int r2 = 3
+ role0Status.desired = r0
+ role1Status.desired = r1
+ role2Status.desired = r2
+ List<RoleInstance> instances = createAndStartNodes()
+ assert instances.size() == r0 + r1 + r2
+ return instances
+ }
+
+ @Test
+ public void testApplicationResource() throws Throwable {
+ def instances = startNodes()
+ ApplicationResource applicationResource =
+ new ApplicationResource(webAppApi)
+ def containers = applicationResource.liveContainers
+ assert containers.size() == instances.size()
+
+ }
+ /**
+ * Get a state accessor for the appState field
+ * @return something to hand down to refreshers and resources
+ */
+ public StateAccessForProviders getStateAccess() {
StateAccessForProviders state = new ProviderAppState("name", appState)
return state
}
+ public WebAppApi getWebAppApi() {
+ WebAppApi api = new WebAppApiImpl(new StubSliderClusterProtocol(),
+ stateAccess,
+ new MockProviderService(),
+ null, null,
+ new MetricsAndMonitoring("metrics"))
+ return api
+ }
+
/**
* Little class to do integer refreshing & so test refresh logic
*/
[03/26] incubator-slider git commit: SLIDER-151,
drop getNode operation from client and tests
Posted by st...@apache.org.
SLIDER-151, drop getNode operation from client and tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/04ebc7e9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/04ebc7e9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/04ebc7e9
Branch: refs/heads/develop
Commit: 04ebc7e9acf61bb1d3be1cfd32f088fcf0b26239
Parents: aced331
Author: Steve Loughran <st...@apache.org>
Authored: Thu Jun 19 14:55:13 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Dec 16 20:25:39 2014 +0000
----------------------------------------------------------------------
.../java/org/apache/slider/client/SliderClient.java | 12 ------------
1 file changed, 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/04ebc7e9/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 4c7640d..813ae0f 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2429,18 +2429,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
/**
- * Get a node from the AM
- * @param uuid uuid of node
- * @return deserialized node
- * @throws IOException IO problems
- * @throws NoSuchNodeException if the node isn't found
- */
- @VisibleForTesting
- public ClusterNode getNode(String uuid) throws IOException, YarnException {
- return createClusterOperations().getNode(uuid);
- }
-
- /**
* Get the instance definition from the far end
*/
@VisibleForTesting
[23/26] incubator-slider git commit: SLIDER-729 move state int enum
from ConfigurationDescription to standalone
Posted by st...@apache.org.
SLIDER-729 move state int enum from ConfigurationDescription to standalone
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c52183c6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c52183c6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c52183c6
Branch: refs/heads/develop
Commit: c52183c6cdf657ea680ae1a14a475c5eead39bc6
Parents: a3e2297
Author: Steve Loughran <st...@apache.org>
Authored: Thu Dec 18 18:03:11 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Dec 18 18:03:11 2014 +0000
----------------------------------------------------------------------
.../apache/slider/api/ClusterDescription.java | 12 ++---
.../java/org/apache/slider/api/StateValues.java | 53 ++++++++++++++++++++
.../org/apache/slider/client/SliderClient.java | 3 +-
.../slider/client/SliderClusterOperations.java | 5 +-
.../slider/server/appmaster/state/AppState.java | 15 +++---
.../server/appmaster/state/RoleInstance.java | 2 +-
.../resources/ContainerListRefresher.java | 3 ++
.../standalone/TestStandaloneAgentWeb.groovy | 1 +
.../apache/slider/test/SliderTestUtils.groovy | 25 ++++++++-
9 files changed, 101 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c52183c6/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java b/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
index 3b0fd2b..8bcff0e 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
@@ -107,28 +107,28 @@ public class ClusterDescription implements Cloneable {
* Specification is incomplete & cannot
* be used: {@value}
*/
- public static final int STATE_INCOMPLETE = 0;
+ public static final int STATE_INCOMPLETE = StateValues.STATE_INCOMPLETE;
/**
* Spec has been submitted: {@value}
*/
- public static final int STATE_SUBMITTED = 1;
+ public static final int STATE_SUBMITTED = StateValues.STATE_SUBMITTED;
/**
* Cluster created: {@value}
*/
- public static final int STATE_CREATED = 2;
+ public static final int STATE_CREATED = StateValues.STATE_CREATED;
/**
* Live: {@value}
*/
- public static final int STATE_LIVE = 3;
+ public static final int STATE_LIVE = StateValues.STATE_LIVE;
/**
* Stopped
*/
- public static final int STATE_STOPPED = 4;
+ public static final int STATE_STOPPED = StateValues.STATE_STOPPED;
/**
* destroyed
*/
- public static final int STATE_DESTROYED = 5;
+ public static final int STATE_DESTROYED = StateValues.STATE_DESTROYED;
/**
* When was the cluster specification created?
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c52183c6/slider-core/src/main/java/org/apache/slider/api/StateValues.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/StateValues.java b/slider-core/src/main/java/org/apache/slider/api/StateValues.java
new file mode 100644
index 0000000..03751e1
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/StateValues.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api;
+
+/**
+ * Enumeration of state values
+ */
+public class StateValues {
+
+ /**
+ * Specification is incomplete & cannot
+ * be used: {@value}
+ */
+ public static final int STATE_INCOMPLETE = 0;
+
+ /**
+ * Spec has been submitted: {@value}
+ */
+ public static final int STATE_SUBMITTED = 1;
+ /**
+ * Cluster created: {@value}
+ */
+ public static final int STATE_CREATED = 2;
+ /**
+ * Live: {@value}
+ */
+ public static final int STATE_LIVE = 3;
+ /**
+ * Stopped
+ */
+ public static final int STATE_STOPPED = 4;
+ /**
+ * destroyed
+ */
+ public static final int STATE_DESTROYED = 5;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c52183c6/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index f9aae6f..225b0e0 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -65,6 +65,7 @@ import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.SliderClusterProtocol;
+import org.apache.slider.api.StateValues;
import org.apache.slider.api.proto.Messages;
import org.apache.slider.common.Constants;
import org.apache.slider.common.SliderExitCodes;
@@ -2769,7 +2770,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
.getClusterDescription();
log.info("Slider AppMaster is accessible");
- if (clusterDescription.state == ClusterDescription.STATE_LIVE) {
+ if (clusterDescription.state == StateValues.STATE_LIVE) {
AggregateConf instanceDefinition = clusterOperations
.getInstanceDefinition();
String imagePath = instanceDefinition.getInternalOperations().get(
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c52183c6/slider-core/src/main/java/org/apache/slider/client/SliderClusterOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClusterOperations.java b/slider-core/src/main/java/org/apache/slider/client/SliderClusterOperations.java
index e99897c..d722902 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClusterOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClusterOperations.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.SliderClusterProtocol;
+import org.apache.slider.api.StateValues;
import org.apache.slider.api.proto.Messages;
import org.apache.slider.common.tools.Duration;
import org.apache.slider.core.conf.AggregateConf;
@@ -245,7 +246,7 @@ public class SliderClusterOperations {
throws WaitTimeoutException, IOException, YarnException {
Duration duration = new Duration(timeout).start();
boolean live = false;
- int state = ClusterDescription.STATE_CREATED;
+ int state = StateValues.STATE_CREATED;
log.info("Waiting {} millis for a live node in role {}", timeout, role);
while (!live) {
@@ -260,7 +261,7 @@ public class SliderClusterOperations {
roleInstance = getNode(containers[0]);
if (roleInstance != null) {
state = roleInstance.state;
- live = state >= ClusterDescription.STATE_LIVE;
+ live = state >= StateValues.STATE_LIVE;
}
}
if (!live) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c52183c6/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index ddba5e2..61a6077 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import static org.apache.slider.api.StateValues.*;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterDescriptionOperations;
@@ -552,7 +553,7 @@ public class AppState {
public void initClusterStatus() {
//copy into cluster status.
ClusterDescription status = ClusterDescription.copy(clusterSpec);
- status.state = ClusterDescription.STATE_CREATED;
+ status.state = STATE_CREATED;
MapOperations infoOps = new MapOperations("info", status.info);
infoOps.mergeWithoutOverwrite(applicationInfo);
SliderUtils.addBuildInfo(infoOps, "status");
@@ -572,7 +573,7 @@ public class AppState {
StatusKeys.INFO_CREATE_TIME_MILLIS,
now);
}
- status.state = ClusterDescription.STATE_LIVE;
+ status.state = STATE_LIVE;
//set the app state to this status
setClusterStatus(status);
@@ -819,7 +820,7 @@ public class AppState {
* indicating the spawned process is up and running.
*/
public void noteAMLive() {
- appMasterNode.state = ClusterDescription.STATE_LIVE;
+ appMasterNode.state = STATE_LIVE;
}
public RoleInstance getAppMasterNode() {
@@ -1074,7 +1075,7 @@ public class AppState {
*/
public void containerStartSubmitted(Container container,
RoleInstance instance) {
- instance.state = ClusterDescription.STATE_SUBMITTED;
+ instance.state = STATE_SUBMITTED;
instance.container = container;
instance.createTime = now();
getStartingNodes().put(container.getId(), instance);
@@ -1275,7 +1276,7 @@ public class AppState {
throw new YarnRuntimeException(
"Container "+ containerId +"%s is already started");
}
- instance.state = ClusterDescription.STATE_LIVE;
+ instance.state = STATE_LIVE;
RoleStatus roleStatus = lookupRoleStatus(instance.roleId);
roleStatus.incStarted();
Container container = instance.container;
@@ -1463,7 +1464,7 @@ public class AppState {
log.info("Removing node ID {}", id);
RoleInstance node = getLiveNodes().remove(id);
if (node != null) {
- node.state = ClusterDescription.STATE_DESTROYED;
+ node.state = STATE_DESTROYED;
node.exitCode = status.getExitStatus();
node.diagnostics = status.getDiagnostics();
getCompletedNodes().put(id, node);
@@ -2007,7 +2008,7 @@ public class AppState {
instance.environment = new String[0];
instance.container = container;
instance.createTime = now();
- instance.state = ClusterDescription.STATE_LIVE;
+ instance.state = STATE_LIVE;
putOwnedContainer(cid, instance);
//role history gets told
roleHistory.onContainerAssigned(container);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c52183c6/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
index 1488fb7..2790a0d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
@@ -59,7 +59,7 @@ public final class RoleInstance implements Cloneable {
public int roleId;
/**
- * state from {@link ClusterDescription}
+ * state from {@link StateValues}
*/
public int state;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c52183c6/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java
index 7a48509..9049186 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java
@@ -27,6 +27,9 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+/**
+ * Refresh the container list.
+ */
public class ContainerListRefresher implements ResourceRefresher<Map<String, SerializedContainerInformation>> {
private final StateAccessForProviders state;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c52183c6/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
index cbb340f..9b614d2 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
@@ -23,6 +23,7 @@ import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.api.StateValues
import static org.apache.slider.api.ResourceKeys.*
import static org.apache.slider.api.StatusKeys.*
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c52183c6/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 39dfe39..b672498 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -41,18 +41,24 @@ import org.apache.hadoop.yarn.webapp.NotFoundException
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ClusterNode
import org.apache.slider.api.RoleKeys
+import org.apache.slider.api.StateValues
import org.apache.slider.client.SliderClient
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.tools.Duration
import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.conf.AggregateConf
+import org.apache.slider.core.conf.ConfTree
+import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.core.exceptions.BadClusterStateException
import org.apache.slider.core.exceptions.SliderException
import org.apache.slider.core.exceptions.WaitTimeoutException
import org.apache.slider.core.main.ServiceLaunchException
import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.persist.ConfTreeSerDeser
+import org.apache.slider.core.persist.JsonSerDeser
import org.apache.slider.core.registry.docstore.PublishedConfigSet
import org.apache.slider.server.appmaster.web.HttpCacheHeaders
+import org.apache.slider.server.appmaster.web.rest.RestPaths
import org.apache.slider.server.services.workflow.ForkedProcessService
import org.junit.Assert
import org.junit.Assume
@@ -367,7 +373,7 @@ class SliderTestUtils extends Assert {
String role)
throws WaitTimeoutException, IOException, SliderException {
int state = client.waitForRoleInstanceLive(role, spintime);
- return state == ClusterDescription.STATE_LIVE;
+ return state == StateValues.STATE_LIVE;
}
public static ClusterDescription dumpClusterStatus(
@@ -1162,4 +1168,21 @@ class SliderTestUtils extends Assert {
}
}
+ public <T> T fetchType(
+ Class<T> clazz, String appmaster, String subpath) {
+ JsonSerDeser serDeser = new JsonSerDeser(clazz)
+
+ def json = getWebPage(
+ appmaster,
+ RestPaths.SLIDER_PATH_APPLICATION + subpath)
+ T ctree = (T) serDeser.fromJson(json)
+ return ctree
+ }
+
+ public ConfTreeOperations fetchConfigTree(
+ YarnConfiguration conf, String appmaster, String subpath) {
+ ConfTree ctree = fetchType(ConfTree, appmaster, subpath)
+ ConfTreeOperations tree = new ConfTreeOperations(ctree)
+ return tree
+ }
}
[26/26] incubator-slider git commit: Merge branch
'feature/SLIDER-151_REST_API' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-151_REST_API' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/57d0d2ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/57d0d2ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/57d0d2ed
Branch: refs/heads/develop
Commit: 57d0d2ed2c6ad94ede6d39bbe791c4391aaa115c
Parents: 8609f16 3c450b9
Author: Steve Loughran <st...@apache.org>
Authored: Thu Dec 18 21:33:04 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Dec 18 21:33:04 2014 +0000
----------------------------------------------------------------------
.../apache/slider/api/ClusterDescription.java | 12 +-
.../org/apache/slider/api/ResourceKeys.java | 2 +-
.../slider/api/SliderClusterProtocol.java | 24 ++-
.../java/org/apache/slider/api/StateValues.java | 53 +++++
.../java/org/apache/slider/api/StatusKeys.java | 41 +++-
.../types/SerializedComponentInformation.java | 41 ++++
.../types/SerializedContainerInformation.java | 57 ++++++
.../org/apache/slider/client/SliderClient.java | 18 +-
.../slider/client/SliderClusterOperations.java | 5 +-
.../org/apache/slider/common/SliderKeys.java | 6 +-
.../apache/slider/common/tools/PortScanner.java | 23 ++-
.../apache/slider/core/build/InstanceIO.java | 14 +-
.../slider/core/conf/ConfTreeOperations.java | 3 +-
.../slider/core/persist/JsonSerDeser.java | 2 +
.../registry/docstore/PublishedContent.java | 24 ---
.../server/appmaster/RoleLaunchService.java | 2 +-
.../server/appmaster/SliderAppMaster.java | 155 ++++++++-------
.../slider/server/appmaster/state/AppState.java | 26 ++-
.../server/appmaster/state/RoleInstance.java | 29 ++-
.../server/appmaster/state/RoleStatus.java | 39 ++--
.../server/appmaster/web/HttpCacheHeaders.java | 35 ++++
.../server/appmaster/web/SliderAMWebApp.java | 1 -
.../server/appmaster/web/WebAppApiImpl.java | 4 +-
.../web/rest/AMWadlGeneratorConfig.java | 4 +-
.../appmaster/web/rest/AMWebServices.java | 24 ++-
.../web/rest/AbstractSliderResource.java | 96 +++++++++
.../server/appmaster/web/rest/RestPaths.java | 81 +++++++-
.../web/rest/SliderJacksonJaxbJsonProvider.java | 1 -
.../appmaster/web/rest/agent/AgentResource.java | 6 +-
.../rest/application/ApplicationResource.java | 199 +++++++++++++++++++
.../web/rest/application/package-info.java | 24 +++
.../application/resources/CachedContent.java | 121 +++++++++++
.../application/resources/ContentCache.java | 43 ++++
.../resources/LiveComponentsRefresher.java | 52 +++++
.../resources/LiveContainersRefresher.java | 51 +++++
.../resources/LiveResourcesRefresher.java | 68 +++++++
.../resources/ResourceRefresher.java | 31 +++
.../resources/ResourceSnapshotRefresher.java | 40 ++++
.../web/rest/management/ManagementResource.java | 30 ++-
.../management/resources/ActionsResource.java | 22 ++
.../resources/AggregateConfResource.java | 2 +-
.../management/resources/ConfTreeResource.java | 3 +-
.../web/rest/publisher/PublisherResource.java | 56 +++---
.../web/rest/registry/RegistryResource.java | 18 +-
.../slider/agent/actions/TestActionList.groovy | 2 +-
.../standalone/TestStandaloneAgentWeb.groovy | 134 +++++++++++--
.../appstate/StubSliderClusterProtocol.groovy | 108 ++++++++++
.../TestMockAppStateAppRestIntegration.groovy | 164 +++++++++++++++
.../TestMockContainerResourceAllocations.groovy | 8 +-
.../publisher/TestPublisherRestResources.groovy | 5 +
.../org/apache/slider/test/KeysForTests.groovy | 1 +
.../apache/slider/test/SliderTestUtils.groovy | 163 ++++++++++++---
.../test/YarnZKMiniClusterTestBase.groovy | 1 +
.../framework/AgentCommandTestBase.groovy | 8 +-
.../funtest/framework/CommandTestBase.groovy | 17 +-
.../funtest/lifecycle/AgentWebPagesIT.groovy | 27 +--
56 files changed, 1907 insertions(+), 319 deletions(-)
----------------------------------------------------------------------
[07/26] incubator-slider git commit: SLIDER-708 reformat IPC source
Posted by st...@apache.org.
SLIDER-708 reformat IPC source
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5c882b1e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5c882b1e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5c882b1e
Branch: refs/heads/develop
Commit: 5c882b1ec2a6131aa54442d01b3b500e58ea9a3b
Parents: a342066
Author: Steve Loughran <st...@apache.org>
Authored: Thu Dec 11 15:27:51 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Dec 16 20:25:39 2014 +0000
----------------------------------------------------------------------
.../slider/api/SliderClusterProtocol.java | 24 ++++++++++++--------
1 file changed, 14 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5c882b1e/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java b/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
index 2f198c5..fc5c21b 100644
--- a/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
+++ b/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
@@ -47,26 +47,29 @@ public interface SliderClusterProtocol extends VersionedProtocol {
/**
* Flex the cluster.
*/
- Messages.FlexClusterResponseProto flexCluster(Messages.FlexClusterRequestProto request) throws IOException,
- YarnException;
+ Messages.FlexClusterResponseProto flexCluster(Messages.FlexClusterRequestProto request)
+ throws IOException, YarnException;
/**
* Get the current cluster status
*/
- Messages.GetJSONClusterStatusResponseProto getJSONClusterStatus(Messages.GetJSONClusterStatusRequestProto request) throws IOException, YarnException;
+ Messages.GetJSONClusterStatusResponseProto getJSONClusterStatus(Messages.GetJSONClusterStatusRequestProto request)
+ throws IOException, YarnException;
/**
* List all running nodes in a role
*/
- Messages.ListNodeUUIDsByRoleResponseProto listNodeUUIDsByRole(Messages.ListNodeUUIDsByRoleRequestProto request) throws IOException, YarnException;
+ Messages.ListNodeUUIDsByRoleResponseProto listNodeUUIDsByRole(Messages.ListNodeUUIDsByRoleRequestProto request)
+ throws IOException, YarnException;
/**
* Get the details on a node
*/
- Messages.GetNodeResponseProto getNode(Messages.GetNodeRequestProto request) throws IOException, YarnException;
+ Messages.GetNodeResponseProto getNode(Messages.GetNodeRequestProto request)
+ throws IOException, YarnException;
/**
* Get the
@@ -74,7 +77,8 @@ public interface SliderClusterProtocol extends VersionedProtocol {
* Unknown nodes are not returned
* <i>Important: the order of the results are undefined</i>
*/
- Messages.GetClusterNodesResponseProto getClusterNodes(Messages.GetClusterNodesRequestProto request) throws IOException, YarnException;
+ Messages.GetClusterNodesResponseProto getClusterNodes(Messages.GetClusterNodesRequestProto request)
+ throws IOException, YarnException;
/**
* Echo back the submitted text (after logging it).
@@ -94,7 +98,8 @@ public interface SliderClusterProtocol extends VersionedProtocol {
* @throws IOException
* @throws YarnException
*/
- Messages.KillContainerResponseProto killContainer(Messages.KillContainerRequestProto request) throws IOException, YarnException;
+ Messages.KillContainerResponseProto killContainer(Messages.KillContainerRequestProto request)
+ throws IOException, YarnException;
/**
* AM to commit suicide. If the Hadoop halt entry point has not been disabled,
@@ -104,9 +109,8 @@ public interface SliderClusterProtocol extends VersionedProtocol {
* @throws IOException
* @throws YarnException
*/
- Messages.AMSuicideResponseProto amSuicide(Messages.AMSuicideRequestProto request) throws
- IOException,
- YarnException;
+ Messages.AMSuicideResponseProto amSuicide(Messages.AMSuicideRequestProto request)
+ throws IOException, YarnException;
/**
* Get the instance definition
[06/26] incubator-slider git commit: SLIDER-708 cleanup and better
javadocs
Posted by st...@apache.org.
SLIDER-708 cleanup and better javadocs
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b42e250e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b42e250e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b42e250e
Branch: refs/heads/develop
Commit: b42e250ee95b83eca43114dcb8a892617af33fba
Parents: 5c882b1
Author: Steve Loughran <st...@apache.org>
Authored: Thu Dec 11 15:28:31 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Dec 16 20:25:39 2014 +0000
----------------------------------------------------------------------
.../registry/docstore/PublishedContent.java | 24 ----------
.../server/appmaster/web/rest/RestPaths.java | 49 ++++++++++++++++++--
2 files changed, 45 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b42e250e/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedContent.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedContent.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedContent.java
deleted file mode 100644
index 67a9b35..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedContent.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.docstore;
-
-public class PublishedContent {
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b42e250e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
index c3c81fc..0df617f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
@@ -25,7 +25,15 @@ public class RestPaths {
public static final String WS_CONTEXT = "ws";
public static final String AGENT_WS_CONTEXT = "ws";
+
+ /**
+ * Root path for the web services context: {@value}
+ */
public static final String WS_CONTEXT_ROOT = "/" + WS_CONTEXT;
+
+ /**
+ * agent content root: {@value}
+ */
public static final String WS_AGENT_CONTEXT_ROOT = "/" + AGENT_WS_CONTEXT;
public static final String SLIDER_CONTEXT_ROOT = WS_CONTEXT_ROOT +"/v1/slider";
public static final String SLIDER_AGENT_CONTEXT_ROOT = WS_AGENT_CONTEXT_ROOT +"/v1/slider";
@@ -34,22 +42,33 @@ public class RestPaths {
public static final String SLIDER_SUBPATH_AGENTS = "/agents";
public static final String SLIDER_SUBPATH_PUBLISHER = "/publisher";
+
+ /**
+ * management path: {@value}
+ */
public static final String SLIDER_PATH_MANAGEMENT = SLIDER_CONTEXT_ROOT
+ SLIDER_SUBPATH_MANAGEMENT;
+
+ /**
+ * Agents: {@value}
+ */
public static final String SLIDER_PATH_AGENTS = SLIDER_AGENT_CONTEXT_ROOT
+ SLIDER_SUBPATH_AGENTS;
+ /**
+ * Publisher: {@value}
+ */
public static final String SLIDER_PATH_PUBLISHER = SLIDER_CONTEXT_ROOT
+ SLIDER_SUBPATH_PUBLISHER;
public static final String SLIDER_SUBPATH_REGISTRY = "/registry";
+
+ /**
+ * Registry: {@value}
+ */
public static final String SLIDER_PATH_REGISTRY = SLIDER_CONTEXT_ROOT
+ SLIDER_SUBPATH_REGISTRY;
- @Deprecated
- public static final String REGISTRY_SERVICE = "v1/service";
- @Deprecated
- public static final String REGISTRY_ANYSERVICE = "v1/anyservice";
/**
* The regular expressions used to define valid configuration names/url path
@@ -65,10 +84,32 @@ public class RestPaths {
public static final String SLIDER_EXPORTS = "exports";
public static final String SLIDER_CLASSPATH = "classpath";
+
+ /**
+ * Codahale Metrics - base path: {@value}
+ */
+
public static final String SYSTEM = "/system";
+
+
+ /**
+ * Codahale Metrics - health: {@value}
+ */
public static final String SYSTEM_HEALTHCHECK = SYSTEM + "/health";
+ /**
+ * Codahale Metrics - metrics: {@value}
+ */
public static final String SYSTEM_METRICS = SYSTEM + "/metrics";
+ /**
+ * Codahale Metrics - metrics as JSON: {@value}
+ */
public static final String SYSTEM_METRICS_JSON = SYSTEM_METRICS + "?format=json";
+ /**
+ * Codahale Metrics - ping: {@value}
+ */
public static final String SYSTEM_PING = SYSTEM + "/ping";
+ /**
+ * Codahale Metrics - thread dump: {@value}
+ */
public static final String SYSTEM_THREADS = SYSTEM + "/threads";
}
[09/26] incubator-slider git commit: SLIDER-724 don't serve web
content until AM is fully configured: AM-side
Posted by st...@apache.org.
SLIDER-724 don't serve web content until AM is fully configured: AM-side
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bfc20d75
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bfc20d75
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bfc20d75
Branch: refs/heads/develop
Commit: bfc20d7597faf3dcb9cac6f2853bdb6db7727de5
Parents: 47ca9e6
Author: Steve Loughran <st...@apache.org>
Authored: Tue Dec 16 20:23:56 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Dec 16 20:25:40 2014 +0000
----------------------------------------------------------------------
.../apache/slider/common/tools/PortScanner.java | 23 +++++++++-
.../server/appmaster/SliderAppMaster.java | 47 ++++++++++++--------
2 files changed, 49 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfc20d75/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java b/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
index 0f4cfbc..60fd008 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
@@ -28,7 +28,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
- *
+ * a scanner which can take an input string for a range or scan the lot.
*/
public class PortScanner {
private static Pattern NUMBER_RANGE = Pattern.compile("^(\\d+)\\s*-\\s*(\\d+)$");
@@ -39,6 +39,8 @@ public class PortScanner {
public PortScanner() {
}
+ int nextPort = 1024;
+
public void setPortRange(String input) {
// first split based on commas
Set<Integer> inputPorts= new TreeSet<Integer>();
@@ -66,7 +68,24 @@ public class PortScanner {
return remainingPortsToCheck;
}
- public int getAvailablePort () throws SliderException{
+ public int getAvailablePort() throws SliderException {
+ if (remainingPortsToCheck!=null) {
+ return getAvailablePortViaPortArray();
+ } else {
+ return getAvailablePortViaCounter();
+ }
+ }
+
+ private int getAvailablePortViaCounter() throws SliderException {
+ int port;
+ do {
+ port = nextPort;
+ nextPort++;
+ } while (!SliderUtils.isPortAvailable(port));
+ return port;
+ }
+
+ private int getAvailablePortViaPortArray() throws SliderException {
boolean found = false;
int availablePort = -1;
Iterator<Integer> portsToCheck = this.remainingPortsToCheck.iterator();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfc20d75/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 429dc95..67e050a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -378,6 +378,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
private RegisterApplicationMasterResponse amRegistrationData;
private PortScanner portScanner;
private SecurityConfiguration securityConfiguration;
+
+ /**
+ * The port for the web application
+ */
private int webAppPort;
/**
@@ -679,6 +683,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
}
//bring up the Slider RPC service
+
+ buildPortScanner(instanceDefinition);
startSliderRPCServer(instanceDefinition);
rpcServiceAddress = rpcService.getConnectAddress();
@@ -705,7 +711,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
.getComponent(SliderKeys.COMPONENT_AM);
certificateManager.initialize(component);
certificateManager.setPassphrase(instanceDefinition.getPassphrase());
-
+
if (component.getOptionBool(
AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, false)) {
uploadServerCertForLocalization(clustername, fs);
@@ -713,7 +719,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
startAgentWebApp(appInformation, serviceConf);
- webAppPort = getPortToRequest(instanceDefinition);
+ webAppPort = getPortToRequest();
+ if (webAppPort == 0) {
+ // failure to find a port
+ throw new BadConfigException("Failed to fix a web application port");
+ }
String scheme = WebAppUtils.HTTP_PREFIX;
appMasterTrackingUrl = scheme + appMasterHostname + ":" + webAppPort;
@@ -965,32 +975,31 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
/**
+ * Build up the port scanner. This may include setting a port range.
+ */
+ private void buildPortScanner(AggregateConf instanceDefinition) {
+ portScanner = new PortScanner();
+ String portRange = instanceDefinition.
+ getAppConfOperations().getGlobalOptions().
+ getOption(SliderKeys.KEY_ALLOWED_PORT_RANGE, "0");
+ if (!"0".equals(portRange)) {
+ portScanner.setPortRange(portRange);
+ }
+ }
+
+ /**
* Locate a port to request for a service such as RPC or web/REST.
* This uses port range definitions in the <code>instanceDefinition</code>
* to fix the port range —if one is set.
* <p>
* The port returned is available at the time of the request; there are
* no guarantees as to how long that situation will last.
- * @param instanceDefinition instance definition containing port range
- * restrictions in in the application configuration
* @return the port to request.
* @throws SliderException
*/
- private int getPortToRequest(AggregateConf instanceDefinition)
+ private int getPortToRequest()
throws SliderException {
- int portToRequest = 0;
- String portRange = instanceDefinition.
- getAppConfOperations().getGlobalOptions().
- getOption(SliderKeys.KEY_ALLOWED_PORT_RANGE, "0");
- if (!"0".equals(portRange)) {
- if (portScanner == null) {
- portScanner = new PortScanner();
- portScanner.setPortRange(portRange);
- }
- portToRequest = portScanner.getAvailablePort();
- }
-
- return portToRequest;
+ return portScanner.getAvailablePort();
}
private void uploadServerCertForLocalization(String clustername,
@@ -1427,7 +1436,7 @@ the registry with/without the new record format
.newReflectiveBlockingService(
protobufRelay);
- int port = getPortToRequest(instanceDefinition);
+ int port = getPortToRequest();
rpcService =
new WorkflowRpcService("SliderRPC", RpcBinder.createProtobufServer(
new InetSocketAddress("0.0.0.0", port),
[05/26] incubator-slider git commit: SLIDER-724 Implement
OfflineFilter to return 503 "unavailable", for AM bootstrap
Posted by st...@apache.org.
SLIDER-724 Implement OfflineFilter to return 503 "unavailable", for AM bootstrap
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fe3ffc9a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fe3ffc9a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fe3ffc9a
Branch: refs/heads/develop
Commit: fe3ffc9a591d569cc0ba79e029fa9b27a6aaece8
Parents: b42e250
Author: Steve Loughran <st...@apache.org>
Authored: Mon Dec 15 14:24:44 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Dec 16 20:25:39 2014 +0000
----------------------------------------------------------------------
.../appmaster/web/utils/OfflineFilter.java | 104 +++++++++++++++++++
1 file changed, 104 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fe3ffc9a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/utils/OfflineFilter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/utils/OfflineFilter.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/utils/OfflineFilter.java
new file mode 100644
index 0000000..ad16f4e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/utils/OfflineFilter.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.http.FilterContainer;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Offline filter.
+ * All filter instances share the static offline flag.
+ * <p>
+ * Callers must use the method {@link #bindFilter(FilterContainer)}
+ * to bind to the web container.
+ */
+public class OfflineFilter implements Filter {
+
+ private static final AtomicBoolean offline = new AtomicBoolean(true);
+
+ private static String offlineMessage = "offline";
+ private static final AtomicInteger retry = new AtomicInteger(10);
+
+ public OfflineFilter() {
+ }
+
+ public static synchronized void goOffline(String message) {
+ Preconditions.checkArgument(message != null, "null message");
+ offline.set(true);
+ offlineMessage = message;
+ }
+
+ public static synchronized String getOfflineMessage() {
+ return offlineMessage;
+ }
+
+ public static int getRetry() {
+ return retry.intValue();
+ }
+
+ public static void setRetry(int retryCount) {
+ retry.set(retryCount);
+ }
+
+ @Override
+ public void init(FilterConfig filterConfig) throws ServletException {
+
+ }
+
+ @Override
+ public void doFilter(ServletRequest request,
+ ServletResponse response,
+ FilterChain chain) throws IOException, ServletException {
+
+ if (!offline.get()) {
+ chain.doFilter(request, response);
+ } else {
+ // service is offline
+ HttpServletResponse httpResponse = (HttpServletResponse) response;
+ httpResponse.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE,
+ getOfflineMessage());
+ }
+ }
+
+ @Override
+ public void destroy() {
+
+ }
+
+ /**
+ * Add the filter to a container
+ * @param container container
+ */
+ public static void bindFilter(FilterContainer container) {
+ container.addFilter("OfflineFilter",
+ "org.apache.slider.server.appmaster.web.utils.OfflineFilter",
+ null);
+ }
+}
[24/26] incubator-slider git commit: SLIDER-711 container listing work
Posted by st...@apache.org.
SLIDER-711 container listing work
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/08b979c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/08b979c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/08b979c2
Branch: refs/heads/develop
Commit: 08b979c252a4067ddf79eeb21d3d76742cc10719
Parents: c52183c
Author: Steve Loughran <st...@apache.org>
Authored: Thu Dec 18 18:04:38 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Dec 18 18:04:38 2014 +0000
----------------------------------------------------------------------
.../types/SerializedContainerInformation.java | 13 +++---
.../slider/core/persist/JsonSerDeser.java | 2 +
.../slider/server/appmaster/state/AppState.java | 8 ++++
.../server/appmaster/state/RoleInstance.java | 19 ++++++++-
.../resources/ContainerListRefresher.java | 1 -
.../slider/agent/actions/TestActionList.groovy | 2 +-
.../standalone/TestStandaloneAgentWeb.groovy | 45 +++++++++++---------
.../apache/slider/test/SliderTestUtils.groovy | 1 -
8 files changed, 60 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/08b979c2/slider-core/src/main/java/org/apache/slider/api/types/SerializedContainerInformation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/SerializedContainerInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/SerializedContainerInformation.java
index dfb70f5..e079e7a 100644
--- a/slider-core/src/main/java/org/apache/slider/api/types/SerializedContainerInformation.java
+++ b/slider-core/src/main/java/org/apache/slider/api/types/SerializedContainerInformation.java
@@ -18,6 +18,7 @@
package org.apache.slider.api.types;
+import org.apache.hadoop.registry.client.binding.JsonSerDeser;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
@@ -43,14 +44,14 @@ public class SerializedContainerInformation {
* or the log cannot be picked up
*/
public String[] output;
+ public String host;
+ public String hostURL;
@Override
public String toString() {
- final StringBuilder sb =
- new StringBuilder("SerializedContainerInformation{");
- sb.append("containerId='").append(containerId).append('\'');
- sb.append(", component='").append(component).append('\'');
- sb.append('}');
- return sb.toString();
+ JsonSerDeser<SerializedContainerInformation> serDeser =
+ new JsonSerDeser<SerializedContainerInformation>(
+ SerializedContainerInformation.class);
+ return serDeser.toString(this);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/08b979c2/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
index 51aefed..c5908bb 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
@@ -265,4 +265,6 @@ public class JsonSerDeser<T> {
mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
return mapper.writeValueAsString(instance);
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/08b979c2/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 61a6077..e9d0371 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -791,9 +791,13 @@ public class AppState {
container.setNodeHttpAddress(nodeHttpAddress);
RoleInstance am = new RoleInstance(container);
am.role = SliderKeys.COMPONENT_AM;
+ am.roleId = SliderKeys.ROLE_AM_PRIORITY_INDEX;
+ am.createTime = System.currentTimeMillis();
+ am.startTime = System.currentTimeMillis();
appMasterNode = am;
//it is also added to the set of live nodes
getLiveNodes().put(containerId, am);
+ putOwnedContainer(containerId, am);
// patch up the role status
RoleStatus roleStatus = roleStatusMap.get(
@@ -1860,6 +1864,10 @@ public class AppState {
List<AbstractRMOperation> operations =
new ArrayList<AbstractRMOperation>(targets.size());
for (RoleInstance instance : targets) {
+ if (instance.roleId == SliderKeys.ROLE_AM_PRIORITY_INDEX) {
+ // don't worry about the AM
+ continue;
+ }
Container possible = instance.container;
ContainerId id = possible.getId();
if (!instance.released) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/08b979c2/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
index 2790a0d..ad4af1d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
@@ -233,11 +233,26 @@ public final class RoleInstance implements Cloneable {
ProtocolTypes.PROTOCOL_TCP, host, port);
addEndpoint(epr);
}
-
+
+ /**
+ * Serialize. Some data structures (e.g output)
+ * may be shared
+ * @return a serialized form for marshalling as JSON
+ */
public SerializedContainerInformation serialize() {
SerializedContainerInformation info = new SerializedContainerInformation();
info.containerId = id;
-
+ info.component = role;
+ info.startTime = startTime;
+ info.createTime = createTime;
+ info.diagnostics = diagnostics;
+ info.state = state;
+ info.host = host;
+ info.hostURL = hostURL;
+ info.released = released ? Boolean.TRUE : null;
+ if (output != null) {
+ info.output = output;
+ }
return info;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/08b979c2/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java
index 9049186..7e74062 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java
@@ -19,7 +19,6 @@
package org.apache.slider.server.appmaster.web.rest.application.resources;
import org.apache.slider.api.types.SerializedContainerInformation;
-import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/08b979c2/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
index 6baf5b1..bf65b0f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
@@ -53,7 +53,7 @@ class TestActionList extends AgentMiniClusterTestBase {
*/
@Test
- public void testSuite() throws Throwable {
+ public void testActionListSuite() throws Throwable {
testListThisUserNoClusters()
testListLiveCluster()
testListMissingCluster()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/08b979c2/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
index 9b614d2..4312cf3 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
@@ -21,9 +21,9 @@ package org.apache.slider.agent.standalone
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ApplicationReport
-import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.api.StateValues
+import org.apache.slider.api.types.SerializedContainerInformation
import static org.apache.slider.api.ResourceKeys.*
import static org.apache.slider.api.StatusKeys.*
@@ -31,7 +31,6 @@ import org.apache.slider.client.SliderClient
import static org.apache.slider.common.SliderKeys.*;
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.core.persist.ConfTreeSerDeser
import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
import org.junit.Test
@@ -41,7 +40,9 @@ import static org.apache.slider.server.appmaster.management.MetricsKeys.*
@CompileStatic
@Slf4j
class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
-
+
+ public static final int WEB_STARTUP_TIME = 30000
+
@Test
public void testStandaloneAgentWeb() throws Throwable {
@@ -65,10 +66,11 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
initConnectionFactory(launcher.configuration)
- execHttpRequest(30000) {
+ execHttpRequest(WEB_STARTUP_TIME) {
GET(realappmaster)
}
- execHttpRequest(30000) {
+
+ execHttpRequest(WEB_STARTUP_TIME) {
def metrics = GET(realappmaster, SYSTEM_METRICS)
log.info metrics
}
@@ -99,7 +101,7 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
})
// now some REST gets
- describe "Application REST GETs"
+ describe "Application REST ${LIVE_RESOURCES}"
ConfTreeOperations tree = fetchConfigTree(conf, appmaster, LIVE_RESOURCES)
@@ -114,20 +116,23 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_COMPLETED)
assert 0 == liveAM.getMandatoryOptionInt(COMPONENT_INSTANCES_RELEASING)
- }
-
- public ConfTreeOperations fetchConfigTree(
- YarnConfiguration conf,
- String appmaster,
- String subpath) {
- ConfTreeSerDeser serDeser = new ConfTreeSerDeser()
-
- def json = getWebPage(
- appmaster,
- SLIDER_PATH_APPLICATION + subpath)
- def ctree = serDeser.fromJson(json)
- ConfTreeOperations tree = new ConfTreeOperations(ctree)
- return tree
+ describe "Application REST ${LIVE_CONTAINERS}"
+
+ Map<String, SerializedContainerInformation> map =
+ fetchType(HashMap, appmaster, LIVE_CONTAINERS)
+ assert map.size() == 1
+ log.info "${map}"
+ SerializedContainerInformation info = (SerializedContainerInformation) map.values()[0]
+ assert info.containerId
+ assert map[info.containerId]
+
+ assert info.component == COMPONENT_AM
+ assert info.createTime > 0
+ assert info.exitCode == null
+ assert info.output == null
+ assert info.released == null
+ assert info.state == StateValues.STATE_LIVE
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/08b979c2/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index b672498..f9857a8 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -54,7 +54,6 @@ import org.apache.slider.core.exceptions.SliderException
import org.apache.slider.core.exceptions.WaitTimeoutException
import org.apache.slider.core.main.ServiceLaunchException
import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.core.persist.ConfTreeSerDeser
import org.apache.slider.core.persist.JsonSerDeser
import org.apache.slider.core.registry.docstore.PublishedConfigSet
import org.apache.slider.server.appmaster.web.HttpCacheHeaders
[02/26] incubator-slider git commit: SLIDER-151 adding actions
Posted by st...@apache.org.
SLIDER-151 adding actions
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7d52f8aa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7d52f8aa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7d52f8aa
Branch: refs/heads/develop
Commit: 7d52f8aa637e30446715e60cc0388cd6cb002df0
Parents: 04ebc7e
Author: Steve Loughran <st...@apache.org>
Authored: Fri Jun 20 16:08:08 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Dec 16 20:25:39 2014 +0000
----------------------------------------------------------------------
.../web/rest/management/ManagementResource.java | 43 ++++++++++++++++++--
1 file changed, 40 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d52f8aa/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java
index 02199cf..8a6b928 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java
@@ -29,6 +29,7 @@ import org.slf4j.LoggerFactory;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.GET;
+import javax.ws.rs.*;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
@@ -46,6 +47,8 @@ import java.net.URL;
public class ManagementResource extends AbstractSliderResource {
protected static final Logger log =
LoggerFactory.getLogger(ManagementResource.class);
+ public static final String CONFIG = "config";
+ public static final String APP_UNDER_MANAGEMENT = "/app";
public ManagementResource(WebAppApi slider) {
super(slider);
@@ -80,9 +83,9 @@ public class ManagementResource extends AbstractSliderResource {
}
@GET
- @Path("/app/configurations/{config}")
+ @Path(APP_UNDER_MANAGEMENT+"/configurations/{config}")
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
- public ConfTreeResource getConfTreeResource(@PathParam("config") String config,
+ public ConfTreeResource getConfTreeResource(@PathParam(CONFIG) String config,
@Context UriInfo uriInfo,
@Context HttpServletResponse res) {
init(res);
@@ -90,11 +93,45 @@ public class ManagementResource extends AbstractSliderResource {
ResourceFactory.createAggregateConfResource(getAggregateConf(),
uriInfo.getBaseUriBuilder()
.path(RestPaths.SLIDER_CONTEXT_ROOT)
- .path(RestPaths.MANAGEMENT + "/app"));
+ .path(RestPaths.MANAGEMENT + APP_UNDER_MANAGEMENT));
return aggregateConf.getConfTree(config);
}
protected AggregateConf getAggregateConf() {
return slider.getAppState().getInstanceDefinitionSnapshot();
}
+
+ @POST
+ @Path("actions/flex")
+ public void actionFlex() {
+ }
+
+ @POST
+ @Path("actions/stop")
+ public void actionStop() {
+ }
+
+
+ @POST
+ @Path("actions/test/suicide")
+ public void actionSuicide() {
+ }
+
+ @POST
+ @Path("actions/test/kill-container")
+ public void actionKillContainer() {
+ }
+
+
+ @GET
+ @Path("containers"+"/components" +"/{name}")
+ public void actionListContainers() {
+ }
+
+ @GET
+ @Path("containers"+"/components" +"/{name}")
+ public void actionListContainersbyComponent() {
+ }
+
+
}
[15/26] incubator-slider git commit: minor slideram code cleanup as
suggested by IDE
Posted by st...@apache.org.
minor slideram code cleanup as suggested by IDE
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5b819686
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5b819686
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5b819686
Branch: refs/heads/develop
Commit: 5b8196863c04efce45d9b250ecf78cfff1dba17d
Parents: fc7c363
Author: Steve Loughran <st...@apache.org>
Authored: Tue Dec 16 11:47:41 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Dec 16 20:25:40 2014 +0000
----------------------------------------------------------------------
.../server/appmaster/SliderAppMaster.java | 39 +++++---------------
1 file changed, 10 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5b819686/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index f5d9f03..429dc95 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -20,7 +20,6 @@ package org.apache.slider.server.appmaster;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.health.HealthCheckRegistry;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.protobuf.BlockingService;
import org.apache.hadoop.conf.Configuration;
@@ -530,7 +529,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
*/
if (action.equals(SliderActions.ACTION_HELP)) {
- log.info(getName() + serviceArgs.usage());
+ log.info("{}: {}", getName(), serviceArgs.usage());
exitCode = SliderExitCodes.EXIT_USAGE;
} else if (action.equals(SliderActions.ACTION_CREATE)) {
exitCode = createAndRunCluster(actionArgs.get(0));
@@ -1177,7 +1176,7 @@ the registry with/without the new record format
* Handler for {@link RegisterComponentInstance action}
* Register/re-register an ephemeral container that is already in the app state
* @param id the component
- * @param description
+ * @param description component description
* @return true if the component is registered
*/
public boolean registerComponent(ContainerId id, String description) throws
@@ -1215,7 +1214,7 @@ the registry with/without the new record format
log.info("Unregistering component {}", id);
if (yarnRegistryOperations == null) {
log.warn("Processing unregister component event before initialization " +
- "completed; init flag =" + initCompleted);
+ "completed; init flag ={}", initCompleted);
return;
}
String cid = RegistryPathUtils.encodeYarnID(id.toString());
@@ -1392,12 +1391,12 @@ the registry with/without the new record format
}
*/
} catch (IOException e) {
- log.info("Failed to unregister application: " + e, e);
+ log.info("Failed to unregister application: {}", e, e);
} catch (InvalidApplicationMasterRequestException e) {
log.info("Application not found in YARN application list;" +
- " it may have been terminated/YARN shutdown in progress: " + e, e);
+ " it may have been terminated/YARN shutdown in progress: {}", e, e);
} catch (YarnException e) {
- log.info("Failed to unregister application: " + e, e);
+ log.info("Failed to unregister application: {}", e, e);
}
return exitCode;
}
@@ -1494,7 +1493,7 @@ the registry with/without the new record format
//for all the operations, exec them
executeRMOperations(operations);
- log.info("Diagnostics: " + getContainerDiagnosticInfo());
+ log.info("Diagnostics: {}", getContainerDiagnosticInfo());
}
@Override //AMRMClientAsync
@@ -1585,7 +1584,7 @@ the registry with/without the new record format
/**
* Look at where the current node state is -and whether it should be changed
- * @param reason
+ * @param reason reason for operation
*/
private synchronized void reviewRequestAndReleaseNodes(String reason) {
log.debug("reviewRequestAndReleaseNodes({})", reason);
@@ -1685,7 +1684,7 @@ the registry with/without the new record format
@Override //AMRMClientAsync
public void onError(Throwable e) {
//callback says it's time to finish
- LOG_YARN.error("AMRMClientAsync.onError() received " + e, e);
+ LOG_YARN.error("AMRMClientAsync.onError() received {}", e, e);
signalAMComplete(new ActionStopSlider("stop",
EXIT_EXCEPTION_THROWN,
FinalApplicationStatus.FAILED,
@@ -1890,14 +1889,6 @@ the registry with/without the new record format
rmOperationHandler.execute(operations);
}
- /**
- * Get the RM operations handler for direct scheduling of work.
- */
- @VisibleForTesting
- public RMOperationHandler getRmOperationHandler() {
- return rmOperationHandler;
- }
-
@Override
public Messages.AMSuicideResponseProto amSuicide(
Messages.AMSuicideRequestProto request)
@@ -2088,7 +2079,7 @@ the registry with/without the new record format
UserGroupInformation.getLoginUser().getShortUserName(),
credentials);
}
- if (credentials.getAllTokens().size() > 0) {
+ if (!credentials.getAllTokens().isEmpty()) {
DataOutputBuffer dob = new DataOutputBuffer();
credentials.writeTokenStorageToStream(dob);
dob.close();
@@ -2153,16 +2144,6 @@ the registry with/without the new record format
LOG_YARN.warn("Failed to stop Container {}", containerId);
}
- /**
- The cluster description published to callers
- This is used as a synchronization point on activities that update
- the CD, and also to update some of the structures that
- feed in to the CD
- */
- public ClusterDescription getClusterSpec() {
- return appState.getClusterSpec();
- }
-
public AggregateConf getInstanceDefinition() {
return appState.getInstanceDefinition();
}
[20/26] incubator-slider git commit: SLIDER-711 verify that cache
headers propagate through production secure & insecure cluster connections
Posted by st...@apache.org.
SLIDER-711 verify that cache headers propagate through production secure & insecure cluster connections
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a5d1f5e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a5d1f5e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a5d1f5e7
Branch: refs/heads/develop
Commit: a5d1f5e7e6a4d0c14501eef9dd434c567cddc115
Parents: 8ad2bfb
Author: Steve Loughran <st...@apache.org>
Authored: Thu Dec 18 12:14:23 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Dec 18 12:14:23 2014 +0000
----------------------------------------------------------------------
.../org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a5d1f5e7/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
index f5c35f3..e5d5f2a 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
@@ -79,6 +79,12 @@ public class AgentWebPagesIT extends AgentCommandTestBase
assert report.url
def root = report.url
+
+ // get the root page, including some checks for cache disabled
+ getWebPage(root, {
+ HttpURLConnection conn ->
+ assertConnectionNotCaching(conn)
+ })
log.info getWebPage(root, RestPaths.SYSTEM_METRICS)
log.info getWebPage(root, RestPaths.SYSTEM_THREADS)
log.info getWebPage(root, RestPaths.SYSTEM_HEALTHCHECK)
[22/26] incubator-slider git commit: SLIDER-724
TestPublishedSliderResources to spin for REST live
Posted by st...@apache.org.
SLIDER-724 TestPublishedSliderResources to spin for REST live
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a3e22973
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a3e22973
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a3e22973
Branch: refs/heads/develop
Commit: a3e229739b5a65088cbea9eb4645d01a2de5879f
Parents: 9dffb40
Author: Steve Loughran <st...@apache.org>
Authored: Thu Dec 18 17:32:43 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Dec 18 17:32:43 2014 +0000
----------------------------------------------------------------------
.../web/rest/publisher/TestPublisherRestResources.groovy | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3e22973/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
index 1964497..e3f38eb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
@@ -106,6 +106,11 @@ class TestPublisherRestResources extends AgentTestBase {
webResource = client.resource(sliderConfigset);
webResource = client.resource(sliderConfigset + "dummy-site");
+
+ execHttpRequest(30000) {
+ GET(sliderConfigset)
+ }
+
PublishedConfiguration config = webResource.type(MediaType.APPLICATION_JSON)
.get(PublishedConfiguration.class);
assert config != null
[12/26] incubator-slider git commit: SLIDER-724 don't serve web
content until AM is fully configured: tests to iterate until happy or timed
out. Includes creation of better exceptions on failures
Posted by st...@apache.org.
SLIDER-724 don't serve web content until AM is fully configured: tests to iterate until happy or timed out. Includes creation of better exceptions on failures
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e654a0f2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e654a0f2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e654a0f2
Branch: refs/heads/develop
Commit: e654a0f27e1a53ed713745305f4dd446b8ef2cf8
Parents: bfc20d7
Author: Steve Loughran <st...@apache.org>
Authored: Tue Dec 16 20:24:29 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Dec 16 20:25:40 2014 +0000
----------------------------------------------------------------------
.../standalone/TestStandaloneAgentWeb.groovy | 31 ++++---
.../apache/slider/test/SliderTestUtils.groovy | 89 ++++++++++++++++----
2 files changed, 92 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e654a0f2/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
index b2d29f3..577ad73 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy
@@ -25,6 +25,7 @@ import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.client.SliderClient
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.server.appmaster.web.rest.RestPaths
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
import org.junit.Test
import static org.apache.slider.server.appmaster.management.MetricsKeys.*
@@ -52,8 +53,10 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
ApplicationReport report = waitForClusterLive(client)
def realappmaster = report.originalTrackingUrl
- GET(realappmaster)
- def metrics = GET(realappmaster, RestPaths.SYSTEM_METRICS)
+ execHttpRequest(30000) {
+ GET(realappmaster)
+ }
+ def metrics = GET(realappmaster, SYSTEM_METRICS)
log.info metrics
sleep(5000)
@@ -61,21 +64,27 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase {
GET(appmaster)
- log.info GET(appmaster, RestPaths.SYSTEM_PING)
- log.info GET(appmaster, RestPaths.SYSTEM_THREADS)
- log.info GET(appmaster, RestPaths.SYSTEM_HEALTHCHECK)
- log.info GET(appmaster, RestPaths.SYSTEM_METRICS_JSON)
+ log.info GET(appmaster, SYSTEM_PING)
+ log.info GET(appmaster, SYSTEM_THREADS)
+ log.info GET(appmaster, SYSTEM_HEALTHCHECK)
+ log.info GET(appmaster, SYSTEM_METRICS_JSON)
- describe "Hadoop HTTP operations"
+ describe "Codahale operations"
// now switch to the Hadoop URL connection, with SPNEGO escalation
getWebPage(conf, appmaster)
- getWebPage(conf, appmaster, RestPaths.SYSTEM_THREADS)
- getWebPage(conf, appmaster, RestPaths.SYSTEM_HEALTHCHECK)
- getWebPage(conf, appmaster, RestPaths.SYSTEM_METRICS_JSON)
+ getWebPage(conf, appmaster, SYSTEM_THREADS)
+ getWebPage(conf, appmaster, SYSTEM_HEALTHCHECK)
+ getWebPage(conf, appmaster, SYSTEM_METRICS_JSON)
- log.info getWebPage(conf, realappmaster, RestPaths.SYSTEM_METRICS_JSON)
+ log.info getWebPage(conf, realappmaster, SYSTEM_METRICS_JSON)
+ // now some REST gets
+ describe "Application REST GETs"
+
+ getWebPage(conf, appmaster, SLIDER_PATH_APPLICATION + LIVE_RESOURCES)
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e654a0f2/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index e612aa3..d3cbae0 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -23,18 +23,21 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.commons.httpclient.HttpClient
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager
+import org.apache.commons.httpclient.URI
import org.apache.commons.httpclient.methods.GetMethod
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileStatus
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
import org.apache.hadoop.hdfs.web.URLConnectionFactory
-import org.apache.hadoop.io.IOUtils
+import org.apache.hadoop.net.NetUtils
import org.apache.hadoop.service.ServiceStateException
import org.apache.hadoop.util.Shell
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.hadoop.registry.client.types.ServiceRecord
+import org.apache.hadoop.yarn.webapp.ForbiddenException
+import org.apache.hadoop.yarn.webapp.NotFoundException
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ClusterNode
import org.apache.slider.api.RoleKeys
@@ -480,17 +483,44 @@ class SliderTestUtils extends Assert {
def client = new HttpClient(new MultiThreadedHttpConnectionManager());
client.httpConnectionManager.params.connectionTimeout = 10000;
GetMethod get = new GetMethod(url);
+ URI destURI = get.getURI()
+ assert destURI.port != 0
+ assert destURI.host
+
get.followRedirects = true;
- int resultCode = client.executeMethod(get);
+ int resultCode
+ try {
+ resultCode = client.executeMethod(get);
+ } catch (IOException e) {
+ throw NetUtils.wrapException(url, 0, null, 0, e)
+ }
def body = get.responseBodyAsString
+
+ uprateFaults(url, resultCode, body)
+ return body;
+ }
+
+ /**
+ * uprate some faults
+ * @param url
+ * @param resultCode
+ * @param body
+ */
+ public static void uprateFaults(String url, int resultCode, String body) {
+
+ if (resultCode == 404) {
+ throw new NotFoundException(url)
+ }
+ if (resultCode == 401) {
+ throw new ForbiddenException(url)
+ }
if (!(resultCode >= 200 && resultCode < 400)) {
def message = "Request to $url failed with exit code $resultCode, body length ${body?.length()}:\n$body"
log.error(message)
- fail(message)
+ throw new IOException(message)
}
- return body;
}
/**
@@ -508,7 +538,30 @@ class SliderTestUtils extends Assert {
return getWebPage(conf, s)
}
- /**
+ /**
+ * Execute any of the http requests, swallowing exceptions until
+ * eventually they time out
+ * @param timeout
+ * @param operation
+ * @return
+ */
+ public static String execHttpRequest(int timeout, Closure operation) {
+ Duration duration = new Duration(timeout).start()
+ Exception ex = new IOException("limit exceeded before starting");
+ while (!duration.limitExceeded) {
+ try {
+ String result = operation();
+ return result;
+ } catch (Exception e) {
+ ex = e;
+ sleep(1000)
+ }
+ }
+ // timeout
+ throw ex;
+ }
+
+ /**
* Fetches a web page asserting that the response code is between 200 and 400.
* Will error on 400 and 500 series response codes and let 200 and 300 through.
*
@@ -523,32 +576,34 @@ class SliderTestUtils extends Assert {
URLConnectionFactory connectionFactory = URLConnectionFactory
.newDefaultURLConnectionFactory(conf);
URL url = new URL(page)
- HttpURLConnection conn =
- (HttpURLConnection) connectionFactory.openConnection(url);
+ assert url.port != 0
+ HttpURLConnection conn = null;
+ int resultCode = 0
+ def body = ""
try {
+ conn = (HttpURLConnection) connectionFactory.openConnection(url);
conn.instanceFollowRedirects = true;
conn.connect()
- int resultCode = conn.responseCode
+ resultCode = conn.responseCode
InputStream stream = conn.errorStream;
if (stream == null) {
stream = conn.inputStream;
}
- def body = stream ? stream.text : "(no body)"
- if (!(resultCode >= 200 && resultCode < 400)) {
- def message = "Request to $url failed with ${conn.responseMessage}, body length ${body?.length()}:\n$body"
- log.error(message)
- fail(message)
- }
- return body;
+ body = stream ? stream.text : "(no body)"
+ } catch (IOException e) {
+ throw NetUtils.wrapException(url.toString(), 0, null, 0, e)
} finally {
conn?.disconnect()
-
}
+ uprateFaults(page, resultCode, body)
+ return body;
}
+
+
- /**
+/**
* Assert that a service operation succeeded
* @param service service
*/
[11/26] incubator-slider git commit: SLIDER-724: avoid startup race
condition on web/REST by delaying webapp launch
Posted by st...@apache.org.
SLIDER-724: avoid startup race condition on web/REST by delaying webapp launch
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fc7c363c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fc7c363c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fc7c363c
Branch: refs/heads/develop
Commit: fc7c363c7fab56c2fcd2d8bddaa378ab21a189e3
Parents: b09bd84
Author: Steve Loughran <st...@apache.org>
Authored: Mon Dec 15 14:28:05 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Dec 16 20:25:40 2014 +0000
----------------------------------------------------------------------
.../server/appmaster/SliderAppMaster.java | 84 +++++++++++++-------
1 file changed, 56 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc7c363c/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 05f2faa..f5d9f03 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -151,9 +151,7 @@ import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.WebAppApiImpl;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
import org.apache.slider.server.services.security.CertificateManager;
-//import org.apache.slider.server.services.security.FsDelegationTokenManager;
import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
-import org.apache.slider.server.appmaster.management.MetricsBindingService;
import org.apache.slider.server.services.utility.WebAppService;
import org.apache.slider.server.services.workflow.ServiceThreadFactory;
import org.apache.slider.server.services.workflow.WorkflowExecutorService;
@@ -381,6 +379,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
private RegisterApplicationMasterResponse amRegistrationData;
private PortScanner portScanner;
private SecurityConfiguration securityConfiguration;
+ private int webAppPort;
/**
* Service Constructor
@@ -715,32 +714,12 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
startAgentWebApp(appInformation, serviceConf);
- int port = getPortToRequest(instanceDefinition);
-
- WebAppApi webAppApi = new WebAppApiImpl(this,
- stateForProviders,
- providerService,
- certificateManager,
- registryOperations,
- metricsAndMonitoring);
- webApp = new SliderAMWebApp(webAppApi);
- WebApps.$for(SliderAMWebApp.BASE_PATH, WebAppApi.class,
- webAppApi,
- RestPaths.WS_CONTEXT)
- .withHttpPolicy(serviceConf, HttpConfig.Policy.HTTP_ONLY)
- .at(port)
- .start(webApp);
+ webAppPort = getPortToRequest(instanceDefinition);
String scheme = WebAppUtils.HTTP_PREFIX;
- appMasterTrackingUrl = scheme + appMasterHostname + ":" + webApp.port();
- WebAppService<SliderAMWebApp> webAppService =
- new WebAppService<SliderAMWebApp>("slider", webApp);
-
- webAppService.init(serviceConf);
- webAppService.start();
- addService(webAppService);
+ appMasterTrackingUrl = scheme + appMasterHostname + ":" + webAppPort;
appInformation.put(StatusKeys.INFO_AM_WEB_URL, appMasterTrackingUrl + "/");
- appInformation.set(StatusKeys.INFO_AM_WEB_PORT, webApp.port());
+ appInformation.set(StatusKeys.INFO_AM_WEB_PORT, webAppPort);
// Register self with ResourceManager
// This will start heartbeating to the RM
@@ -828,8 +807,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
appState.buildAppMasterNode(appMasterContainerID,
appMasterHostname,
- webApp.port(),
- appMasterHostname + ":" + webApp.port());
+ webAppPort,
+ appMasterHostname + ":" + webAppPort);
// build up environment variables that the AM wants set in every container
// irrespective of provider and role.
@@ -901,6 +880,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
startQueueProcessing();
+ deployWebApplication(serviceConf, webAppPort);
+
+
// Start the Slider AM provider
sliderAMProvider.start();
@@ -918,6 +900,40 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
return finish();
}
+ /**
+ * Deploy the web application.
+ * <p>
+ * Creates and starts the web application, and adds a
+ * <code>WebAppService</code> service under the AM, to ensure
+ * a managed web application shutdown.
+ *
+ * @param serviceConf AM configuration
+ * @param port port to deploy the web application on
+ */
+ private void deployWebApplication(Configuration serviceConf, int port) {
+ WebAppApi webAppApi = new WebAppApiImpl(this,
+ stateForProviders,
+ providerService,
+ certificateManager,
+ registryOperations,
+ metricsAndMonitoring);
+ webApp = new SliderAMWebApp(webAppApi);
+ WebApps.$for(SliderAMWebApp.BASE_PATH,
+ WebAppApi.class,
+ webAppApi,
+ RestPaths.WS_CONTEXT)
+ .withHttpPolicy(serviceConf, HttpConfig.Policy.HTTP_ONLY)
+ .at(port)
+ .start(webApp);
+
+ WebAppService<SliderAMWebApp> webAppService =
+ new WebAppService<SliderAMWebApp>("slider", webApp);
+
+ webAppService.init(serviceConf);
+ webAppService.start();
+ addService(webAppService);
+ }
+
private void processAMCredentials(SecurityConfiguration securityConfiguration)
throws IOException {
// process the initial user to obtain the set of user
@@ -949,6 +965,18 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
containerCredentials = credentials;
}
+ /**
+ * Locate a port to request for a service such as RPC or web/REST.
+ * This uses port range definitions in the <code>instanceDefinition</code>
+ * to fix the port range —if one is set.
+ * <p>
+ * The port returned is available at the time of the request; there are
+ * no guarantees as to how long that situation will last.
+ * @param instanceDefinition instance definition containing port range
+ * restrictions in in the application configuration
+ * @return the port to request.
+ * @throws SliderException
+ */
private int getPortToRequest(AggregateConf instanceDefinition)
throws SliderException {
int portToRequest = 0;
@@ -984,7 +1012,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
fs.getFileSystem().setPermission(destPath,
- new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE));
+ new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE));
}
protected void login(String principal, File localKeytabFile)