You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@storm.apache.org by govind-menon <gi...@git.apache.org> on 2018/07/05 08:44:45 UTC
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
GitHub user govind-menon opened a pull request:
https://github.com/apache/storm/pull/2752
Storm 1311 Migration of UI from clj to Java
This PR basically duplicates the entire UI core in Java as an embedded Jetty Server. The follow on PR will delete the clj code and substitute the cli command (I'm separating it so they can be compared more easily).
You can merge this pull request into a Git repository by running:
$ git pull https://github.com/govind-menon/storm STORM-1311-Jetty
Alternatively you can review and apply these changes as the patch at:
https://github.com/apache/storm/pull/2752.patch
To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:
This closes #2752
----
commit 35409496e24b76e6d5f628da0b399582b8a70d8b
Author: Govind Menon <go...@...>
Date: 2018-05-29T19:57:41Z
STORM-1311: UI Migration from clj to java
commit fc6498cac5136bb5d29d140ff1b411c2205a38e0
Author: Govind Menon <go...@...>
Date: 2018-06-01T17:41:35Z
STORM-1311: Using Jetty instead of Grizzly
commit dec452f51a61a4e8e787666d11ddd4cc5e9fee30
Author: Govind Menon <go...@...>
Date: 2018-06-27T05:17:50Z
STORM-1311: Adding all GET routes
commit 75529bb8cb7db7b1cdd50445db675e5d5741d66e
Author: Govind Menon <go...@...>
Date: 2018-06-27T15:30:40Z
STORM-1311: Leftover from merge
commit 2f5893ba1bf252f769a2b47c41315bd3bd1414a8
Author: Govind Menon <go...@...>
Date: 2018-07-05T08:20:31Z
STORM-1311: Added rest of puts and made static file serving configuration generic
commit 66ebc86e4214f4cfce3c67b3080737805d72e312
Author: Govind Menon <go...@...>
Date: 2018-07-05T08:43:18Z
STORM-1311: Cleaning up POM files
----
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200474709
--- Diff: storm-core/src/jvm/org/apache/storm/ui/resources/StormApiResource.java ---
@@ -0,0 +1,741 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.resources;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+import javax.ws.rs.core.UriInfo;
+
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.NotAliveException;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.ui.UIHelpers;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Root resource (exposed at "storm" path).
+ */
+@Path("/")
+public class StormApiResource {
+
+ public static final Logger LOG = LoggerFactory.getLogger(StormApiResource.class);
+
+ @Context
+ private UriInfo info;
+
+ @Context
+ private HttpServletRequest servletRequest;
+
+
+ @Context
+ private ServletContext servletContext;
+
+ public static Map<String, Object> config = Utils.readStormConfig();
+ /**
+ * /api/v1/cluster/configuration -> nimbus configuration.
+ */
+
+ @GET
+ @Path("/cluster/configuration")
+ @Produces("application/json")
+ public Response getClusterConfiguration() {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ return Response.status(Response.Status.OK).entity(nimbusClient.getClient().getNimbusConf()).build();
+ } catch (AuthorizationException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.UNAUTHORIZED).build();
+ } catch (TException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ /**
+ * /api/v1/cluster/summary -> cluster summary.
+ */
+ @GET
+ @Path("/cluster/summary")
+ @Produces("application/json")
+ public Response getClusterSummary() {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ String user = servletRequest.getRemoteUser();
+ return Response.status(Response.Status.OK).entity(JSONValue.toJSONString(
+ UIHelpers.getClusterSummary(
+ nimbusClient.getClient().getClusterInfo(), user, config))).build();
+ } catch (AuthorizationException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.UNAUTHORIZED).build();
+
+ } catch (TException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ /**
+ * /api/v1/nimbus/summary -> nimbus summary.
+ */
+ @GET
+ @Path("/nimbus/summary")
+ @Produces("application/json")
+ public Response getNimbusSummary() {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ String user = servletRequest.getRemoteUser();
+ return Response.status(Response.Status.OK).entity(JSONValue.toJSONString(
+ UIHelpers.getNimbusSummary(
+ nimbusClient.getClient().getClusterInfo(), config))).build();
+ } catch (AuthorizationException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.UNAUTHORIZED).build();
+ } catch (TException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ }
+
+ }
+
+ /**
+ * /api/v1/owner-resources -> owner resources.
+ */
+ @GET
+ @Path("/owner-resources")
+ @Produces("application/json")
+ public Response getOwnerResources() {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ return Response.status(Response.Status.OK).entity(JSONValue.toJSONString(
+ UIHelpers.getOwnerResourceSummaries(
+ nimbusClient.getClient().getOwnerResourceSummaries(null), config))).build();
+ } catch (AuthorizationException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.UNAUTHORIZED).build();
+ } catch (TException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ /**
+ * /api/v1/owner-resources/:id -> owner resources.
+ */
+ @GET
+ @Path("/owner-resources/{id}")
+ @Produces("application/json")
+ public Response getOwnerResource(@PathParam("id") String id) {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ return Response.status(Response.Status.OK).entity(JSONValue.toJSONString(
+ UIHelpers.getOwnerResourceSummary(
+ nimbusClient.getClient().getOwnerResourceSummaries(id),nimbusClient.getClient(),
+ id, config))).build();
+ } catch (AuthorizationException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.UNAUTHORIZED).build();
+ } catch (TException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ /**
+ * /api/v1/history/summary -> topo history.
+ */
+ @GET
+ @Path("/history/summary")
+ @Produces("application/json")
+ public Response getHistorySummary() {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ String user = servletRequest.getRemoteUser();
+ return Response.status(Response.Status.OK).entity(JSONValue.toJSONString(
+ UIHelpers.getTopologyHistoryInfo(
+ nimbusClient.getClient().getTopologyHistory(user)))).build();
+ } catch (AuthorizationException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.UNAUTHORIZED).build();
+ } catch (TException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ /**
+ * /api/v1/supervisor/summary -> topo history.
+ */
+ @GET
+ @Path("/supervisor/summary")
+ @Produces("application/json")
+ public Response getSupervisorSummary(@Context SecurityContext securityContext) {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ return Response.status(Response.Status.OK).entity(JSONValue.toJSONString(
+ UIHelpers.getSupervisorSummary(
+ nimbusClient.getClient().getClusterInfo().get_supervisors(),
+ securityContext, config
+ ))).build();
+ } catch (AuthorizationException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.UNAUTHORIZED).build();
+ } catch (TException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ /**
+ * /api/v1/supervisor -> topo history.
+ */
+ @GET
+ @Path("/supervisor/{id}")
+ @Produces("application/json")
+ public Response getSupervisor(@PathParam("id") String id, @QueryParam("host") String host,
+ @QueryParam("sys") boolean sys) {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ return Response.status(Response.Status.OK).entity(JSONValue.toJSONString(
+ UIHelpers.getSupervisorPageInfo(
--- End diff --
The resulting map is missing logviewerPort and logviewerScheme entries.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200406284
--- Diff: storm-core/src/jvm/org/apache/storm/ui/UIHelpers.java ---
@@ -59,6 +116,12 @@
new Object[]{ "d", null }
};
+ /**
+ * Prettigy uptime string.
--- End diff --
nit: prettify?
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200431722
--- Diff: storm-core/src/jvm/org/apache/storm/ui/resources/StormApiResource.java ---
@@ -0,0 +1,741 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.resources;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+import javax.ws.rs.core.UriInfo;
+
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.NotAliveException;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.ui.UIHelpers;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Root resource (exposed at "storm" path).
+ */
+@Path("/")
+public class StormApiResource {
+
+ public static final Logger LOG = LoggerFactory.getLogger(StormApiResource.class);
+
+ @Context
+ private UriInfo info;
--- End diff --
This is not used.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200407479
--- Diff: storm-core/src/jvm/org/apache/storm/ui/UIHelpers.java ---
@@ -77,6 +140,11 @@ public static String prettyUptimeStr(String val, Object[][] dividers) {
return Joiner.on(" ").join(tmp);
}
+ /**
+ * See above javadoc.
--- End diff --
nit: This takes as long to write as `Prettify uptime string.` so can we just use it instead?
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200440315
--- Diff: storm-core/src/jvm/org/apache/storm/ui/resources/StormApiResource.java ---
@@ -0,0 +1,741 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.resources;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+import javax.ws.rs.core.UriInfo;
+
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.NotAliveException;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.ui.UIHelpers;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Root resource (exposed at "storm" path).
+ */
+@Path("/")
+public class StormApiResource {
+
+ public static final Logger LOG = LoggerFactory.getLogger(StormApiResource.class);
+
+ @Context
+ private UriInfo info;
+
+ @Context
+ private HttpServletRequest servletRequest;
+
+
+ @Context
+ private ServletContext servletContext;
+
+ public static Map<String, Object> config = Utils.readStormConfig();
+ /**
+ * /api/v1/cluster/configuration -> nimbus configuration.
+ */
+
+ @GET
+ @Path("/cluster/configuration")
+ @Produces("application/json")
+ public Response getClusterConfiguration() {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ return Response.status(Response.Status.OK).entity(nimbusClient.getClient().getNimbusConf()).build();
--- End diff --
We are not handling the callback query parameter on any of the APIs, but it is everywhere in the old REST calls.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r204259528
--- Diff: storm-core/src/jvm/org/apache/storm/command/GetErrors.java ---
@@ -25,6 +25,11 @@
import org.json.simple.JSONValue;
public class GetErrors {
+ /**
+ * Only get errors for a topology.
+ * @param args Used to accept the topology name.
+ * @throws Exception Ignored.
--- End diff --
rewrite this to `@throws Exception on errors` on consistency with BasicDrpcClient, or just remove `throws Exception`.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r206238030
--- Diff: storm-webapp/src/main/java/org/apache/storm/daemon/ui/filters/AuthorizedUserFilter.java ---
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.ui.filters;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.security.Principal;
+import java.util.Map;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.ResourceInfo;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.ext.Provider;
+
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.StormCommon;
+import org.apache.storm.daemon.ui.resources.NimbusOp;
+import org.apache.storm.security.auth.IAuthorizer;
+import org.apache.storm.security.auth.ReqContext;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Provider
+public class AuthorizedUserFilter implements ContainerRequestFilter {
+
+ public static final Logger LOG = LoggerFactory.getLogger(AuthorizedUserFilter.class);
+ public static Map<String, Object> conf = Utils.readStormConfig();
+ public static IAuthorizer uiImpersonationHandler;
+ public static IAuthorizer uiAclHandler;
+
+ @Context private ResourceInfo resourceInfo;
+
+ static {
+ try {
+ uiImpersonationHandler = StormCommon.mkAuthorizationHandler(
+ (String) conf.get(DaemonConfig.NIMBUS_IMPERSONATION_AUTHORIZER), conf);
+ uiAclHandler = StormCommon.mkAuthorizationHandler(
+ (String) conf.get(DaemonConfig.NIMBUS_AUTHORIZER), conf);
+ } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
+ LOG.error("Error initializing AuthorizedUserFilter: ", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void filter(ContainerRequestContext containerRequestContext) throws IOException {
+ NimbusOp annotation = resourceInfo.getResourceMethod().getAnnotation(NimbusOp.class);
+ if (annotation == null) {
+ return;
+ }
+ String op = annotation.value();
+ if (op == null) {
+ return;
+ }
+
+ Map topoConf = null;
+ if (containerRequestContext.getUriInfo().getPathParameters().containsKey("id")) {
--- End diff --
It feels a bit like `?id=` is always the topology id is a bit of magic. Could we have a follow on JIRA to make it so each API can configure it, perhaps through the annotation?
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200476039
--- Diff: storm-core/src/jvm/org/apache/storm/ui/resources/StormApiResource.java ---
@@ -0,0 +1,741 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.resources;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+import javax.ws.rs.core.UriInfo;
+
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.NotAliveException;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.ui.UIHelpers;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Root resource (exposed at "storm" path).
+ */
+@Path("/")
+public class StormApiResource {
+
+ public static final Logger LOG = LoggerFactory.getLogger(StormApiResource.class);
+
+ @Context
+ private UriInfo info;
+
+ @Context
+ private HttpServletRequest servletRequest;
+
+
+ @Context
+ private ServletContext servletContext;
+
+ public static Map<String, Object> config = Utils.readStormConfig();
+ /**
+ * /api/v1/cluster/configuration -> nimbus configuration.
+ */
+
+ @GET
+ @Path("/cluster/configuration")
+ @Produces("application/json")
+ public Response getClusterConfiguration() {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ return Response.status(Response.Status.OK).entity(nimbusClient.getClient().getNimbusConf()).build();
+ } catch (AuthorizationException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.UNAUTHORIZED).build();
+ } catch (TException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ /**
+ * /api/v1/cluster/summary -> cluster summary.
+ */
+ @GET
+ @Path("/cluster/summary")
+ @Produces("application/json")
+ public Response getClusterSummary() {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ String user = servletRequest.getRemoteUser();
+ return Response.status(Response.Status.OK).entity(JSONValue.toJSONString(
+ UIHelpers.getClusterSummary(
+ nimbusClient.getClient().getClusterInfo(), user, config))).build();
+ } catch (AuthorizationException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.UNAUTHORIZED).build();
+
+ } catch (TException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ /**
+ * /api/v1/nimbus/summary -> nimbus summary.
+ */
+ @GET
+ @Path("/nimbus/summary")
+ @Produces("application/json")
+ public Response getNimbusSummary() {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ String user = servletRequest.getRemoteUser();
+ return Response.status(Response.Status.OK).entity(JSONValue.toJSONString(
+ UIHelpers.getNimbusSummary(
+ nimbusClient.getClient().getClusterInfo(), config))).build();
+ } catch (AuthorizationException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.UNAUTHORIZED).build();
+ } catch (TException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ }
+
+ }
+
+ /**
+ * /api/v1/owner-resources -> owner resources.
+ */
+ @GET
+ @Path("/owner-resources")
+ @Produces("application/json")
+ public Response getOwnerResources() {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ return Response.status(Response.Status.OK).entity(JSONValue.toJSONString(
+ UIHelpers.getOwnerResourceSummaries(
+ nimbusClient.getClient().getOwnerResourceSummaries(null), config))).build();
+ } catch (AuthorizationException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.UNAUTHORIZED).build();
+ } catch (TException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ /**
+ * /api/v1/owner-resources/:id -> owner resources.
+ */
+ @GET
+ @Path("/owner-resources/{id}")
+ @Produces("application/json")
+ public Response getOwnerResource(@PathParam("id") String id) {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ return Response.status(Response.Status.OK).entity(JSONValue.toJSONString(
+ UIHelpers.getOwnerResourceSummary(
+ nimbusClient.getClient().getOwnerResourceSummaries(id),nimbusClient.getClient(),
+ id, config))).build();
+ } catch (AuthorizationException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.UNAUTHORIZED).build();
+ } catch (TException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ /**
+ * /api/v1/history/summary -> topo history.
+ */
+ @GET
+ @Path("/history/summary")
+ @Produces("application/json")
+ public Response getHistorySummary() {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ String user = servletRequest.getRemoteUser();
+ return Response.status(Response.Status.OK).entity(JSONValue.toJSONString(
+ UIHelpers.getTopologyHistoryInfo(
+ nimbusClient.getClient().getTopologyHistory(user)))).build();
+ } catch (AuthorizationException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.UNAUTHORIZED).build();
+ } catch (TException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ /**
+ * /api/v1/supervisor/summary -> topo history.
+ */
+ @GET
+ @Path("/supervisor/summary")
+ @Produces("application/json")
+ public Response getSupervisorSummary(@Context SecurityContext securityContext) {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ return Response.status(Response.Status.OK).entity(JSONValue.toJSONString(
+ UIHelpers.getSupervisorSummary(
+ nimbusClient.getClient().getClusterInfo().get_supervisors(),
+ securityContext, config
+ ))).build();
+ } catch (AuthorizationException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.UNAUTHORIZED).build();
+ } catch (TException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ /**
+ * /api/v1/supervisor -> topo history.
+ */
+ @GET
+ @Path("/supervisor/{id}")
+ @Produces("application/json")
+ public Response getSupervisor(@PathParam("id") String id, @QueryParam("host") String host,
+ @QueryParam("sys") boolean sys) {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ return Response.status(Response.Status.OK).entity(JSONValue.toJSONString(
+ UIHelpers.getSupervisorPageInfo(
+ nimbusClient.getClient().getSupervisorPageInfo(id, host, sys),
+ config
+ ))).build();
+ } catch (AuthorizationException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.UNAUTHORIZED).build();
+
+ } catch (NotAliveException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ } catch (TException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ /**
+ * /api/v1/topology/summary -> topo history.
+ */
+ @GET
+ @Path("/topology/summary")
+ @Produces("application/json")
+ public Response getTopologySummary(@Context SecurityContext securityContext) {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ return Response.status(Response.Status.OK).entity(JSONValue.toJSONString(
+ UIHelpers.getAllTopologiesSummary(
+ nimbusClient.getClient().getClusterInfo().get_topologies(),
+ config
+ ))).build();
+ } catch (AuthorizationException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.UNAUTHORIZED).build();
+ } catch (TException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ /**
+ * /api/v1/topology -> topo.
+ */
+ @GET
+ @Path("/topology/{id}")
+ @Produces("application/json")
+ public Response getTopology(@PathParam("id") String id,
+ @DefaultValue(":all-time") @QueryParam("window") String window,
+ @QueryParam("sys") boolean sys) {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ return Response.status(Response.Status.OK).entity(JSONValue.toJSONString(
+ UIHelpers.getTopologySummary(
+ nimbusClient.getClient().getTopologyPageInfo(
+ id, window, sys),
+ window, config,
+ servletRequest.getRemoteUser()
+ ))).build();
+ } catch (AuthorizationException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.UNAUTHORIZED).build();
+
+ } catch (NotAliveException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ } catch (TException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ /**
+ * /api/v1/topology-workers/:id -> topo workers.
+ */
+ @GET
+ @Path("/topology-workers/{id}")
+ @Produces("application/json")
+ public Response getTopologyWorkers(@PathParam("id") String id) {
--- End diff --
We might want to test this API a few times. In the clojure we were URL decoding the id again. Not sure why, but this is something we should look into.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by govind-menon <gi...@git.apache.org>.
Github user govind-menon commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r201717451
--- Diff: storm-client/src/jvm/org/apache/storm/Constants.java ---
@@ -55,5 +55,7 @@
public static final String COMMON_ONHEAP_MEMORY_RESOURCE_NAME = "onheap.memory.mb";
public static final String COMMON_OFFHEAP_MEMORY_RESOURCE_NAME = "offheap.memory.mb";
public static final String COMMON_TOTAL_MEMORY_RESOURCE_NAME = "memory.mb";
+
+ public static final String STORM_API_URL_PREFIX = "/api/v1/";
--- End diff --
It's now used in the StormApiResource
---
[GitHub] storm issue #2752: Storm 1311 Migration of UI from clj to Java
Posted by govind-menon <gi...@git.apache.org>.
Github user govind-menon commented on the issue:
https://github.com/apache/storm/pull/2752
@HeartSaVioR I addressed your review comments. Currently fixing checkstyle.
---
[GitHub] storm issue #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on the issue:
https://github.com/apache/storm/pull/2752
I am seeing these when I run DRPC and the logviewer. They appear to still work so this is minor, but it would be nice to fix them in a follow on JIRA if you cannot do it quickly now.
```
WARNING: A provider org.apache.storm.daemon.logviewer.webapp.LogviewerResource registered in SERVER runtime does not implement any provider interfaces applicable in the SERVER runtime. Due to constraint configuration problems the provider org.apache.storm.daemon.logviewer.webapp.LogviewerResource will be ignored.
```
```
WARNING: A provider org.apache.storm.daemon.drpc.webapp.DRPCResource registered in SERVER runtime does not implement any provider interfaces applicable in the SERVER runtime. Due to constraint configuration problems the provider org.apache.storm.daemon.drpc.webapp.DRPCResource will be ignored.
```
Also I found that the exception mapping does not appear to be working. If you go to the topology page for a topology that is not up (i.e. http://localhost:8080/topology.html?id=wc-1-1533264135 ) I see a stack trace for the NotAliveException being printed out to stdout or stderr (not sure which)
```
NotAliveException(msg:wc-1-1533264135)
at org.apache.storm.generated.Nimbus$getTopologyConf_result$getTopologyConf_resultStandardScheme.read(Nimbus.java:45081)
at org.apache.storm.generated.Nimbus$getTopologyConf_result$getTopologyConf_resultStandardScheme.read(Nimbus.java:45059)
at org.apache.storm.generated.Nimbus$getTopologyConf_result.read(Nimbus.java:44993)
...
```
But the request is returning an HTML response
```
<html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
<title>Error 500 Request failed.</title>
</head>
<body><h2>HTTP ERROR 500</h2>
<p>Problem accessing /api/v1/topology/wc-1-1533264135. Reason:
<pre> Request failed.</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.7.v20170914</a><hr/>
</body>
</html>
```
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r206235453
--- Diff: storm-webapp/src/main/java/org/apache/storm/daemon/ui/UIServer.java ---
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.ui;
+
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.EnumSet;
+import java.util.Map;
+import javax.servlet.DispatcherType;
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.drpc.webapp.ReqContextFilter;
+import org.apache.storm.daemon.ui.exceptionmappers.AuthorizationExceptionMapper;
+import org.apache.storm.daemon.ui.exceptionmappers.NotAliveExceptionMapper;
+import org.apache.storm.daemon.ui.exceptionmappers.TExceptionMapper;
+import org.apache.storm.daemon.ui.filters.AuthorizedUserFilter;
+import org.apache.storm.daemon.ui.filters.HeaderResponseFilter;
+import org.apache.storm.daemon.ui.filters.HeaderResponseServletFilter;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.security.auth.IHttpCredentialsPlugin;
+import org.apache.storm.security.auth.ServerAuthUtils;
+import org.apache.storm.utils.Utils;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.servlet.DefaultServlet;
+import org.eclipse.jetty.servlet.FilterHolder;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+
+import org.glassfish.jersey.server.ResourceConfig;
+import org.glassfish.jersey.servlet.ServletContainer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+
+import static org.apache.storm.utils.ConfigUtils.FILE_SEPARATOR;
+import static org.apache.storm.utils.ConfigUtils.STORM_HOME;
+
+/**
+ * Main class.
+ *
+ */
+public class UIServer {
+
+ public static final Logger LOG = LoggerFactory.getLogger(UIServer.class);
+
+ public static final String STORM_API_URL_PREFIX = "/api/v1/";
+
+ /**
+ * addRequestContextFilter.
+ * @param context context
+ * @param configName configName
+ * @param conf conf
+ */
+ public static void addRequestContextFilter(ServletContextHandler context,
+ String configName, Map<String, Object> conf) {
+ IHttpCredentialsPlugin auth = ServerAuthUtils.getHttpCredentialsPlugin(conf, (String) conf.get(configName));
+ ReqContextFilter filter = new ReqContextFilter(auth);
+ context.addFilter(new FilterHolder(filter), "/*", EnumSet.allOf(DispatcherType.class));
+ }
+
+ /**
+ * main.
+ * @param args args
+ */
+ public static void main(String[] args) {
+
+ Server jettyServer = new Server();
+ ServerConnector connector = new ServerConnector(jettyServer);
+ // connector.setPort((Integer) conf.get(DaemonConfig.UI_PORT));
+ connector.setPort(4443);
--- End diff --
Need to remove the hard coded port number
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200408787
--- Diff: storm-core/src/jvm/org/apache/storm/ui/UIServer.java ---
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui;
+
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.drpc.webapp.ReqContextFilter;
+import org.apache.storm.security.auth.IHttpCredentialsPlugin;
+import org.apache.storm.security.auth.ServerAuthUtils;
+import org.apache.storm.ui.filters.AuthorizedUserFilter;
+import org.apache.storm.utils.Utils;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.servlet.DefaultServlet;
+import org.eclipse.jetty.servlet.FilterHolder;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+
+import org.glassfish.jersey.server.ResourceConfig;
+import org.glassfish.jersey.servlet.ServletContainer;
+
+
+import javax.servlet.DispatcherType;
+import java.util.EnumSet;
+import java.util.Map;
+
+/**
+ * Main class.
+ *
+ */
+public class UIServer {
+
+ public static void addRequestContextFilter(ServletContextHandler context,
+ String configName, Map<String, Object> conf) {
+ IHttpCredentialsPlugin auth = ServerAuthUtils.getHttpCredentialsPlugin(conf, (String) conf.get(configName));
+ ReqContextFilter filter = new ReqContextFilter(auth);
+ context.addFilter(new FilterHolder(filter), "/*", EnumSet.allOf(DispatcherType.class));
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ Map<String, Object> conf = Utils.readStormConfig();
+ Server jettyServer = new Server();
+ ServerConnector connector = new ServerConnector(jettyServer);
+ connector.setPort(4443);
+ //connector.setPort((Integer) conf.get(DaemonConfig.UI_PORT));
+ jettyServer.addConnector(connector);
+
+ ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
+ context.setContextPath("/");
+ jettyServer.setHandler(context);
+
+
+ ResourceConfig resourceConfig =
+ new ResourceConfig()
+ .packages("org.apache.storm.ui.resources")
+ .register(AuthorizedUserFilter.class);
+
+ ServletHolder jerseyServlet = new ServletHolder(new ServletContainer(resourceConfig));
+ jerseyServlet.setInitOrder(0);
+ context.addServlet(jerseyServlet, "/api/v1/*");
+ addRequestContextFilter(context, DaemonConfig.DRPC_HTTP_CREDS_PLUGIN, conf);
+
+ // add special pathspec of static content mapped to the homePath
+ ServletHolder holderHome = new ServletHolder("static-home", DefaultServlet.class);
+ holderHome.setInitParameter("resourceBase",UIServer.class.getProtectionDomain().getCodeSource().getLocation().toExternalForm() + "/WEB-INF/");
+ holderHome.setInitParameter("dirAllowed","true");
+ holderHome.setInitParameter("pathInfoOnly","true");
+ context.addServlet(holderHome,"/*");
+
+
+ // Lastly, the default servlet for root content (always needed, to satisfy servlet spec)
+ ServletHolder holderPwd = new ServletHolder("default", DefaultServlet.class);
+ holderPwd.setInitParameter("dirAllowed","true");
--- End diff --
question: does dirAllowed let someone do a '/foo/../../bar/' like URL? I assume that it does not, but I want to be sure.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r204258860
--- Diff: external/storm-opentsdb/pom.xml ---
@@ -36,7 +36,7 @@
</developers>
<properties>
- <jersey.version>2.23</jersey.version>
+ <jersey.version>2.27</jersey.version>
--- End diff --
If we would want to sync the jersey version to root pom, this property would not be needed.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200433651
--- Diff: storm-core/test/jvm/org/apache/storm/ui/StormApiResourceTest.java ---
@@ -0,0 +1,19 @@
+package org.apache.storm.ui;
+
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.WebTarget;
+
+import org.glassfish.grizzly.http.server.HttpServer;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+
+public class StormApiResourceTest {
+
+ private HttpServer server;
+ private WebTarget target;
--- End diff --
Is this just a place holder for a test or are we actually going to get some tests?
---
[GitHub] storm issue #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on the issue:
https://github.com/apache/storm/pull/2752
localhost nimbus still does not show up on the UI if it is the only nimbus.
I am also getting an error in the visualization.
When hitting http://localhost:4443/api/v1/topology/wc-1-1532972479/visualization?sys=true it is causing an error on the nimbus side looking for the wrong blob (looks like it somehow think the topology is called `:all-time`).
```
2018-07-30 12:43:17.361 o.a.s.t.ProcessFunction pool-21-thread-17 [ERROR] Internal error processing getTopologyInfoWithOpts
java.lang.RuntimeException: java.lang.IllegalArgumentException: :all-time-stormconf.ser does not appear to be a valid blob key
at org.apache.storm.daemon.nimbus.Nimbus.getTopologyInfoWithOpts(Nimbus.java:3912) ~[storm-server-2.0.0-SNAPSHOT.jar:2.0.0-SNAPSHOT]
at org.apache.storm.generated.Nimbus$Processor$getTopologyInfoWithOpts.getResult(Nimbus.java:4576) ~[storm-client-2.0.0-SNAPSHOT.jar:2.0.0-SNAPSHOT]
at org.apache.storm.generated.Nimbus$Processor$getTopologyInfoWithOpts.getResult(Nimbus.java:4555) ~[storm-client-2.0.0-SNAPSHOT.jar:2.0.0-SNAPSHOT]
at org.apache.storm.thrift.ProcessFunction.process(ProcessFunction.java:38) [shaded-deps-2.0.0-SNAPSHOT.jar:2.0.0-SNAPSHOT]
at org.apache.storm.thrift.TBaseProcessor.process(TBaseProcessor.java:39) [shaded-deps-2.0.0-SNAPSHOT.jar:2.0.0-SNAPSHOT]
at org.apache.storm.security.auth.SimpleTransportPlugin$SimpleWrapProcessor.process(SimpleTransportPlugin.java:169) [storm-client-2.0.0-SNAPSHOT.jar:2.0.0-SNAPSHOT]
at org.apache.storm.thrift.server.AbstractNonblockingServer$FrameBuffer.invoke(AbstractNonblockingServer.java:518) [shaded-deps-2.0.0-SNAPSHOT.jar:2.0.0-SNAPSHOT]
at org.apache.storm.thrift.server.Invocation.run(Invocation.java:18) [shaded-deps-2.0.0-SNAPSHOT.jar:2.0.0-SNAPSHOT]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_171]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_171]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_171]
Caused by: java.lang.IllegalArgumentException: :all-time-stormconf.ser does not appear to be a valid blob key
at org.apache.storm.blobstore.BlobStore.validateKey(BlobStore.java:63) ~[storm-client-2.0.0-SNAPSHOT.jar:2.0.0-SNAPSHOT]
at org.apache.storm.blobstore.LocalFsBlobStore.getBlob(LocalFsBlobStore.java:387) ~[storm-server-2.0.0-SNAPSHOT.jar:2.0.0-SNAPSHOT]
at org.apache.storm.blobstore.BlobStore.readBlobTo(BlobStore.java:307) ~[storm-client-2.0.0-SNAPSHOT.jar:2.0.0-SNAPSHOT]
at org.apache.storm.blobstore.BlobStore.readBlob(BlobStore.java:336) ~[storm-client-2.0.0-SNAPSHOT.jar:2.0.0-SNAPSHOT]
at org.apache.storm.daemon.nimbus.TopoCache.readTopoConf(TopoCache.java:150) ~[storm-server-2.0.0-SNAPSHOT.jar:2.0.0-SNAPSHOT]
at org.apache.storm.daemon.nimbus.Nimbus.readTopoConfAsNimbus(Nimbus.java:679) ~[storm-server-2.0.0-SNAPSHOT.jar:2.0.0-SNAPSHOT]
at org.apache.storm.daemon.nimbus.Nimbus.tryReadTopoConf(Nimbus.java:840) ~[storm-server-2.0.0-SNAPSHOT.jar:2.0.0-SNAPSHOT]
at org.apache.storm.daemon.nimbus.Nimbus.getCommonTopoInfo(Nimbus.java:2716) ~[storm-server-2.0.0-SNAPSHOT.jar:2.0.0-SNAPSHOT]
at org.apache.storm.daemon.nimbus.Nimbus.getTopologyInfoWithOpts(Nimbus.java:3826) ~[storm-server-2.0.0-SNAPSHOT.jar:2.0.0-SNAPSHOT]
... 10 more
```
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r207542398
--- Diff: storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java ---
@@ -137,7 +137,7 @@ public Response searchLogs(@Context HttpServletRequest request) throws IOExcepti
String user = httpCredsHandler.getUserName(request);
String topologyId = request.getParameter("topoId");
String portStr = request.getParameter("port");
- String callback = request.getParameter("callback");
+ String callback = request.getParameter("callbackParameterName");
--- End diff --
So why are we changing `/searchLogs?callback=bar` to `/searchLogs?callbackParameterName=bar`? If that is what we are doing it is a regression, if not I am really missing something.
---
[GitHub] storm issue #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on the issue:
https://github.com/apache/storm/pull/2752
I am also getting no nimbus instances on the UI when running locally.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r207554122
--- Diff: storm-webapp/src/main/java/org/apache/storm/daemon/ui/UIHelpers.java ---
@@ -0,0 +1,1941 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF)
+ * under one or more contributor license agreements.
+ * See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+
+package org.apache.storm.daemon.ui;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import javax.servlet.DispatcherType;
+import javax.servlet.Servlet;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+
+import org.apache.storm.Config;
+import org.apache.storm.Constants;
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.generated.Bolt;
+import org.apache.storm.generated.BoltAggregateStats;
+import org.apache.storm.generated.ClusterSummary;
+import org.apache.storm.generated.CommonAggregateStats;
+import org.apache.storm.generated.ComponentAggregateStats;
+import org.apache.storm.generated.ComponentPageInfo;
+import org.apache.storm.generated.ExecutorInfo;
+import org.apache.storm.generated.ExecutorSummary;
+import org.apache.storm.generated.GetInfoOptions;
+import org.apache.storm.generated.GlobalStreamId;
+import org.apache.storm.generated.Grouping;
+import org.apache.storm.generated.KillOptions;
+import org.apache.storm.generated.LogConfig;
+import org.apache.storm.generated.LogLevel;
+import org.apache.storm.generated.LogLevelAction;
+import org.apache.storm.generated.Nimbus;
+import org.apache.storm.generated.NimbusSummary;
+import org.apache.storm.generated.NodeInfo;
+import org.apache.storm.generated.NumErrorsChoice;
+import org.apache.storm.generated.OwnerResourceSummary;
+import org.apache.storm.generated.ProfileAction;
+import org.apache.storm.generated.ProfileRequest;
+import org.apache.storm.generated.RebalanceOptions;
+import org.apache.storm.generated.SpoutSpec;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.generated.SupervisorPageInfo;
+import org.apache.storm.generated.SupervisorSummary;
+import org.apache.storm.generated.TopologyHistoryInfo;
+import org.apache.storm.generated.TopologyInfo;
+import org.apache.storm.generated.TopologyPageInfo;
+import org.apache.storm.generated.TopologyStats;
+import org.apache.storm.generated.TopologySummary;
+import org.apache.storm.generated.WorkerSummary;
+import org.apache.storm.logging.filters.AccessLoggingFilter;
+import org.apache.storm.stats.StatsUtil;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.TopologySpoutLag;
+import org.apache.storm.utils.Utils;
+import org.apache.storm.utils.VersionInfo;
+import org.apache.storm.utils.WebAppUtils;
+import org.eclipse.jetty.http.HttpStatus;
+import org.eclipse.jetty.http.HttpVersion;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.HttpConnectionFactory;
+import org.eclipse.jetty.server.SecureRequestCustomizer;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.SslConnectionFactory;
+import org.eclipse.jetty.servlet.FilterHolder;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.servlets.CrossOriginFilter;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.json.simple.JSONValue;
+
+public class UIHelpers {
+
+ private static final Object[][] PRETTY_SEC_DIVIDERS = {
+ new Object[]{ "s", 60 },
+ new Object[]{ "m", 60 },
+ new Object[]{ "h", 24 },
+ new Object[]{ "d", null }
+ };
+
+ private static final Object[][] PRETTY_MS_DIVIDERS = {
+ new Object[]{ "ms", 1000 },
+ new Object[]{ "s", 60 },
+ new Object[]{ "m", 60 },
+ new Object[]{ "h", 24 },
+ new Object[]{ "d", null }
+ };
+
+ /**
+ * Prettify uptime string.
+ * @param val val.
+ * @param dividers dividers.
+ * @return prettified uptime string.
+ */
+ public static String prettyUptimeStr(String val, Object[][] dividers) {
+ int uptime = Integer.parseInt(val);
+ LinkedList<String> tmp = new LinkedList<>();
+ for (Object[] divider : dividers) {
+ if (uptime > 0) {
+ String state = (String) divider[0];
+ Integer div = (Integer) divider[1];
+ if (div != null) {
+ tmp.addFirst(uptime % div + state);
+ uptime = uptime / div;
+ } else {
+ tmp.addFirst(uptime + state);
+ }
+ }
+ }
+ return Joiner.on(" ").join(tmp);
+ }
+
+ /**
+ * Prettify uptime string.
+ * @param sec uptime in seconds.
+ * @return prettified uptime string.
+ */
+ public static String prettyUptimeSec(String sec) {
+ return prettyUptimeStr(sec, PRETTY_SEC_DIVIDERS);
+ }
+
+ /**
+ * prettyUptimeSec.
+ * @param secs secs
+ * @return prettyUptimeSec
+ */
+ public static String prettyUptimeSec(int secs) {
+ return prettyUptimeStr(String.valueOf(secs), PRETTY_SEC_DIVIDERS);
+ }
+
+ /**
+ * prettyUptimeMs.
+ * @param ms ms
+ * @return prettyUptimeMs
+ */
+ public static String prettyUptimeMs(String ms) {
+ return prettyUptimeStr(ms, PRETTY_MS_DIVIDERS);
+ }
+
+ /**
+ * prettyUptimeMs.
+ * @param ms ms
+ * @return prettyUptimeMs
+ */
+ public static String prettyUptimeMs(int ms) {
+ return prettyUptimeStr(String.valueOf(ms), PRETTY_MS_DIVIDERS);
+ }
+
+ /**
+ * url formatter for log links.
+ * @param fmt string format
+ * @param args hostname and other arguments.
+ * @return string formatter
+ */
+ public static String urlFormat(String fmt, Object... args) {
+ String[] argsEncoded = new String[args.length];
+ for (int i = 0; i < args.length; i++) {
+ argsEncoded[i] = URLEncoder.encode(String.valueOf(args[i]));
+ }
+ return String.format(fmt, argsEncoded);
+ }
+
+ /**
+ * Prettified executor info.
+ * @param e from Nimbus call
+ * @return prettified executor info string
+ */
+ public static String prettyExecutorInfo(ExecutorInfo e) {
+ return "[" + e.get_task_start() + "-" + e.get_task_end() + "]";
+ }
+
+ /**
+ * Unauthorized user json.
+ * @param user User id.
+ * @return Unauthorized user json.
+ */
+ public static Map<String, Object> unauthorizedUserJson(String user) {
+ return ImmutableMap.of(
+ "error", "No Authorization",
+ "errorMessage", String.format("User %s is not authorized.", user));
+ }
+
+ private static ServerConnector mkSslConnector(Server server, Integer port, String ksPath,
+ String ksPassword, String ksType,
+ String keyPassword, String tsPath,
+ String tsPassword, String tsType,
+ Boolean needClientAuth, Boolean wantClientAuth,
+ Integer headerBufferSize) {
+ SslContextFactory factory = new SslContextFactory();
+ factory.setExcludeCipherSuites("SSL_RSA_WITH_RC4_128_MD5", "SSL_RSA_WITH_RC4_128_SHA");
+ factory.setExcludeProtocols("SSLv3");
+ factory.setRenegotiationAllowed(false);
+ factory.setKeyStorePath(ksPath);
+ factory.setKeyStoreType(ksType);
+ factory.setKeyStorePassword(ksPassword);
+ factory.setKeyManagerPassword(keyPassword);
+
+ if (tsPath != null && tsPassword != null && tsType != null) {
+ factory.setTrustStorePath(tsPath);
+ factory.setTrustStoreType(tsType);
+ factory.setTrustStorePassword(tsPassword);
+ }
+
+ if (needClientAuth != null && needClientAuth) {
+ factory.setNeedClientAuth(true);
+ } else if (wantClientAuth != null && wantClientAuth) {
+ factory.setWantClientAuth(true);
+ }
+
+ HttpConfiguration httpsConfig = new HttpConfiguration();
+ httpsConfig.addCustomizer(new SecureRequestCustomizer());
+ if (null != headerBufferSize) {
+ httpsConfig.setRequestHeaderSize(headerBufferSize);
+ }
+ ServerConnector sslConnector = new ServerConnector(
+ server,
+ new SslConnectionFactory(factory, HttpVersion.HTTP_1_1.asString()),
+ new HttpConnectionFactory(httpsConfig)
+ );
+ sslConnector.setPort(port);
+ return sslConnector;
+ }
+
+ public static void configSsl(Server server, Integer port, String ksPath,
+ String ksPassword, String ksType,
+ String keyPassword, String tsPath,
+ String tsPassword, String tsType,
+ Boolean needClientAuth, Boolean wantClientAuth) {
+ configSsl(server, port, ksPath, ksPassword, ksType, keyPassword,
+ tsPath, tsPassword, tsType, needClientAuth, wantClientAuth, null);
+ }
+
+ /**
+ * configSsl.
+ * @param server server
+ * @param port port
+ * @param ksPath ksPath
+ * @param ksPassword ksPassword
+ * @param ksType ksType
+ * @param keyPassword keyPassword
+ * @param tsPath tsPath
+ * @param tsPassword tsPassword
+ * @param tsType tsType
+ * @param needClientAuth needClientAuth
+ * @param wantClientAuth wantClientAuth
+ * @param headerBufferSize headerBufferSize
+ */
+ public static void configSsl(Server server, Integer port, String ksPath,
+ String ksPassword, String ksType,
+ String keyPassword, String tsPath,
+ String tsPassword, String tsType,
+ Boolean needClientAuth,
+ Boolean wantClientAuth, Integer headerBufferSize) {
+ if (port > 0) {
+ server.addConnector(
+ mkSslConnector(
+ server, port, ksPath, ksPassword, ksType, keyPassword,
+ tsPath, tsPassword, tsType,
+ needClientAuth, wantClientAuth, headerBufferSize
+ )
+ );
+ }
+ }
+
+ /**
+ * corsFilterHandle.
+ * @return corsFilterHandle
+ */
+ public static FilterHolder corsFilterHandle() {
+ FilterHolder filterHolder = new FilterHolder(new CrossOriginFilter());
+ filterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, "*");
+ filterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, "GET, POST, PUT");
+ filterHolder.setInitParameter(
+ CrossOriginFilter.ALLOWED_ORIGINS_PARAM,
+ "X-Requested-With, X-Requested-By, Access-Control-Allow-Origin,"
+ + " Content-Type, Content-Length, Accept, Origin");
+ filterHolder.setInitParameter(CrossOriginFilter.ACCESS_CONTROL_ALLOW_ORIGIN_HEADER, "*");
+ return filterHolder;
+ }
+
+ /**
+ * mkAccessLoggingFilterHandle.
+ * @return mkAccessLoggingFilterHandle
+ */
+ public static FilterHolder mkAccessLoggingFilterHandle() {
+
+ return new FilterHolder(new AccessLoggingFilter());
+
+ }
+
+ public static void configFilter(Server server, Servlet servlet, List<FilterConfiguration> filtersConfs) {
+ configFilter(server, servlet, filtersConfs, null);
+ }
+
+ /**
+ * Config filter.
+ * @param server Server
+ * @param servlet Servlet
+ * @param filtersConfs FiltersConfs
+ * @param params Filter params
+ */
+ public static void configFilter(Server server, Servlet servlet,
+ List<FilterConfiguration> filtersConfs,
+ Map<String, String> params) {
+ if (filtersConfs != null) {
+ ServletHolder servletHolder = new ServletHolder(servlet);
+ servletHolder.setInitOrder(0);
+ if (params != null) {
+ servletHolder.setInitParameters(params);
+ }
+ ServletContextHandler context = new ServletContextHandler(server, "/");
+ context.addServlet(servletHolder, "/");
+ configFilters(context, filtersConfs);
+ server.setHandler(context);
+ }
+ }
+
+ /**
+ * Config filters.
+ * @param context Servlet context
+ * @param filtersConfs filter confs
+ */
+ public static void configFilters(ServletContextHandler context,
+ List<FilterConfiguration> filtersConfs) {
+ context.addFilter(corsFilterHandle(), "/*", EnumSet.allOf(DispatcherType.class));
+ for (FilterConfiguration filterConf : filtersConfs) {
+ String filterName = filterConf.getFilterName();
+ String filterClass = filterConf.getFilterClass();
+ Map<String, String> filterParams = filterConf.getFilterParams();
+ if (filterClass != null) {
+ FilterHolder filterHolder = new FilterHolder();
+ filterHolder.setClassName(filterClass);
+ if (filterName != null) {
+ filterHolder.setName(filterName);
+ } else {
+ filterHolder.setName(filterClass);
+ }
+ if (filterParams != null) {
+ filterHolder.setInitParameters(filterParams);
+ } else {
+ filterHolder.setInitParameters(new HashMap<>());
+ }
+ context.addFilter(filterHolder, "/*", EnumSet.allOf(DispatcherType.class));
+ }
+ }
+ context.addFilter(mkAccessLoggingFilterHandle(), "/*", EnumSet.allOf(DispatcherType.class));
+ }
+
+ /**
+ * Construct a Jetty Server instance.
+ */
+ public static Server jettyCreateServer(Integer port, String host, Integer httpsPort) {
+ return jettyCreateServer(port, host, httpsPort, null);
+ }
+
+ /**
+ * Construct a Jetty Server instance.
+ */
+ public static Server jettyCreateServer(Integer port, String host,
+ Integer httpsPort, Integer headerBufferSize) {
+ Server server = new Server();
+
+ if (httpsPort == null || httpsPort <= 0) {
+ HttpConfiguration httpConfig = new HttpConfiguration();
+ httpConfig.setSendDateHeader(true);
+ if (null != headerBufferSize) {
+ httpConfig.setRequestHeaderSize(headerBufferSize);
+ }
+ ServerConnector httpConnector = new ServerConnector(
+ server, new HttpConnectionFactory(httpConfig)
+ );
+ httpConnector.setPort(ObjectReader.getInt(port, 80));
+ httpConnector.setIdleTimeout(200000);
+ httpConnector.setHost(host);
+ server.addConnector(httpConnector);
+ }
+
+ return server;
+ }
+
+ /**
+ * Modified version of run-jetty
+ * Assumes configurator sets handler.
+ */
+ public static void stormRunJetty(Integer port, String host,
+ Integer httpsPort, Integer headerBufferSize,
+ IConfigurator configurator) throws Exception {
+ Server s = jettyCreateServer(port, host, httpsPort, headerBufferSize);
+ if (configurator != null) {
+ configurator.execute(s);
+ }
+ s.start();
+ }
+
+ public static void stormRunJetty(Integer port, Integer headerBufferSize,
+ IConfigurator configurator) throws Exception {
+ stormRunJetty(port, null, null, headerBufferSize, configurator);
+ }
+
+ /**
+ * wrapJsonInCallback.
+ * @param callback callbackParameterName
+ * @param response response
+ * @return wrapJsonInCallback
+ */
+ public static String wrapJsonInCallback(String callback, String response) {
+ return callback + "(" + response + ");";
+ }
+
+ /**
+ * getJsonResponseHeaders.
+ * @param callback callbackParameterName
+ * @param headers headers
+ * @return getJsonResponseHeaders
+ */
+ public static Map getJsonResponseHeaders(String callback, Map headers) {
+ Map<String, String> headersResult = new HashMap<>();
+ headersResult.put("Cache-Control", "no-cache, no-store");
+ headersResult.put("Access-Control-Allow-Origin", "*");
+ headersResult.put("Access-Control-Allow-Headers",
+ "Content-Type, Access-Control-Allow-Headers, "
+ + "Access-Controler-Allow-Origin, "
+ + "X-Requested-By, X-Csrf-Token, "
+ + "Authorization, X-Requested-With");
+ if (callback != null) {
+ headersResult.put("Content-Type", "application/javascript;charset=utf-8");
+ } else {
+ headersResult.put("Content-Type", "application/json;charset=utf-8");
+ }
+ if (headers != null) {
+ headersResult.putAll(headers);
+ }
+ return headersResult;
+ }
+
+ public static String getJsonResponseBody(Object data, String callback, boolean needSerialize) {
+ String serializedData = needSerialize ? JSONValue.toJSONString(data) : (String) data;
+ return callback != null ? wrapJsonInCallback(callback, serializedData) : serializedData;
+ }
+
+ /**
+ * Converts exception into json map.
+ * @param ex Exception to be converted.
+ * @param statusCode Status code to be returned.
+ * @return Map to be converted into json.
+ */
+ public static Map exceptionToJson(Exception ex, int statusCode) {
+ StringWriter sw = new StringWriter();
+ ex.printStackTrace(new PrintWriter(sw));
+ return ImmutableMap.of(
+ "error", statusCode
+ + " "
+ + HttpStatus.getMessage(statusCode),
+ "errorMessage", sw.toString());
+ }
+
+ public static Response makeStandardResponse(Object data, String callback) {
+ return makeStandardResponse(data, callback, true, Response.Status.OK);
+ }
+
+ public static Response makeStandardResponse(Object data, String callback, Response.Status status) {
+ return makeStandardResponse(data, callback, true, status);
+ }
+
+ /**
+ * makeStandardResponse.
+ * @param data data
+ * @param callback callbackParameterName
+ * @param needsSerialization needsSerialization
+ * @return makeStandardResponse
+ */
+ public static Response makeStandardResponse(
+ Object data, String callback, boolean needsSerialization, Response.Status status) {
+ String body = getJsonResponseBody(data, callback, needsSerialization);
+ Response.ResponseBuilder responseBuilder = Response.status(status).entity(body);
+ Map<String, String> headers = getJsonResponseHeaders(callback, null);
+ for (Map.Entry<String, String> headerEntry: headers.entrySet()) {
+ responseBuilder.header(headerEntry.getKey(), headerEntry.getValue());
+ }
+ return responseBuilder.build();
+ }
+
+ /**
+ * Converts thrift call result into map fit for UI/api.
+ * @param clusterSummary Obtained from Nimbus.
+ * @param user User Making request
+ * @param conf Storm Conf
+ * @return Cluster Summary for display on UI/monitoring purposes via API
+ */
+ public static Map<String, Object> getClusterSummary(ClusterSummary clusterSummary, String user,
+ Map<String, Object> conf) {
+ Map<String, Object> result = new HashMap();
+ List<SupervisorSummary> supervisorSummaries = clusterSummary.get_supervisors();
+ List<TopologySummary> topologySummaries = clusterSummary.get_topologies();
+
+ int usedSlots =
+ supervisorSummaries.stream().mapToInt(
+ SupervisorSummary::get_num_used_workers).sum();
+ int totalSlots =
+ supervisorSummaries.stream().mapToInt(
+ SupervisorSummary::get_num_workers).sum();
+
+ int totalTasks =
+ topologySummaries.stream().mapToInt(
+ TopologySummary::get_num_tasks).sum();
+ int totalExecutors =
+ topologySummaries.stream().mapToInt(
+ TopologySummary::get_num_executors).sum();
+
+ double supervisorTotalMemory =
+ supervisorSummaries.stream().mapToDouble(x -> x.get_total_resources().getOrDefault(
+ Constants.COMMON_TOTAL_MEMORY_RESOURCE_NAME,
+ x.get_total_resources().get(Config.SUPERVISOR_MEMORY_CAPACITY_MB)
+ )
+ ).sum();
+
+ double supervisorTotalCpu =
+ supervisorSummaries.stream().mapToDouble(x -> x.get_total_resources().getOrDefault(
+ Constants.COMMON_CPU_RESOURCE_NAME,
+ x.get_total_resources().get(Config.SUPERVISOR_CPU_CAPACITY)
+ )
+ ).sum();
+
+ double supervisorUsedMemory =
+ supervisorSummaries.stream().mapToDouble(SupervisorSummary::get_used_mem).sum();
+ double supervisorUsedCpu =
+ supervisorSummaries.stream().mapToDouble(SupervisorSummary::get_used_cpu).sum();
+ double supervisorFragementedCpu =
+ supervisorSummaries.stream().mapToDouble(
+ SupervisorSummary::get_fragmented_cpu).sum();
+ double supervisorFragmentedMem =
+ supervisorSummaries.stream().mapToDouble(
+ SupervisorSummary::get_fragmented_mem).sum();
+
+
+ result.put("user", user);
+ result.put("stormVersion", VersionInfo.getVersion());
+ result.put("supervisors", supervisorSummaries.size());
+ result.put("topologies", clusterSummary.get_topologies_size());
+ result.put("slotsUsed", usedSlots);
+ result.put("slotsTotal", totalSlots);
+ result.put("slotsFree", totalSlots - usedSlots);
+ result.put("tasksTotal", totalTasks);
+ result.put("totalExecutors", totalExecutors);
+
+ result.put("totalMem", supervisorTotalMemory);
+ result.put("totalCpu", supervisorTotalCpu);
+ result.put("availMem", supervisorTotalMemory - supervisorUsedMemory);
+ result.put("availCpu", supervisorTotalCpu - supervisorUsedCpu);
+ result.put("fragmentedMem", supervisorFragmentedMem);
+ result.put("fragmentedCpu", supervisorFragementedCpu);
+ result.put("schedulerDisplayResource",
+ conf.get(DaemonConfig.SCHEDULER_DISPLAY_RESOURCE));
+ result.put("memAssignedPercentUtil", supervisorTotalMemory > 0
+ ? String.valueOf((supervisorTotalMemory - supervisorUsedMemory) * 100.0
+ / supervisorTotalMemory) : "0.0");
+ result.put("cpuAssignedPercentUtil", supervisorTotalCpu > 0
+ ? String.valueOf((supervisorTotalCpu - supervisorUsedCpu) * 100.0
+ / supervisorTotalCpu) : "0.0");
+ result.put("bugtracker-url", conf.get(DaemonConfig.UI_PROJECT_BUGTRACKER_URL));
+ result.put("central-log-url", conf.get(DaemonConfig.UI_CENTRAL_LOGGING_URL));
+ return result;
+ }
+
+ /**
+ * Prettify OwnerResourceSummary.
+ * @param ownerResourceSummary ownerResourceSummary
+ * @return Map of prettified OwnerResourceSummary.
+ */
+ public static Map<String, Object> unpackOwnerResourceSummary(
+ OwnerResourceSummary ownerResourceSummary) {
+
+ Double memoryGuarantee = Double.valueOf(-1);
+ if (ownerResourceSummary.is_set_memory_guarantee()) {
+ memoryGuarantee = ownerResourceSummary.get_memory_guarantee();
+ }
+
+ Double cpuGuaranteee = Double.valueOf(-1);
+ if (ownerResourceSummary.is_set_cpu_guarantee()) {
+ cpuGuaranteee = ownerResourceSummary.get_cpu_guarantee();
+ }
+
+ int isolatedNodeGuarantee = -1;
+ if (ownerResourceSummary.is_set_isolated_node_guarantee()) {
+ isolatedNodeGuarantee = ownerResourceSummary.get_isolated_node_guarantee();
+ }
+
+ Double memoryGuaranteeRemaining = Double.valueOf(-1);
+ if (ownerResourceSummary.is_set_memory_guarantee_remaining()) {
+ memoryGuaranteeRemaining = ownerResourceSummary.get_memory_guarantee_remaining();
+ }
+
+ Double cpuGuaranteeRemaining = Double.valueOf(-1);
+ if (ownerResourceSummary.is_set_cpu_guarantee_remaining()) {
+ cpuGuaranteeRemaining = ownerResourceSummary.get_cpu_guarantee_remaining();
+ }
+
+ Map<String, Object> result = new HashMap();
+ result.put("owner", ownerResourceSummary.get_owner());
+ result.put("totalTopologies", ownerResourceSummary.get_total_topologies());
+ result.put("totalExecutors", ownerResourceSummary.get_total_executors());
+ result.put("totalWorkers", ownerResourceSummary.get_total_workers());
+ result.put("totalTasks", ownerResourceSummary.get_total_tasks());
+ result.put("totalMemoryUsage", ownerResourceSummary.get_memory_usage());
+ result.put("totalCpuUsage", ownerResourceSummary.get_cpu_usage());
+
+ result.put("memoryGuarantee", memoryGuarantee != -1 ? memoryGuarantee : "N/A");
+ result.put("cpuGuarantee", cpuGuaranteee != -1 ? cpuGuaranteee : "N/A");
+ result.put("isolatedNodes", isolatedNodeGuarantee);
+
+ result.put("memoryGuaranteeRemaining",
+ memoryGuaranteeRemaining != -1 ? memoryGuaranteeRemaining : "N/A");
+ result.put("cpuGuaranteeRemaining",
+ cpuGuaranteeRemaining != -1 ? cpuGuaranteeRemaining : "N/A");
+ result.put("totalReqOnHeapMem", ownerResourceSummary.get_requested_on_heap_memory());
+
+ result.put("totalReqOffHeapMem", ownerResourceSummary.get_requested_off_heap_memory());
+
+ result.put("totalReqMem", ownerResourceSummary.get_requested_total_memory());
+ result.put("totalReqCpu", ownerResourceSummary.get_requested_cpu());
+ result.put("totalAssignedOnHeapMem",
+ ownerResourceSummary.get_assigned_on_heap_memory()
+ );
+ result.put("totalAssignedOffHeapMem", ownerResourceSummary.get_assigned_off_heap_memory());
+
+ return result;
+ }
+
+ /**
+ * Get prettified ownerResourceSummaries.
+ * @param ownerResourceSummaries ownerResourceSummaries from thrift call
+ * @param conf Storm conf
+ * @return map to be converted to json.
+ */
+ public static Map<String, Object> getOwnerResourceSummaries(
+ List<OwnerResourceSummary> ownerResourceSummaries, Map<String, Object> conf) {
+ Map<String, Object> result = new HashMap();
+
+ result.put("schedulerDisplayResource", conf.get(DaemonConfig.SCHEDULER_DISPLAY_RESOURCE));
+
+ List<Map<String, Object>> ownerSummaries = new ArrayList();
+
+ for (OwnerResourceSummary ownerResourceSummary : ownerResourceSummaries) {
+ ownerSummaries.add(unpackOwnerResourceSummary(ownerResourceSummary));
+ }
+ result.put("owners", ownerSummaries);
+ return result;
+ }
+
+ /**
+ * getTopologyMap.
+ * @param topologySummary topologySummary
+ * @return getTopologyMap
+ */
+ public static Map<String, Object> getTopologyMap(TopologySummary topologySummary) {
+ Map<String, Object> result = new HashMap();
+ result.put("id", topologySummary.get_id());
+ result.put("encodedId", URLEncoder.encode(topologySummary.get_id()));
+ result.put("owner", topologySummary.get_owner());
+ result.put("name", topologySummary.get_name());
+ result.put("status", topologySummary.get_status());
+ result.put("uptime", UIHelpers.prettyUptimeSec(topologySummary.get_uptime_secs()));
+ result.put("uptimeSeconds", topologySummary.get_uptime_secs());
+ result.put("tasksTotal", topologySummary.get_num_tasks());
+ result.put("workersTotal", topologySummary.get_num_workers());
+ result.put("executorsTotal", topologySummary.get_num_executors());
+ result.put("replicationCount", topologySummary.get_replication_count());
+ result.put("schedulerInfo", topologySummary.get_sched_status());
+ result.put("requestedMemOnHeap", topologySummary.get_requested_memonheap());
+ result.put("requestedMemOffHeap", topologySummary.get_requested_memoffheap());
+ result.put("requestedTotalMem",
+ topologySummary.get_requested_memoffheap()
+ + topologySummary.get_assigned_memonheap());
+ result.put("requestedCpu", topologySummary.get_requested_cpu());
+ result.put("assignedMemOnHeap", topologySummary.get_assigned_memonheap());
+ result.put("assignedMemOffHeap", topologySummary.get_assigned_memoffheap());
+ result.put("assignedTotalMem",
+ topologySummary.get_assigned_memoffheap()
+ + topologySummary.get_assigned_memonheap());
+ result.put("assignedCpu", topologySummary.get_assigned_cpu());
+ result.put("topologyVersion", topologySummary.get_topology_version());
+ result.put("stormVersion", topologySummary.get_storm_version());
+ return result;
+ }
+
+ /**
+ * Get a specific owner resource summary.
+ * @param ownerResourceSummaries Result from thrift call.
+ * @param client client
+ * @param id Owner id.
+ * @param config Storm conf.
+ * @return prettified owner resource summary.
+ */
+ public static Map<String, Object> getOwnerResourceSummary(
+ List<OwnerResourceSummary> ownerResourceSummaries,
+ Nimbus.Iface client, String id, Map<String, Object> config) throws TException {
+ Map<String, Object> result = new HashMap();
+ result.put("schedulerDisplayResource", config.get(DaemonConfig.SCHEDULER_DISPLAY_RESOURCE));
+
+ if (ownerResourceSummaries.isEmpty()) {
+ return unpackOwnerResourceSummary(new OwnerResourceSummary(id));
+ }
+
+ List<TopologySummary> topologies = null;
+ topologies = client.getClusterInfo().get_topologies();
+ List<Map> topologySummaries = getTopologiesMap(id, topologies);
+
+ result.putAll(unpackOwnerResourceSummary(ownerResourceSummaries.get(0)));
+ result.put("topologies", topologySummaries);
+
+ return result;
+ }
+
+ /**
+ * getTopologiesMap.
+ * @param id id
+ * @param topologies topologies
+ * @return getTopologiesMap
+ */
+ private static List<Map> getTopologiesMap(String id, List<TopologySummary> topologies) {
+ List<Map> topologySummaries = new ArrayList();
+
+ for (TopologySummary topologySummary : topologies) {
+ if (id == null || topologySummary.get_owner().equals(id)) {
+ topologySummaries.add(getTopologyMap(topologySummary));
+ }
+ }
+ return topologySummaries;
+ }
+
+ /**
+ * getLogviewerLink.
+ * @param host host
+ * @param fname fname
+ * @param config config
+ * @param port port
+ * @return getLogviewerLink.
+ */
+ public static String getLogviewerLink(String host, String fname,
+ Map<String, Object> config, int port) {
+ if (isSecureLogviewer(config)) {
+ return UIHelpers.urlFormat("https://%s:%s/api/v1/log?file=%s",
+ host, config.get(DaemonConfig.LOGVIEWER_HTTPS_PORT), fname);
+ } else {
+ return UIHelpers.urlFormat("http://%s:%s/api/v1/log?file=%s",
+ host, config.get(DaemonConfig.LOGVIEWER_PORT), fname);
+ }
+ }
+
+ /**
+ * Get log link to nimbus log.
+ * @param host nimbus host name
+ * @param config storm config
+ * @return log link.
+ */
+ public static String getNimbusLogLink(String host, Map<String, Object> config) {
+ if (isSecureLogviewer(config)) {
+ return UIHelpers.urlFormat("https://%s:%s/api/v1/daemonlog?file=nimbus.log",
+ host, config.get(DaemonConfig.LOGVIEWER_HTTPS_PORT));
+ }
+ return UIHelpers.urlFormat("http://%s:%s/api/v1/daemonlog?file=nimbus.log",
+ host, config.get(DaemonConfig.LOGVIEWER_PORT));
+ }
+
+ /**
+ * Get log link to supervisor log.
+ * @param host supervisor host name
+ * @param config storm config
+ * @return log link.
+ */
+ public static String getSupervisorLogLink(String host, Map<String, Object> config) {
+ if (isSecureLogviewer(config)) {
+ return UIHelpers.urlFormat("https://%s:%s/api/v1/daemonlog?file=supervisor.log",
+ host, config.get(DaemonConfig.LOGVIEWER_HTTPS_PORT));
+ }
+ return UIHelpers.urlFormat("http://%s:%s/api/v1/daemonlog?file=supervisor.log",
+ host, config.get(DaemonConfig.LOGVIEWER_PORT));
+ }
+
+ /**
+ * Get log link to supervisor log.
+ * @param host supervisor host name
+ * @param config storm config
+ * @return log link.
+ */
+ public static String getWorkerLogLink(String host, int port,
+ Map<String, Object> config, String topologyId) {
+ return getLogviewerLink(host,
+ WebAppUtils.logsFilename(
+ topologyId, String.valueOf(port)),
+ config, port
+ );
+ }
+
+ /**
+ * Get supervisor info in a map.
+ * @param supervisorSummary from nimbus call.
+ * @param config Storm config.
+ * @return prettified supervisor info map.
+ */
+ public static Map<String, Object> getPrettifiedSupervisorMap(
+ SupervisorSummary supervisorSummary,
+ Map<String, Object> config) {
+ Map<String, Object> result = new HashMap();
+
+ result.put("id", supervisorSummary.get_supervisor_id());
+ result.put("host", supervisorSummary.get_host());
+ result.put("uptime", UIHelpers.prettyUptimeSec(supervisorSummary.get_uptime_secs()));
+ result.put("uptimeSeconds", supervisorSummary.get_uptime_secs());
+ result.put("slotsTotal", supervisorSummary.get_num_workers());
+ result.put("slotsUsed", supervisorSummary.get_num_used_workers());
+ result.put("slotsFree",
+ Integer.max(supervisorSummary.get_num_workers()
+ - supervisorSummary.get_num_used_workers(), 0));
+ Map<String, Double> totalResources = supervisorSummary.get_total_resources();
+ Double totalMemory = totalResources.getOrDefault(
+ Constants.COMMON_TOTAL_MEMORY_RESOURCE_NAME,
+ totalResources.get(Config.SUPERVISOR_MEMORY_CAPACITY_MB)
+ );
+ result.put("totalMem",
+ totalMemory
+ );
+ Double totalCpu = totalResources.getOrDefault(
+ Constants.COMMON_CPU_RESOURCE_NAME,
+ totalResources.get(Config.SUPERVISOR_CPU_CAPACITY)
+ );
+ result.put("totalCpu",
+ totalCpu);
+ result.put("usedMem", supervisorSummary.get_used_mem());
+ result.put("usedCpu", supervisorSummary.get_used_cpu());
+ result.put(
+ "logLink",
+ getSupervisorLogLink(supervisorSummary.get_host(), config)
+ );
+ result.put("availMem", totalMemory - supervisorSummary.get_used_mem());
+ result.put("availCpu", totalCpu - supervisorSummary.get_used_cpu());
+ result.put("version", supervisorSummary.get_version());
+ return result;
+ }
+
+ /**
+ * Get topology history.
+ * @param topologyHistory from Nimbus call.
+ * @return map ready to be returned.
+ */
+ public static Map<String, Object> getTopologyHistoryInfo(TopologyHistoryInfo topologyHistory) {
+ Map<String, Object> result = new HashMap();
+ result.put("topo-history", topologyHistory.get_topo_ids());
+ return result;
+ }
+
+ /**
+ * Check if logviewer is secure.
+ * @param config Storm config.
+ * @return true if logiviwer is secure.
+ */
+ public static boolean isSecureLogviewer(Map<String, Object> config) {
+ if (config.containsKey(DaemonConfig.LOGVIEWER_HTTPS_PORT)) {
+ int logviewerPort = (int) config.get(DaemonConfig.LOGVIEWER_HTTPS_PORT);
+ if (logviewerPort >= 0) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Get logviewer port depending on whether the logviewer is secure or not.
+ * @param config Storm config.
+ * @return appropriate port.
+ */
+ public static int getLogviewerPort(Map<String, Object> config) {
+ if (isSecureLogviewer(config)) {
+ return (int) config.get(DaemonConfig.LOGVIEWER_HTTPS_PORT);
+ }
+ return (int) config.get(DaemonConfig.LOGVIEWER_PORT);
+ }
+
+ /**
+ * getWorkerSummaries.
+ * @param supervisorPageInfo supervisorPageInfo
+ * @param config config
+ * @return getWorkerSummaries
+ */
+ public static List<Map> getWorkerSummaries(SupervisorPageInfo supervisorPageInfo,
+ Map<String, Object> config) {
+ List<Map> workerSummaries = new ArrayList();
+ if (supervisorPageInfo.is_set_worker_summaries()) {
+ for (WorkerSummary workerSummary : supervisorPageInfo.get_worker_summaries()) {
+ workerSummaries.add(getWorkerSummaryMap(workerSummary, config));
+ }
+ }
+ return workerSummaries;
+ }
+
+ /**
+ * getWorkerSummaryMap.
+ * @param workerSummary workerSummary
+ * @param config config
+ * @return getWorkerSummaryMap
+ */
+ private static Map getWorkerSummaryMap(WorkerSummary workerSummary, Map<String, Object> config) {
+ Map<String, Object> result = new HashMap();
+ result.put("supervisorId", workerSummary.get_supervisor_id());
+ result.put("host", workerSummary.get_host());
+ result.put("port", workerSummary.get_port());
+ result.put("topologyId", workerSummary.get_topology_id());
+ result.put("topologyName", workerSummary.get_topology_name());
+ result.put("executorsTotal", workerSummary.get_num_executors());
+ result.put("assignedMemOnHeap", workerSummary.get_assigned_memonheap());
+ result.put("assignedMemOffHeap", workerSummary.get_assigned_memoffheap());
+ result.put("assignedCpu", workerSummary.get_assigned_cpu());
+ result.put("componentNumTasks", workerSummary.get_component_to_num_tasks());
+ result.put("uptime", UIHelpers.prettyUptimeSec(workerSummary.get_uptime_secs()));
+ result.put("uptimeSeconds", workerSummary.get_uptime_secs());
+ result.put("workerLogLink", getWorkerLogLink(workerSummary.get_host(),
+ workerSummary.get_port(), config, workerSummary.get_topology_id()));
+ return result;
+ }
+
+ /**
+ * getSupervisorSummary.
+ * @param supervisors supervisor summary list.
+ * @param securityContext security context injected.
+ * @param config Storm config.
+ * @return Prettified JSON.
+ */
+ public static Map<String, Object> getSupervisorSummary(
+ List<SupervisorSummary> supervisors,
+ SecurityContext securityContext, Map<String, Object> config) {
+ Map<String, Object> result = new HashMap();
+ addLogviewerInfo(config, result);
+ List<Map> supervisorMaps = getSupervisorsMap(supervisors, config);
+ result.put("supervisors", supervisorMaps);
+ result.put("schedulerDisplayResource",
+ config.get(DaemonConfig.SCHEDULER_DISPLAY_RESOURCE)
+ );
+
+ return result;
+ }
+
+ /**
+ * getSupervisorsMap.
+ * @param supervisors supervisors
+ * @param config config
+ * @return getSupervisorsMap
+ */
+ private static List<Map> getSupervisorsMap(List<SupervisorSummary> supervisors, Map<String, Object> config) {
+ List<Map> supervisorMaps = new ArrayList();
+ for (SupervisorSummary supervisorSummary : supervisors) {
+ supervisorMaps.add(getPrettifiedSupervisorMap(supervisorSummary, config));
+ }
+ return supervisorMaps;
+ }
+
+ /**
+ * addLogviewerInfo.
+ * @param config config
+ * @param result result
+ */
+ private static void addLogviewerInfo(Map<String, Object> config, Map<String, Object> result) {
+ result.put("logviewerPort", getLogviewerPort(config));
+ String logviewerScheme = "http";
+ if (isSecureLogviewer(config)) {
+ logviewerScheme = "https";
+ }
+ result.put("logviewerScheme", logviewerScheme);
+ }
+
+ /**
+ * getSupervisorPageInfo.
+ * @param supervisorPageInfo supervisorPageInfo
+ * @param config config
+ * @return getSupervisorPageInfo
+ */
+ public static Map<String, Object> getSupervisorPageInfo(
+ SupervisorPageInfo supervisorPageInfo, Map<String,Object> config) {
+ Map<String, Object> result = new HashMap();
+ result.put("workers", getWorkerSummaries(supervisorPageInfo, config));
+ result.put("schedulerDisplayResource", config.get(DaemonConfig.SCHEDULER_DISPLAY_RESOURCE));
+ List<Map> supervisorMaps = getSupervisorsMap(supervisorPageInfo.get_supervisor_summaries(), config);
+ result.put("supervisors", supervisorMaps);
+ addLogviewerInfo(config, result);
+ return result;
+ }
+
+ /**
+ * getAllTopologiesSummary.
+ * @param topologies topologies
+ * @param config config
+ * @return getAllTopologiesSummary
+ */
+ public static Map<String, Object> getAllTopologiesSummary(
+ List<TopologySummary> topologies, Map<String,Object> config) {
+ Map<String, Object> result = new HashMap();
+ result.put("topologies", getTopologiesMap(null, topologies));
+ result.put("schedulerDisplayResource", config.get(DaemonConfig.SCHEDULER_DISPLAY_RESOURCE));
+ return result;
+ }
+
+ /**
+ * getWindowHint.
+ * @param window window
+ * @return getWindowHint
+ */
+ public static String getWindowHint(String window) {
+ if (window.equals(":all-time")) {
+ return "All time";
+ }
+ return UIHelpers.prettyUptimeSec(window);
+ }
+
+ /**
+ * getStatDisplayMap.
+ * @param rawDisplayMap rawDisplayMap
+ * @return getStatDisplayMap
+ */
+ public static Map<String, Double> getStatDisplayMap(Map<String, Double> rawDisplayMap) {
+ Map<String, Double> result = new HashMap();
+ for (Map.Entry<String, Double> entry : rawDisplayMap.entrySet()) {
+ result.put(getWindowHint(entry.getKey()), entry.getValue());
+ }
+
+ return result;
+ }
+
+ /**
+ * getTopologySummary.
+ * @param topologyPageInfo topologyPageInfo
+ * @param window window
+ * @param config config
+ * @param remoteUser remoteUser
+ * @return getTopologySummary
+ */
+ public static Map<String, Object> getTopologySummary(TopologyPageInfo topologyPageInfo,
+ String window, Map<String, Object> config, String remoteUser) {
+ Map<String, Object> result = new HashMap();
+ Map<String, Object> topologyConf = (Map<String, Object>) JSONValue.parse(topologyPageInfo.get_topology_conf());
+ long messageTimeout = (long) topologyConf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS);
+ Map<String, Object> unpackedTopologyPageInfo =
+ unpackTopologyInfo(topologyPageInfo, window, config);
+ result.putAll(unpackedTopologyPageInfo);
+ result.put("user", remoteUser);
+ result.put("window", window);
+ result.put("windowHint", getWindowHint(window));
+ result.put("msgTimeout", messageTimeout);
+ result.put("configuration", topologyConf);
+ result.put("visualizationTable", new ArrayList());
+ result.put("schedulerDisplayResource", config.get(DaemonConfig.SCHEDULER_DISPLAY_RESOURCE));
+ return result;
+ }
+
+ private static Map<String, Long> getStatDisplayMapLong(Map<String,Long> windowToTransferred) {
+ Map<String, Long> result = new HashMap();
+ for (Map.Entry<String, Long> entry : windowToTransferred.entrySet()) {
+ result.put(entry.getKey(), entry.getValue());
+ }
+ return result;
+ }
+
+ private static Map<String, Object> getCommonAggStatsMap(CommonAggregateStats commonAggregateStats) {
+ Map<String, Object> result = new HashMap();
+ result.put("executors", commonAggregateStats.get_num_executors());
+ result.put("tasks", commonAggregateStats.get_num_tasks());
+ result.put("emitted", commonAggregateStats.get_emitted());
+ result.put("transferred", commonAggregateStats.get_transferred());
+ result.put("acked", commonAggregateStats.get_acked());
+ result.put("failed", commonAggregateStats.get_failed());
+ result.put(
+ "requestedMemOnHeap",
+ commonAggregateStats.get_resources_map().get(Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME)
+ );
+ result.put(
+ "requestedMemOffHeap",
+ commonAggregateStats.get_resources_map().get(Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME));
+ result.put(
+ "requestedCpu" ,
+ commonAggregateStats.get_resources_map().get(Constants.COMMON_CPU_RESOURCE_NAME));
+ return result;
+ }
+
+ private static Map<String, Object> getSpoutAggStatsMap(
+ ComponentAggregateStats componentAggregateStats, String spoutId) {
+ Map<String, Object> result = new HashMap();
+ result.putAll(getCommonAggStatsMap(componentAggregateStats.get_common_stats()));
+ result.put("spoutId", spoutId);
+ result.put("encodedSpoutId", URLEncoder.encode(spoutId));
+ result.put("completeLatency",
+ componentAggregateStats.get_specific_stats().get_spout().get_complete_latency_ms());
+ return result;
+ }
+
+ private static Map<String, Object> getBoltAggStatsMap(
+ ComponentAggregateStats componentAggregateStats, String boltId) {
+ Map<String, Object> result = new HashMap();
+ result.putAll(getCommonAggStatsMap(componentAggregateStats.get_common_stats()));
+ result.put("boltId", boltId);
+ result.put("encodedBoltId", URLEncoder.encode(boltId));
+ BoltAggregateStats boltAggregateStats = componentAggregateStats.get_specific_stats().get_bolt();
+ result.put("capacity", StatsUtil.floatStr(boltAggregateStats.get_capacity()));
+ result.put("executeLatency", StatsUtil.floatStr(boltAggregateStats.get_execute_latency_ms()));
+ result.put("executed", boltAggregateStats.get_executed());
+ result.put("processLatency", StatsUtil.floatStr(boltAggregateStats.get_process_latency_ms()));
+ return result;
+ }
+
+ private static List<Map> getTopologyStatsMap(TopologyStats topologyStats) {
+ List<Map> result = new ArrayList();
+
+ Map<String, Long> emittedStatDisplayMap = getStatDisplayMapLong(topologyStats.get_window_to_emitted());
+ Map<String, Long> transferred = getStatDisplayMapLong(topologyStats.get_window_to_transferred());
+ Map<String, Double> completeLatency = getStatDisplayMap(topologyStats.get_window_to_complete_latencies_ms());
+ Map<String, Long> acked = getStatDisplayMapLong(topologyStats.get_window_to_acked());
+ Map<String, Long> failed = getStatDisplayMapLong(topologyStats.get_window_to_failed());
+ for (String window : emittedStatDisplayMap.keySet()) {
+ Map<String, Object> temp = new HashMap();
+ temp.put("windowPretty", getWindowHint(window));
+ temp.put("window", window);
+ temp.put("emitted", emittedStatDisplayMap.get(window));
+ temp.put("transferred", transferred.get(window));
+ temp.put("completeLatency", completeLatency.get(window));
+ temp.put("acked", acked.get(window));
+ temp.put("failed", failed.get(window));
+
+
+ result.add(temp);
+ }
+ return result;
+ }
+
+ private static Map<String,Object> unpackTopologyInfo(TopologyPageInfo topologyPageInfo, String window, Map<String,Object> config) {
+ Map<String, Object> result = new HashMap();
+ result.put("id", topologyPageInfo.get_id());
+ result.put("encodedId", URLEncoder.encode(topologyPageInfo.get_id()));
+ result.put("owner", topologyPageInfo.get_owner());
+ result.put("name", topologyPageInfo.get_name());
+ result.put("status", topologyPageInfo.get_status());
+ result.put("uptime", UIHelpers.prettyUptimeSec(topologyPageInfo.get_uptime_secs()));
+ result.put("uptimeSeconds", topologyPageInfo.get_uptime_secs());
+ result.put("tasksTotal", topologyPageInfo.get_num_tasks());
+ result.put("workersTotal", topologyPageInfo.get_num_workers());
+ result.put("executorsTotal", topologyPageInfo.get_num_executors());
+ result.put("schedulerInfo", topologyPageInfo.get_sched_status());
+ result.put("requestedMemOnHeap", topologyPageInfo.get_requested_memonheap());
+ result.put("requestedMemOffHeap", topologyPageInfo.get_requested_memoffheap());
+ result.put("requestedCpu", topologyPageInfo.get_requested_cpu());
+ result.put("requestedTotalMem",
+ topologyPageInfo.get_requested_memonheap() + topologyPageInfo.get_requested_memoffheap()
+ );
+ result.put("assignedMemOnHeap", topologyPageInfo.get_assigned_memonheap());
+ result.put("assignedMemOffHeap", topologyPageInfo.get_assigned_memoffheap());
+ result.put("assignedTotalMem", topologyPageInfo.get_assigned_memonheap()
+ + topologyPageInfo.get_assigned_memoffheap());
+ result.put("assignedCpu", topologyPageInfo.get_assigned_cpu());
+ result.put("requestedRegularOnHeapMem", topologyPageInfo.get_requested_regular_on_heap_memory());
+ result.put("requestedSharedOnHeapMem", topologyPageInfo.get_requested_shared_on_heap_memory());
+ result.put("requestedRegularOffHeapMem", topologyPageInfo.get_requested_regular_off_heap_memory());
+ result.put("requestedSharedOffHeapMem", topologyPageInfo.get_requested_shared_off_heap_memory());
+ result.put("assignedRegularOnHeapMem", topologyPageInfo.get_assigned_regular_on_heap_memory());
+ result.put("assignedSharedOnHeapMem", topologyPageInfo.get_assigned_shared_on_heap_memory());
+ result.put("assignedRegularOffHeapMem", topologyPageInfo.get_assigned_regular_off_heap_memory());
+ result.put("assignedSharedOffHeapMem", topologyPageInfo.get_assigned_shared_off_heap_memory());
+ result.put("topologyStats", getTopologyStatsMap(topologyPageInfo.get_topology_stats()));
+ List<Map> workerSummaries = new ArrayList();
+ if (topologyPageInfo.is_set_workers()) {
+ for (WorkerSummary workerSummary : topologyPageInfo.get_workers()) {
+ workerSummaries.add(getWorkerSummaryMap(workerSummary, config));
+ }
+ }
+ result.put("workers", workerSummaries);
+
+ Map<String, ComponentAggregateStats> spouts = topologyPageInfo.get_id_to_spout_agg_stats();
+ List<Map> spoutStats = new ArrayList();
+
+ for (Map.Entry<String, ComponentAggregateStats> spoutEntry : spouts.entrySet()) {
+ spoutStats.add(getSpoutAggStatsMap(spoutEntry.getValue(), spoutEntry.getKey()));
+ }
+ result.put("spouts", spoutStats);
+
+ Map<String, ComponentAggregateStats> bolts = topologyPageInfo.get_id_to_bolt_agg_stats();
+ List<Map> boltStats = new ArrayList();
+
+ for (Map.Entry<String, ComponentAggregateStats> boltEntry : bolts.entrySet()) {
+ boltStats.add(getBoltAggStatsMap(boltEntry.getValue(), boltEntry.getKey()));
+ }
+ result.put("bolts", boltStats);
+
+
+ result.put("configuration", topologyPageInfo.get_topology_conf());
+ boolean debuggingEnabled = false;
+ if (topologyPageInfo.is_set_debug_options()) {
+ debuggingEnabled = topologyPageInfo.get_debug_options().is_enable();
+ }
+ result.put("debug", debuggingEnabled);
+ double samplingPct = 10;
+ if (debuggingEnabled) {
+ samplingPct = topologyPageInfo.get_debug_options().get_samplingpct();
+ }
+ result.put("samplingPct", samplingPct);
+ result.put("replicationCount", topologyPageInfo.get_replication_count());
+ result.put("topologyVersion", topologyPageInfo.get_topology_version());
+ result.put("stormVersion", topologyPageInfo.get_storm_version());
+ return result;
+ }
+
+ /**
+ * getTopologyWorkers.
+ * @param topologyInfo topologyInfo
+ * @param config config
+ * @return getTopologyWorkers.
+ */
+ public static Map<String, Object> getTopologyWorkers(TopologyInfo topologyInfo, Map config) {
+ List<Map> executorSummaries = new ArrayList();
+ for (ExecutorSummary executorSummary : topologyInfo.get_executors()) {
+ Map<String, Object> executorSummaryMap = new HashMap();
+ executorSummaryMap.put("host", executorSummary.get_host());
+ executorSummaryMap.put("port", executorSummary.get_port());
+ executorSummaries.add(executorSummaryMap);
+ }
+ HashSet hashSet = new HashSet();
+ hashSet.addAll(executorSummaries);
+ executorSummaries.clear();
+ executorSummaries.addAll(hashSet);
+ Map<String, Object> result = new HashMap();
+ result.put("hostPortList", executorSummaries);
+ addLogviewerInfo(config, result);
+ return result;
+ }
+
+
+ /**
+ * getTopologyLag.
+ * @param userTopology userTopology
+ * @param config config
+ * @return getTopologyLag.
+ */
+ public static Map<String, Map<String, Object>> getTopologyLag(StormTopology userTopology, Map<String,Object> config) {
+ return TopologySpoutLag.lag(userTopology, config);
+ }
+
+ /**
+ * getBoltExecutors.
+ * @param executorSummaries executorSummaries
+ * @param stormTopology stormTopology
+ * @param sys sys
+ * @return getBoltExecutors.
+ */
+ public static Map<String, List<ExecutorSummary>> getBoltExecutors(List<ExecutorSummary> executorSummaries,
+ StormTopology stormTopology, boolean sys) {
+ Map<String, List<ExecutorSummary>> result = new HashMap();
+ for (ExecutorSummary executorSummary : executorSummaries) {
+ if (StatsUtil.componentType(stormTopology, executorSummary.get_component_id()).equals("bolt")
+ && (sys || !Utils.isSystemId(executorSummary.get_component_id()))) {
+ List<ExecutorSummary> executorSummaryList = result.getOrDefault(executorSummary.get_component_id(), new ArrayList());
+ executorSummaryList.add(executorSummary);
+ result.put(executorSummary.get_component_id(), executorSummaryList);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * getSpoutExecutors.
+ * @param executorSummaries executorSummaries
+ * @param stormTopology stormTopology
+ * @return getSpoutExecutors.
+ */
+ public static Map<String, List<ExecutorSummary>> getSpoutExecutors(List<ExecutorSummary> executorSummaries,
+ StormTopology stormTopology) {
+ Map<String, List<ExecutorSummary>> result = new HashMap();
+ for (ExecutorSummary executorSummary : executorSummaries) {
+ if (StatsUtil.componentType(stormTopology, executorSummary.get_component_id()).equals("spout")) {
+ List<ExecutorSummary> executorSummaryList = result.getOrDefault(executorSummary.get_component_id(), new ArrayList());
+ executorSummaryList.add(executorSummary);
+ result.put(executorSummary.get_component_id(), executorSummaryList);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * sanitizeStreamName.
+ * @param streamName streamName
+ * @return sanitizeStreamName
+ */
+ public static String sanitizeStreamName(String streamName) {
+ Pattern pattern = Pattern.compile("(?![A-Za-z_\\-:\\.]).");
+ Pattern pattern2 = Pattern.compile("^[A-Za-z]");
+ Matcher matcher = pattern2.matcher(streamName);
+ Matcher matcher2 = pattern.matcher("\\s" + streamName);
+ if (matcher.find()) {
+ matcher2 = pattern.matcher(streamName);
+ }
+ return matcher2.replaceAll("_");
+ }
+
+ /**
+ * sanitizeTransferredStats.
+ * @param stats stats
+ * @return sanitizeTransferredStats
+ */
+ public static Map<String, Map<String,Long>> sanitizeTransferredStats(Map<String, Map<String,Long>> stats) {
+ Map<String, Map<String,Long>> result = new HashMap();
+ for (Map.Entry<String, Map<String,Long>> entry : stats.entrySet()) {
+ Map<String,Long> temp = new HashMap();
+ for (Map.Entry<String,Long> innerEntry : entry.getValue().entrySet()) {
+ temp.put(sanitizeStreamName(innerEntry.getKey()), innerEntry.getValue());
+ }
+ result.put(entry.getKey(), temp);
+ }
+ return result;
+ }
+
+ /**
+ * getStatMapFromExecutorSummary.
+ * @param executorSummary executorSummary
+ * @return getStatMapFromExecutorSummary
+ */
+ public static Map<String, Object> getStatMapFromExecutorSummary(ExecutorSummary executorSummary) {
+ Map<String, Object> result = new HashMap();
+ result.put("host", executorSummary.get_host());
+ result.put("port", executorSummary.get_port());
+ result.put("uptime_secs", executorSummary.get_uptime_secs());
+ result.put("transferred", null);
+ if (executorSummary.is_set_stats()) {
+ result.put("transferred", sanitizeTransferredStats(executorSummary.get_stats().get_transferred()));
+ }
+ return result;
+ }
+
+ /**
+ * getInputMap.
+ * @param entryInput entryInput
+ * @return getInputMap
+ */
+ public static Map<String, Object> getInputMap(Map.Entry<GlobalStreamId,Grouping> entryInput) {
+ Map<String, Object> result = new HashMap();
+ result.put("component", entryInput.getKey().get_componentId());
+ result.put("stream", entryInput.getKey().get_streamId());
+ result.put("sani-stream", sanitizeStreamName(entryInput.getKey().get_streamId()));
+ result.put("grouping", entryInput.getValue());
+ return result;
+ }
+
+ /**
+ * getVisualizationData.
+ * @param client client
+ * @param window window
+ * @param topoId topoId
+ * @param sys sys
+ * @return getVisualizationData
+ * @throws TException TException
+ */
+ public static Map<String, Object> getVisualizationData(
+ Nimbus.Iface client, String window, String topoId, boolean sys) throws TException {
+ GetInfoOptions getInfoOptions = new GetInfoOptions();
+ getInfoOptions.set_num_err_choice(NumErrorsChoice.ONE);
+ TopologyInfo topologyInfo = client.getTopologyInfoWithOpts(topoId, getInfoOptions);
+ StormTopology stormTopology = client.getTopology(topoId);
+ Map<String, List<ExecutorSummary>> boltSummaries = getBoltExecutors(topologyInfo.get_executors(), stormTopology, sys);
+ Map<String, List<ExecutorSummary>> spoutSummaries = getSpoutExecutors(topologyInfo.get_executors(), stormTopology);
+
+ Map<String, SpoutSpec> spoutSpecs = stormTopology.get_spouts();
+ Map<String, Bolt> boltSpecs = stormTopology.get_bolts();
+
+ Map<String, Object> result = new HashMap();
+
+ for (Map.Entry<String, SpoutSpec> spoutSpecMapEntry : spoutSpecs.entrySet()) {
+ String spoutComponentId = spoutSpecMapEntry.getKey();
+ if (spoutSummaries.containsKey(spoutComponentId)) {
+ Map<String, Object> spoutData = new HashMap();
+ spoutData.put("type", "spout");
+ spoutData.put("capacity", 0);
+ Map<String, Map> spoutStreamsStats = StatsUtil.spoutStreamsStats(spoutSummaries.get(spoutComponentId), true);
+ spoutData.put("latency", spoutStreamsStats.get("complete-latencies").get(window));
+ spoutData.put("transferred", spoutStreamsStats.get("transferred").get(window));
+ spoutData.put("stats", spoutSummaries.get(
+ spoutComponentId).stream().map(
+ UIHelpers::getStatMapFromExecutorSummary).collect(Collectors.toList()));
+ spoutData.put("link", UIHelpers.urlFormat("/component.html?id=%s&topology_id=%s", spoutComponentId, topoId));
+
+ result.put("inputs",
+ spoutSpecMapEntry.getValue().get_common().get_inputs().entrySet().stream().map(
+ UIHelpers::getInputMap).collect(Collectors.toList())
+ );
+ result.put(spoutComponentId, spoutData);
+ }
+ }
+
+ for (Map.Entry<String, Bolt> boltEntry : boltSpecs.entrySet()) {
+ String boltComponentId = boltEntry.getKey();
+ if (boltSummaries.containsKey(boltComponentId) && (sys || Utils.isSystemId(boltComponentId))) {
+ Map<String, Object> boltMap = new HashMap();
+ boltMap.put("type", "bolt");
+ boltMap.put("capacity", StatsUtil.computeBoltCapacity(boltSummaries.get(boltComponentId)));
+ Map<String, Map> boltStreamsStats = StatsUtil.boltStreamsStats(boltSummaries.get(boltComponentId), true);
+ boltMap.put("latency", boltStreamsStats.get("process-latencies").get(window));
+ boltMap.put("transferred", boltStreamsStats.get("transferred").get(window));
+ boltMap.put("stats", boltSummaries.get(
+ boltComponentId).stream().map(
+ UIHelpers::getStatMapFromExecutorSummary).collect(Collectors.toList()));
+ boltMap.put("link", UIHelpers.urlFormat("/component.html?id=%s&topology_id=%s", boltComponentId, topoId));
+
+ result.put("inputs",
+ boltEntry.getValue().get_common().get_inputs().entrySet().stream().map(
+ UIHelpers::getInputMap).collect(Collectors.toList())
+ );
+ result.put(boltComponentId, boltMap);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * getStreamBox.
+ * @param visualization visualization
+ * @return getStreamBox
+ */
+ public static Map<String, Object> getStreamBox(Object visualization) {
+ Map<String, Object> visualizationData = (Map<String, Object>) visualization;
+ Map<String, Object> result = new HashMap();
+ Map<String, Object> temp = (Map<String, Object>) visualizationData.get("inputs");
+ result.put("stream", temp.get("stream"));
+ result.put("sani-stream", temp.get("sani-stream"));
+ result.put("checked", !Utils.isSystemId((String) temp.get("stream")));
+ return result;
+ }
+
+ /**
+ * getBuildVisualization.
+ * @param client client
+ * @param config config
+ * @param window window
+ * @param id id
+ * @param sys sys
+ * @ret
--- End diff --
Not totally sure what happened here but the visualization does not work because the groupings are not being mapped to JSON correctly.
```
{"inputs":[{"component":"__acker","sani-stream":"_s__ack_reset_timeout","stream":"__ack_reset_timeout","grouping":<Grouping direct:NullStruct()>},{"component":"__acker","sani-stream":"_s__ack_ack","stream":"__ack_ack","grouping":<Grouping direct:NullStruct()>},{"component":"__acker","sani-stream":"_s__ack_fail","stream":"__ack_fail","grouping":<Grouping direct:NullStruct()>}],"spout":{"transferred":2800,"stats":[{"transferred":{"10800":{"default":2800},"600":{"default":2800},"86400":{"default":2800},":all-time":{"default":2800}},"port":6700,"host":"10.0.1.12","uptime_secs":61},{"transferred":{"10800":{"default":560},"600":{"default":560},"86400":{"default":560},":all-time":{"default":560}},"port":6701,"host":"10.0.1.12","uptime_secs":60},{"transferred":{"10800":{"default":560},"600":{"default":560},"86400":{"default":560},":all-time":{"default":560}},"port":6702,"host":"10.0.1.12","uptime_secs":61},{"transferred":{"10800":{"default":560},"600":{"default":560},"86400":{"default":5
60},":all-time":{"default":560}},"port":6700,"host":"10.0.1.12","uptime_secs":61},{"transferred":{"10800":{"default":560},"600":{"default":560},"86400":{"default":560},":all-time":{"default":560}},"port":6702,"host":"10.0.1.12","uptime_secs":61}],"latency":null,"link":"\/component.html?id=spout&topology_id=wc-1-1533304535","type":"spout","capacity":0}}
```
`<Grouping direct:NullStruct()>` is not what we want as output.
---
[GitHub] storm issue #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on the issue:
https://github.com/apache/storm/pull/2752
The errors being returned by the mappers are becoming html and not json. We need them to be json, and I think we want them to support the callback as well if possible.
Also is there a way to log the exception.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r204276964
--- Diff: storm-core/src/jvm/org/apache/storm/ui/exceptionmappers/NotAliveExceptionMapper.java ---
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.exceptionmappers;
+
+import org.apache.storm.generated.NotAliveException;
+
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+import javax.ws.rs.ext.Provider;
+
+@Provider
+public class NotAliveExceptionMapper implements ExceptionMapper<NotAliveException> {
+ public Response toResponse(NotAliveException ex) {
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
--- End diff --
I'd rather not accept unless the response is at least similar detail on current. (This also means current response is not a good format IMHO, and I don't want to make it worse.)
---
[GitHub] storm issue #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on the issue:
https://github.com/apache/storm/pull/2752
I'll start looking at the code, but please fix the checkstyle issues so that travis can finish the integration test.
---
[GitHub] storm issue #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on the issue:
https://github.com/apache/storm/pull/2752
Please squash the commits.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r206239558
--- Diff: storm-webapp/src/main/java/org/apache/storm/daemon/ui/resources/StormApiResource.java ---
@@ -0,0 +1,702 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.ui.resources;
+
+import com.codahale.metrics.Meter;
+import java.net.URLDecoder;
+import java.util.Map;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+
+import org.apache.storm.daemon.ui.UIHelpers;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Root resource (exposed at "storm" path).
+ */
+@Path("/")
+public class StormApiResource {
+
+ public static final Logger LOG = LoggerFactory.getLogger(StormApiResource.class);
+
+ @Context
+ private HttpServletRequest servletRequest;
+
+ public static Map<String, Object> config = Utils.readStormConfig();
+
+ public static Meter clusterConfigurationRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-cluster-configuration-http-requests");
+
+ public static Meter clusterSummaryRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-cluster-summary-http-requests");
+
+ public static Meter nimbusSummaryRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-nimbus-summary-http-requests");
+
+ public static Meter supervisorRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-supervisor-http-requests");
+
+ public static Meter supervisorSummaryRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-supervisor-summary-http-requests");
+
+ public static Meter allTopologiesSummaryRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-all-topologies-summary-http-requests");
+
+ public static Meter topologyPageRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-topology-page-http-requests");
+
+ public static Meter topologyMetricRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-topology-metric-http-requests");
+
+ public static Meter buildVisualizationRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-build-visualization-http-requests");
+
+ public static Meter mkVisualizationDataRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-mk-visualization-data-http-requests");
+
+ public static Meter componentPageRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-component-page-http-requests");
+
+ public static Meter logConfigRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-log-config-http-requests");
+
+ public static Meter activateTopologyRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-activate-topology-http-requests");
+
+ public static Meter deactivateTopologyRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-deactivate-topology-http-requests");
+
+ public static Meter debugTopologyRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-debug-topology-http-requests");
+
+ public static Meter componentOpResponseRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-component-op-response-http-requests");
+
+ public static Meter topologyOpResponseMeter =
+ StormMetricsRegistry.registerMeter("ui:num-topology-op-response-http-requests");
+
+ public static Meter topologyLagRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-topology-lag-http-requests");
+
+ public static Meter getOwnerResourceSummariesMeter =
+ StormMetricsRegistry.registerMeter("ui:num-get-owner-resource-summaries-http-request");
+
+
+ /**
+ * /api/v1/cluster/configuration -> nimbus configuration.
+ */
+
+ @GET
+ @Path("/cluster/configuration")
+ @Produces("application/json")
+ public Response getClusterConfiguration(@QueryParam("callback") String callback) throws TException {
--- End diff --
So if a TException is thrown here, and the callback is set will the json returned by the TExceptionMappers reflect the callback or not?
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r206238984
--- Diff: storm-webapp/src/main/java/org/apache/storm/daemon/ui/resources/NimbusOp.java ---
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.ui.resources;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+@Documented
+@Retention(value = RetentionPolicy.RUNTIME)
+public @interface NimbusOp {
--- End diff --
nit: because this really is controlling security, could we rename it to something like `AuthNimbusOp`, or something that implies security a bit more?
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r206236992
--- Diff: storm-webapp/src/main/java/org/apache/storm/daemon/ui/exceptionmappers/TExceptionMapper.java ---
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.ui.exceptionmappers;
+
+import java.util.HashMap;
+import java.util.Map;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+import javax.ws.rs.ext.Provider;
+
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.thrift.TException;
+import org.json.simple.JSONValue;
+
+@Provider
+public class TExceptionMapper implements ExceptionMapper<TException> {
+
+ /**
+ * getResponse.
+ * @param ex ex
+ * @param responseStatus responseStatus
+ * @return getResponse
+ */
+ public static Response getResponse(Exception ex, Response.Status responseStatus) {
+ Response.ResponseBuilder builder = Response.status(responseStatus);
+ Map<String, String> body = new HashMap<>();
+ body.put("error", ex.toString());
+ body.put("errorMessage", ex.toString());
+ return builder.entity(JSONValue.toJSONString(body)).type("application/json").build();
+ }
+
+ /**
+ * getResponse.
+ * @param ex ex
+ * @return getResponse
+ */
+ public static Response getResponse(AuthorizationException ex) {
+ Response.ResponseBuilder builder = Response.status(Response.Status.UNAUTHORIZED);
+ Map<String, String> body = new HashMap<>();
+ body.put("error", ex.toString());
+ body.put("errorMessage", ex.get_msg());
+ return builder.entity(JSONValue.toJSONString(body)).type("application/json").build();
--- End diff --
Can we combine this with the other `getResponse`?
```
public static Response getResponse(AuthorizationException ex) {
return getResponse(ex, Response.Status.UNAUTHORIZED);
}
```
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r204260307
--- Diff: storm-dist/binary/storm-webapp-bin/src/main/assembly/storm-webapp.xml ---
@@ -27,6 +27,18 @@
<useProjectArtifact>false</useProjectArtifact>
<outputDirectory>lib-webapp</outputDirectory>
<unpack>false</unpack>
+ <excludes>
--- End diff --
Once we also move UI to webapp this will be no longer needed at all.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r204258951
--- Diff: storm-client/src/jvm/org/apache/storm/Constants.java ---
@@ -55,5 +55,7 @@
public static final String COMMON_ONHEAP_MEMORY_RESOURCE_NAME = "onheap.memory.mb";
public static final String COMMON_OFFHEAP_MEMORY_RESOURCE_NAME = "offheap.memory.mb";
public static final String COMMON_TOTAL_MEMORY_RESOURCE_NAME = "memory.mb";
+
+ public static final String STORM_API_URL_PREFIX = "/api/v1/";
--- End diff --
I can only see the usage from UIServer (storm-core, ideally needs to be moved to webserver module), and I'm not sure client side needs to know about this.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r204259143
--- Diff: storm-core/pom.xml ---
@@ -32,6 +32,9 @@
<properties>
<worker-launcher.conf.dir>/etc/storm</worker-launcher.conf.dir>
<worker-launcher.build.dir>${project.build.directory}/native/worker-launcher</worker-launcher.build.dir>
+ <jersey.version>2.27</jersey.version>
--- End diff --
Can we rely on root pom version of jersey?
---
[GitHub] storm issue #2752: Storm 1311 Migration of UI from clj to Java
Posted by govind-menon <gi...@git.apache.org>.
Github user govind-menon commented on the issue:
https://github.com/apache/storm/pull/2752
I've addressed the feedback in the new commit - currently checkstyle is failing (which is mostly Javadocs) - should have that up soon
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200403478
--- Diff: storm-client/src/jvm/org/apache/storm/Constants.java ---
@@ -55,5 +55,7 @@
public static final String COMMON_ONHEAP_MEMORY_RESOURCE_NAME = "onheap.memory.mb";
public static final String COMMON_OFFHEAP_MEMORY_RESOURCE_NAME = "offheap.memory.mb";
public static final String COMMON_TOTAL_MEMORY_RESOURCE_NAME = "memory.mb";
+
+ public static final String STORM_API_URL_PREFIX = "/api/v1/";
--- End diff --
This is never used.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by govind-menon <gi...@git.apache.org>.
Github user govind-menon commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r201717347
--- Diff: storm-core/src/jvm/org/apache/storm/ui/UIServer.java ---
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui;
+
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.drpc.webapp.ReqContextFilter;
+import org.apache.storm.security.auth.IHttpCredentialsPlugin;
+import org.apache.storm.security.auth.ServerAuthUtils;
+import org.apache.storm.ui.filters.AuthorizedUserFilter;
+import org.apache.storm.utils.Utils;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.servlet.DefaultServlet;
+import org.eclipse.jetty.servlet.FilterHolder;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+
+import org.glassfish.jersey.server.ResourceConfig;
+import org.glassfish.jersey.servlet.ServletContainer;
+
+
+import javax.servlet.DispatcherType;
+import java.util.EnumSet;
+import java.util.Map;
+
+/**
+ * Main class.
+ *
+ */
+public class UIServer {
+
+ public static void addRequestContextFilter(ServletContextHandler context,
+ String configName, Map<String, Object> conf) {
+ IHttpCredentialsPlugin auth = ServerAuthUtils.getHttpCredentialsPlugin(conf, (String) conf.get(configName));
+ ReqContextFilter filter = new ReqContextFilter(auth);
+ context.addFilter(new FilterHolder(filter), "/*", EnumSet.allOf(DispatcherType.class));
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ Map<String, Object> conf = Utils.readStormConfig();
+ Server jettyServer = new Server();
+ ServerConnector connector = new ServerConnector(jettyServer);
+ connector.setPort(4443);
+ //connector.setPort((Integer) conf.get(DaemonConfig.UI_PORT));
+ jettyServer.addConnector(connector);
+
+ ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
+ context.setContextPath("/");
+ jettyServer.setHandler(context);
+
+
+ ResourceConfig resourceConfig =
+ new ResourceConfig()
+ .packages("org.apache.storm.ui.resources")
+ .register(AuthorizedUserFilter.class);
+
+ ServletHolder jerseyServlet = new ServletHolder(new ServletContainer(resourceConfig));
+ jerseyServlet.setInitOrder(0);
+ context.addServlet(jerseyServlet, "/api/v1/*");
+ addRequestContextFilter(context, DaemonConfig.DRPC_HTTP_CREDS_PLUGIN, conf);
+
+ // add special pathspec of static content mapped to the homePath
+ ServletHolder holderHome = new ServletHolder("static-home", DefaultServlet.class);
+ holderHome.setInitParameter("resourceBase",UIServer.class.getProtectionDomain().getCodeSource().getLocation().toExternalForm() + "/WEB-INF/");
+ holderHome.setInitParameter("dirAllowed","true");
+ holderHome.setInitParameter("pathInfoOnly","true");
+ context.addServlet(holderHome,"/*");
+
+
+ // Lastly, the default servlet for root content (always needed, to satisfy servlet spec)
+ ServletHolder holderPwd = new ServletHolder("default", DefaultServlet.class);
+ holderPwd.setInitParameter("dirAllowed","true");
--- End diff --
I checked and it does not allow that
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r204276473
--- Diff: storm-core/src/jvm/org/apache/storm/ui/filters/AuthorizedUserFilter.java ---
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.filters;
+
+import java.io.IOException;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Method;
+import java.net.InetAddress;
+import java.security.Principal;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.StringJoiner;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.ResourceInfo;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.PathSegment;
+import javax.ws.rs.ext.Provider;
+
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.StormCommon;
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.security.auth.IAuthorizer;
+import org.apache.storm.security.auth.ReqContext;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.ui.resources.NimbusOp;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Provider
+public class AuthorizedUserFilter implements ContainerRequestFilter {
+
+ public static final Logger LOG = LoggerFactory.getLogger(AuthorizedUserFilter.class);
+ public static Map<String, Object> conf = Utils.readStormConfig();
+ public static IAuthorizer uiImpersonationHandler;
+ public static IAuthorizer uiAclHandler;
+
+ @Context private ResourceInfo resourceInfo;
+
+ static {
+ try {
+ uiImpersonationHandler = StormCommon.mkAuthorizationHandler(
+ (String) conf.get(DaemonConfig.NIMBUS_IMPERSONATION_AUTHORIZER), conf);
+ uiAclHandler = StormCommon.mkAuthorizationHandler(
+ (String) conf.get(DaemonConfig.NIMBUS_AUTHORIZER), conf);
+ } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
+ LOG.error("Error initializing AuthorizedUserFilter: ", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void filter(ContainerRequestContext containerRequestContext) throws IOException {
+ NimbusOp annotation = resourceInfo.getResourceMethod().getAnnotation(NimbusOp.class);
+ if (annotation == null) {
+ return;
+ }
+ String op = annotation.value();
+ if (op == null) {
+ return;
+ }
+
+ Map topoConf = null;
+ if (containerRequestContext.getUriInfo().getPathParameters().containsKey("id")) {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ try {
+ topoConf = (Map) JSONValue.parse(nimbusClient.getClient().getTopologyConf(
+ containerRequestContext.getUriInfo().getPathParameters().get("id").get(0)));
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+ }
+
+ ReqContext reqContext = ReqContext.context();
+
+ if (reqContext.isImpersonating()) {
+ if (uiImpersonationHandler != null) {
+ if (!uiImpersonationHandler.permit(reqContext, op, topoConf)) {
+ Principal realPrincipal = reqContext.realPrincipal();
+ Principal principal = reqContext.principal();
+ String user = "unknown";
+ if (principal != null) {
+ user = principal.getName();
+ }
+ String realUser = "unknown";
+ if (realPrincipal != null) {
+ realUser = realPrincipal.getName();
+ }
+ InetAddress remoteAddress = reqContext.remoteAddress();
+
+ throw new IOException(
+ "user '" + realUser + "' is not authorized to impersonate user '"
+ + user + "' from host '" + remoteAddress.toString() + "'. Please"
+ + "see SECURITY.MD to learn how to configure impersonation ACL."
+ );
+ }
+
+ LOG.warn(" principal " + reqContext.realPrincipal().toString() + " is trying to impersonate "
--- End diff --
Let's use `{}` for logging case.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200429329
--- Diff: storm-core/src/jvm/org/apache/storm/ui/resources/StormApiResource.java ---
@@ -0,0 +1,741 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.resources;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+import javax.ws.rs.core.UriInfo;
+
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.NotAliveException;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.ui.UIHelpers;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Root resource (exposed at "storm" path).
+ */
+@Path("/")
+public class StormApiResource {
+
+ public static final Logger LOG = LoggerFactory.getLogger(StormApiResource.class);
+
+ @Context
+ private UriInfo info;
+
+ @Context
+ private HttpServletRequest servletRequest;
+
+
+ @Context
+ private ServletContext servletContext;
+
+ public static Map<String, Object> config = Utils.readStormConfig();
+ /**
+ * /api/v1/cluster/configuration -> nimbus configuration.
+ */
+
+ @GET
+ @Path("/cluster/configuration")
+ @Produces("application/json")
+ public Response getClusterConfiguration() {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ return Response.status(Response.Status.OK).entity(nimbusClient.getClient().getNimbusConf()).build();
+ } catch (AuthorizationException e) {
--- End diff --
Nit: could we look at using an ExceptionMapper instead?
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200407111
--- Diff: storm-core/src/jvm/org/apache/storm/ui/UIHelpers.java ---
@@ -59,6 +116,12 @@
new Object[]{ "d", null }
};
+ /**
+ * Prettigy uptime string.
+ * @param val val.
--- End diff --
nit: val is a stringified integer. Why it is not just an int? but that is for a follow on JIRA not here.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200430165
--- Diff: storm-core/src/jvm/org/apache/storm/ui/resources/StormApiResource.java ---
@@ -0,0 +1,741 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.resources;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+import javax.ws.rs.core.UriInfo;
+
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.NotAliveException;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.ui.UIHelpers;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Root resource (exposed at "storm" path).
+ */
+@Path("/")
+public class StormApiResource {
+
+ public static final Logger LOG = LoggerFactory.getLogger(StormApiResource.class);
+
+ @Context
+ private UriInfo info;
+
+ @Context
+ private HttpServletRequest servletRequest;
+
+
+ @Context
+ private ServletContext servletContext;
+
+ public static Map<String, Object> config = Utils.readStormConfig();
+ /**
+ * /api/v1/cluster/configuration -> nimbus configuration.
+ */
+
+ @GET
+ @Path("/cluster/configuration")
+ @Produces("application/json")
+ public Response getClusterConfiguration() {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
--- End diff --
Please put the NimbusClient in the try so it is closed when we are done with it instead of waiting for GC? and we don't have to re-read the stom config for each request.
```
try (NimbusClient nimbusClient = NimbusClient.getConfiguredClient(config)) {
return Response.status(Response.Status.OK)...
}
```
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r204262254
--- Diff: storm-core/src/jvm/org/apache/storm/ui/UIHelpers.java ---
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF)
--- End diff --
Is it for fixing checkstyle, or unneeded change?
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200432708
--- Diff: storm-core/src/jvm/org/apache/storm/ui/resources/StormApiResource.java ---
@@ -0,0 +1,741 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.resources;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+import javax.ws.rs.core.UriInfo;
+
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.NotAliveException;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.ui.UIHelpers;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Root resource (exposed at "storm" path).
+ */
+@Path("/")
+public class StormApiResource {
+
+ public static final Logger LOG = LoggerFactory.getLogger(StormApiResource.class);
+
+ @Context
+ private UriInfo info;
+
+ @Context
+ private HttpServletRequest servletRequest;
+
+
+ @Context
+ private ServletContext servletContext;
--- End diff --
This is not being used.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200439312
--- Diff: storm-core/src/jvm/org/apache/storm/ui/UIHelpers.java ---
@@ -295,9 +391,1104 @@ public static String getJsonResponseBody(Object data, String callback, boolean n
return callback != null ? wrapJsonInCallback(callback, serializedData) : serializedData;
}
+ /**
+ * Converts exception into json map.
+ * @param ex Exception to be converted.
+ * @param statusCode Status code to be returned.
+ * @return Map to be converted into json.
+ */
public static Map exceptionToJson(Exception ex, int statusCode) {
StringWriter sw = new StringWriter();
ex.printStackTrace(new PrintWriter(sw));
- return ImmutableMap.of("error", statusCode + " " + HttpStatus.getMessage(statusCode), "errorMessage", sw.toString());
+ return ImmutableMap.of(
+ "error", statusCode
+ + " "
+ + HttpStatus.getMessage(statusCode),
+ "errorMessage", sw.toString());
+ }
+
+ /**
+ * Converts thrift call result into map fit for UI/api.
+ * @param clusterSummary Obtained from Nimbus.
+ * @param user User Making request
+ * @param conf Storm Conf
+ * @return Cluster Summary for display on UI/monitoring purposes via API
+ */
+ public static Map<String, Object> getClusterSummary(ClusterSummary clusterSummary, String user,
+ Map<String, Object> conf) {
+ Map<String, Object> result = new HashMap();
+ List<SupervisorSummary> supervisorSummaries = clusterSummary.get_supervisors();
+ List<TopologySummary> topologySummaries = clusterSummary.get_topologies();
+
+ Integer usedSlots =
+ supervisorSummaries.stream().mapToInt(
+ SupervisorSummary::get_num_used_workers).sum();
+ Integer totalSlots =
+ supervisorSummaries.stream().mapToInt(
+ SupervisorSummary::get_num_workers).sum();
+
+ Integer totalTasks =
+ topologySummaries.stream().mapToInt(
+ TopologySummary::get_num_tasks).sum();
+ Integer totalExecutors =
+ topologySummaries.stream().mapToInt(
+ TopologySummary::get_num_executors).sum();
+
+ Double supervisorTotalMemory =
+ supervisorSummaries.stream().mapToDouble(x -> x.get_total_resources().getOrDefault(
+ Constants.COMMON_TOTAL_MEMORY_RESOURCE_NAME,
+ x.get_total_resources().get(Config.SUPERVISOR_MEMORY_CAPACITY_MB)
+ )
+ ).sum();
+
+ Double supervisorTotalCpu =
+ supervisorSummaries.stream().mapToDouble(x -> x.get_total_resources().getOrDefault(
+ Constants.COMMON_CPU_RESOURCE_NAME,
+ x.get_total_resources().get(Config.SUPERVISOR_CPU_CAPACITY)
+ )
+ ).sum();
+
+ Double supervisorUsedMemory =
+ supervisorSummaries.stream().mapToDouble(SupervisorSummary::get_used_mem).sum();
+ Double supervisorUsedCpu =
+ supervisorSummaries.stream().mapToDouble(SupervisorSummary::get_used_cpu).sum();
+ Double supervisorFragementedCpu =
+ supervisorSummaries.stream().mapToDouble(
+ SupervisorSummary::get_fragmented_cpu).sum();
+ Double supervisorFragmentedMem =
+ supervisorSummaries.stream().mapToDouble(
+ SupervisorSummary::get_fragmented_mem).sum();
+
+
+ result.put("user", user);
+ result.put("stormVersion", VersionInfo.getVersion());
+ result.put("supervisors", supervisorSummaries.size());
+ result.put("topologies", clusterSummary.get_topologies_size());
+ result.put("slotsUsed", usedSlots);
+ result.put("slotsTotal", totalSlots);
+ result.put("slotsFree", totalSlots - usedSlots);
+ result.put("tasksTotal", totalTasks);
+ result.put("totalExecutors", totalExecutors);
+
+ result.put("totalMem", supervisorTotalMemory);
+ result.put("totalCpu", supervisorTotalCpu);
+ result.put("availMem", supervisorTotalMemory - supervisorUsedMemory);
+ result.put("availCpu", supervisorTotalCpu - supervisorUsedCpu);
+ result.put("fragmentedMem", supervisorFragmentedMem);
+ result.put("fragmentedCpu", supervisorFragementedCpu);
+ result.put("schedulerDisplayResource",
+ conf.get(DaemonConfig.SCHEDULER_DISPLAY_RESOURCE));
+ result.put("memAssignedPercentUtil", supervisorTotalMemory > 0
+ ? ((supervisorTotalMemory - supervisorUsedMemory) * 100.0)
+ / supervisorTotalMemory : 0.0);
+ result.put("cpuAssignedPercentUtil", supervisorTotalCpu > 0
--- End diff --
The old config was putting in a String for `"memAssignedPercentUtil"` and `"cpuAssignedPercentUtil"` except when `total-{mem,cpu}` was `nil` or <= 0. At which point it returned `0.0` the number not the string. This looks like a bug to me, and we should be consistent in what we return. Sadly I think we need them both to be Strings, at least for backwards compatability :(.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r204259460
--- Diff: storm-core/src/clj/org/apache/storm/ui/core.clj ---
@@ -1613,24 +1613,24 @@
https-want-client-auth (conf UI-HTTPS-WANT-CLIENT-AUTH)
https-need-client-auth (conf UI-HTTPS-NEED-CLIENT-AUTH)]
(StormMetricsRegistry/startMetricsReporters conf)
- (UIHelpers/stormRunJetty (int (conf UI-PORT))
- (conf UI-HOST)
- https-port
- header-buffer-size
- (reify IConfigurator
+ (UIHelpers/stormRunJetty (int (conf UI-PORT))
+ (conf UI-HOST)
+ https-port
+ header-buffer-size
+ (reify IConfigurator
(execute [this server]
(UIHelpers/configSsl server
- https-port
- https-ks-path
- https-ks-password
- https-ks-type
- https-key-password
- https-ts-path
- https-ts-password
- https-ts-type
- https-need-client-auth
- https-want-client-auth
- header-buffer-size)
+ https-port
+ https-ks-path
+ https-ks-password
+ https-ks-type
+ https-key-password
+ https-ts-path
+ https-ts-password
+ https-ts-type
+ https-need-client-auth
+ https-want-client-auth
+ header-buffer-size)
--- End diff --
I think we will remove clj files later, but let's revert unneeded changes so that we can see smaller set of change.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200403856
--- Diff: storm-core/pom.xml ---
@@ -32,6 +32,9 @@
<properties>
<worker-launcher.conf.dir>/etc/storm</worker-launcher.conf.dir>
<worker-launcher.build.dir>${project.build.directory}/native/worker-launcher</worker-launcher.build.dir>
+ <jersey.version>2.27</jersey.version>
+ <grizzly.version>2.4.3</grizzly.version>
+ <jersey-grizzly2.version>1.19.4</jersey-grizzly2.version>
--- End diff --
At some point can we move all of this to the web-app, or split the UI off into it's own module if we need to. This can be a follow on JIRA. I just don't see a reason to keep it here when it no longer needs clojure.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by asfgit <gi...@git.apache.org>.
Github user asfgit closed the pull request at:
https://github.com/apache/storm/pull/2752
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r201738893
--- Diff: storm-core/src/jvm/org/apache/storm/ui/exceptionmappers/AuthorizationExceptionMapper.java ---
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.exceptionmappers;
+
+import org.apache.storm.generated.AuthorizationException;
+
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+import javax.ws.rs.ext.Provider;
+
+@Provider
+public class AuthorizationExceptionMapper implements ExceptionMapper<AuthorizationException> {
+ public Response toResponse(AuthorizationException ex) {
+ return Response.status(javax.ws.rs.core.Response.Status.UNAUTHORIZED).build();
--- End diff --
nit: you don't need the `javax.ws.rs.core.` here.
```
return Response.status(Response.Status.UNAUTHORIZED).build();
```
should work
---
[GitHub] storm issue #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on the issue:
https://github.com/apache/storm/pull/2752
I may have found out more. In the logs I see
```
Caused by: java.io.IOException: Unable to fetch topo conf for topo id wc-1-1533264135
at org.apache.storm.daemon.ui.filters.AuthorizedUserFilter.filter(AuthorizedUserFilter.java:85) ~[storm-webapp-2.0.0-SNAPSHOT.jar:2.0.0-SNAPSHOT]
at org.glassfish.jersey.server.ContainerFilteringStage.apply(ContainerFilteringStage.java:132) ~[jersey-server-2.27.jar:?]
```
So it looks like the authorizer is throwing an IOException which is causing the 500 error
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r204262044
--- Diff: storm-core/src/jvm/org/apache/storm/ui/filters/HeaderResponseServletFilter.java ---
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.filters;
+
+import com.codahale.metrics.Meter;
+import org.apache.storm.Constants;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+
+public class HeaderResponseServletFilter implements Filter {
+ public static final Logger LOG = LoggerFactory.getLogger(HeaderResponseServletFilter.class);
+
+ public static Meter webRequestMeter =
+ StormMetricsRegistry.registerMeter("num-web-requests");
+
+ public static Meter mainPageRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-main-page-http-requests");
+ @Override
+ public void init(FilterConfig filterConfig) throws ServletException {
+
+ }
+
+ @Override
+ public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException {
+ webRequestMeter.mark();
+ HttpServletRequest httpRequest = (HttpServletRequest) servletRequest;
+ HttpServletResponse httpResponse = (HttpServletResponse) servletResponse;
+ if ((httpRequest.getPathInfo()).equals("/index.html")) {
--- End diff --
It looks like porting `request-middleware` to filter and the conditions between `request-middleware` and here are not same.
https://github.com/apache/storm/blob/a7e817bcd1424d300ab5bad06c5d9f4729d9f347/storm-core/src/clj/org/apache/storm/ui/helpers.clj#L39-L55
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200404317
--- Diff: storm-core/pom.xml ---
@@ -194,10 +249,7 @@
<artifactId>java-hamcrest</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>javax.servlet</groupId>
- <artifactId>javax.servlet-api</artifactId>
- </dependency>
+ <!-- https://mvnrepository.com/artifact/javax.servlet/javax.servlet-api -->
--- End diff --
Not sure why we need the comment, if we removed the dependency.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200436771
--- Diff: storm-core/src/jvm/org/apache/storm/ui/UIHelpers.java ---
@@ -295,9 +391,1104 @@ public static String getJsonResponseBody(Object data, String callback, boolean n
return callback != null ? wrapJsonInCallback(callback, serializedData) : serializedData;
}
+ /**
+ * Converts exception into json map.
+ * @param ex Exception to be converted.
+ * @param statusCode Status code to be returned.
+ * @return Map to be converted into json.
+ */
public static Map exceptionToJson(Exception ex, int statusCode) {
StringWriter sw = new StringWriter();
ex.printStackTrace(new PrintWriter(sw));
- return ImmutableMap.of("error", statusCode + " " + HttpStatus.getMessage(statusCode), "errorMessage", sw.toString());
+ return ImmutableMap.of(
+ "error", statusCode
+ + " "
+ + HttpStatus.getMessage(statusCode),
+ "errorMessage", sw.toString());
+ }
+
+ /**
+ * Converts thrift call result into map fit for UI/api.
+ * @param clusterSummary Obtained from Nimbus.
+ * @param user User Making request
+ * @param conf Storm Conf
+ * @return Cluster Summary for display on UI/monitoring purposes via API
+ */
+ public static Map<String, Object> getClusterSummary(ClusterSummary clusterSummary, String user,
+ Map<String, Object> conf) {
+ Map<String, Object> result = new HashMap();
+ List<SupervisorSummary> supervisorSummaries = clusterSummary.get_supervisors();
+ List<TopologySummary> topologySummaries = clusterSummary.get_topologies();
+
+ Integer usedSlots =
--- End diff --
nit: Please make all of these `int` and not `Integer`. `sum()` returns an `int` and we are not auto-boxing it to an `Integer` in multiple places, so it is going to be a bit cleaner (imho).
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200403357
--- Diff: conf/defaults.yaml ---
@@ -88,7 +88,7 @@ nimbus.topology.blobstore.deletion.delay.ms: 300000
### ui.* configs are for the master
ui.host: 0.0.0.0
-ui.port: 8080
+ui.port: 4443
--- End diff --
I don't think we need to change the default UI port.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200405995
--- Diff: storm-core/src/jvm/org/apache/storm/ui/StormUiApplication.java ---
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui;
+
+import java.util.HashSet;
+import java.util.Set;
+import javax.ws.rs.ApplicationPath;
+import javax.ws.rs.core.Application;
+
+import org.apache.storm.ui.resources.StormApiResource;
+// import org.apache.storm.ui.resources.StormUiResource;
--- End diff --
I assume that these are going to be un-commented once we enable serving the UI itself?
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200437014
--- Diff: storm-core/src/jvm/org/apache/storm/ui/UIHelpers.java ---
@@ -295,9 +391,1104 @@ public static String getJsonResponseBody(Object data, String callback, boolean n
return callback != null ? wrapJsonInCallback(callback, serializedData) : serializedData;
}
+ /**
+ * Converts exception into json map.
+ * @param ex Exception to be converted.
+ * @param statusCode Status code to be returned.
+ * @return Map to be converted into json.
+ */
public static Map exceptionToJson(Exception ex, int statusCode) {
StringWriter sw = new StringWriter();
ex.printStackTrace(new PrintWriter(sw));
- return ImmutableMap.of("error", statusCode + " " + HttpStatus.getMessage(statusCode), "errorMessage", sw.toString());
+ return ImmutableMap.of(
+ "error", statusCode
+ + " "
+ + HttpStatus.getMessage(statusCode),
+ "errorMessage", sw.toString());
+ }
+
+ /**
+ * Converts thrift call result into map fit for UI/api.
+ * @param clusterSummary Obtained from Nimbus.
+ * @param user User Making request
+ * @param conf Storm Conf
+ * @return Cluster Summary for display on UI/monitoring purposes via API
+ */
+ public static Map<String, Object> getClusterSummary(ClusterSummary clusterSummary, String user,
+ Map<String, Object> conf) {
+ Map<String, Object> result = new HashMap();
+ List<SupervisorSummary> supervisorSummaries = clusterSummary.get_supervisors();
+ List<TopologySummary> topologySummaries = clusterSummary.get_topologies();
+
+ Integer usedSlots =
+ supervisorSummaries.stream().mapToInt(
+ SupervisorSummary::get_num_used_workers).sum();
+ Integer totalSlots =
+ supervisorSummaries.stream().mapToInt(
+ SupervisorSummary::get_num_workers).sum();
+
+ Integer totalTasks =
+ topologySummaries.stream().mapToInt(
+ TopologySummary::get_num_tasks).sum();
+ Integer totalExecutors =
+ topologySummaries.stream().mapToInt(
+ TopologySummary::get_num_executors).sum();
+
+ Double supervisorTotalMemory =
--- End diff --
nit: same for `double` vs `Double` here.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r204276809
--- Diff: storm-core/src/jvm/org/apache/storm/ui/filters/AuthorizedUserFilter.java ---
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.filters;
+
+import java.io.IOException;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Method;
+import java.net.InetAddress;
+import java.security.Principal;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.StringJoiner;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.ResourceInfo;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.PathSegment;
+import javax.ws.rs.ext.Provider;
+
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.StormCommon;
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.security.auth.IAuthorizer;
+import org.apache.storm.security.auth.ReqContext;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.ui.resources.NimbusOp;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Provider
+public class AuthorizedUserFilter implements ContainerRequestFilter {
+
+ public static final Logger LOG = LoggerFactory.getLogger(AuthorizedUserFilter.class);
+ public static Map<String, Object> conf = Utils.readStormConfig();
+ public static IAuthorizer uiImpersonationHandler;
+ public static IAuthorizer uiAclHandler;
+
+ @Context private ResourceInfo resourceInfo;
+
+ static {
+ try {
+ uiImpersonationHandler = StormCommon.mkAuthorizationHandler(
+ (String) conf.get(DaemonConfig.NIMBUS_IMPERSONATION_AUTHORIZER), conf);
+ uiAclHandler = StormCommon.mkAuthorizationHandler(
+ (String) conf.get(DaemonConfig.NIMBUS_AUTHORIZER), conf);
+ } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
+ LOG.error("Error initializing AuthorizedUserFilter: ", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void filter(ContainerRequestContext containerRequestContext) throws IOException {
+ NimbusOp annotation = resourceInfo.getResourceMethod().getAnnotation(NimbusOp.class);
+ if (annotation == null) {
+ return;
+ }
+ String op = annotation.value();
+ if (op == null) {
+ return;
+ }
+
+ Map topoConf = null;
+ if (containerRequestContext.getUriInfo().getPathParameters().containsKey("id")) {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ try {
+ topoConf = (Map) JSONValue.parse(nimbusClient.getClient().getTopologyConf(
+ containerRequestContext.getUriInfo().getPathParameters().get("id").get(0)));
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+ }
+
+ ReqContext reqContext = ReqContext.context();
+
+ if (reqContext.isImpersonating()) {
+ if (uiImpersonationHandler != null) {
+ if (!uiImpersonationHandler.permit(reqContext, op, topoConf)) {
+ Principal realPrincipal = reqContext.realPrincipal();
+ Principal principal = reqContext.principal();
+ String user = "unknown";
+ if (principal != null) {
+ user = principal.getName();
+ }
+ String realUser = "unknown";
+ if (realPrincipal != null) {
+ realUser = realPrincipal.getName();
+ }
+ InetAddress remoteAddress = reqContext.remoteAddress();
+
+ throw new IOException(
+ "user '" + realUser + "' is not authorized to impersonate user '"
+ + user + "' from host '" + remoteAddress.toString() + "'. Please"
+ + "see SECURITY.MD to learn how to configure impersonation ACL."
+ );
+ }
+
+ LOG.warn(" principal " + reqContext.realPrincipal().toString() + " is trying to impersonate "
+ + reqContext.principal().toString() + " but "
+ + conf.get(DaemonConfig.NIMBUS_IMPERSONATION_AUTHORIZER)
+ + " has no authorizer configured. This is a potential security hole."
+ + "Please see SECURITY.MD to learn how to configure an impersonation authorizer.");
+ }
+ }
+
+ if (uiAclHandler != null) {
+ if (!uiAclHandler.permit(reqContext, op, null)) {
--- End diff --
https://github.com/apache/storm/blob/a7e817bcd1424d300ab5bad06c5d9f4729d9f347/storm-core/src/clj/org/apache/storm/ui/core.clj#L109-L114
`topology-conf` is passed in original code. Unless specific reason to do so, let's keep it as it is.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r204259185
--- Diff: storm-core/pom.xml ---
@@ -32,6 +32,9 @@
<properties>
<worker-launcher.conf.dir>/etc/storm</worker-launcher.conf.dir>
<worker-launcher.build.dir>${project.build.directory}/native/worker-launcher</worker-launcher.build.dir>
+ <jersey.version>2.27</jersey.version>
+ <grizzly.version>2.4.3</grizzly.version>
+ <jersey-grizzly2.version>1.19.4</jersey-grizzly2.version>
--- End diff --
I'd like to see this placed in web-app or individual module before merging this in. Once we pull this in and release, follow-up issues would be easy to forget. And our final goal for breaking down `storm-core` is to remove `storm-core`, so moving this to web-app also helps on reducing the work on this.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r204260022
--- Diff: storm-core/src/jvm/org/apache/storm/utils/Monitor.java ---
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF)
--- End diff --
Could you explain why this change is necessary?
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200429402
--- Diff: storm-core/src/jvm/org/apache/storm/ui/resources/StormApiResource.java ---
@@ -0,0 +1,741 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.resources;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+import javax.ws.rs.core.UriInfo;
+
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.NotAliveException;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.ui.UIHelpers;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Root resource (exposed at "storm" path).
+ */
+@Path("/")
+public class StormApiResource {
+
+ public static final Logger LOG = LoggerFactory.getLogger(StormApiResource.class);
+
+ @Context
+ private UriInfo info;
+
+ @Context
+ private HttpServletRequest servletRequest;
+
+
+ @Context
+ private ServletContext servletContext;
+
+ public static Map<String, Object> config = Utils.readStormConfig();
+ /**
+ * /api/v1/cluster/configuration -> nimbus configuration.
+ */
+
+ @GET
+ @Path("/cluster/configuration")
+ @Produces("application/json")
+ public Response getClusterConfiguration() {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ return Response.status(Response.Status.OK).entity(nimbusClient.getClient().getNimbusConf()).build();
+ } catch (AuthorizationException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.UNAUTHORIZED).build();
+ } catch (TException e) {
+ e.printStackTrace();
--- End diff --
Could we log the exception instead of just printing the stack trace?
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200408980
--- Diff: storm-core/src/jvm/org/apache/storm/ui/UIServer.java ---
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui;
+
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.drpc.webapp.ReqContextFilter;
+import org.apache.storm.security.auth.IHttpCredentialsPlugin;
+import org.apache.storm.security.auth.ServerAuthUtils;
+import org.apache.storm.ui.filters.AuthorizedUserFilter;
+import org.apache.storm.utils.Utils;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.servlet.DefaultServlet;
+import org.eclipse.jetty.servlet.FilterHolder;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+
+import org.glassfish.jersey.server.ResourceConfig;
+import org.glassfish.jersey.servlet.ServletContainer;
+
+
+import javax.servlet.DispatcherType;
+import java.util.EnumSet;
+import java.util.Map;
+
+/**
+ * Main class.
+ *
+ */
+public class UIServer {
+
+ public static void addRequestContextFilter(ServletContextHandler context,
+ String configName, Map<String, Object> conf) {
+ IHttpCredentialsPlugin auth = ServerAuthUtils.getHttpCredentialsPlugin(conf, (String) conf.get(configName));
+ ReqContextFilter filter = new ReqContextFilter(auth);
+ context.addFilter(new FilterHolder(filter), "/*", EnumSet.allOf(DispatcherType.class));
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ Map<String, Object> conf = Utils.readStormConfig();
+ Server jettyServer = new Server();
+ ServerConnector connector = new ServerConnector(jettyServer);
+ connector.setPort(4443);
+ //connector.setPort((Integer) conf.get(DaemonConfig.UI_PORT));
+ jettyServer.addConnector(connector);
+
+ ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
+ context.setContextPath("/");
+ jettyServer.setHandler(context);
+
+
+ ResourceConfig resourceConfig =
+ new ResourceConfig()
+ .packages("org.apache.storm.ui.resources")
+ .register(AuthorizedUserFilter.class);
+
+ ServletHolder jerseyServlet = new ServletHolder(new ServletContainer(resourceConfig));
+ jerseyServlet.setInitOrder(0);
+ context.addServlet(jerseyServlet, "/api/v1/*");
+ addRequestContextFilter(context, DaemonConfig.DRPC_HTTP_CREDS_PLUGIN, conf);
+
+ // add special pathspec of static content mapped to the homePath
+ ServletHolder holderHome = new ServletHolder("static-home", DefaultServlet.class);
+ holderHome.setInitParameter("resourceBase",UIServer.class.getProtectionDomain().getCodeSource().getLocation().toExternalForm() + "/WEB-INF/");
+ holderHome.setInitParameter("dirAllowed","true");
+ holderHome.setInitParameter("pathInfoOnly","true");
+ context.addServlet(holderHome,"/*");
+
+
+ // Lastly, the default servlet for root content (always needed, to satisfy servlet spec)
+ ServletHolder holderPwd = new ServletHolder("default", DefaultServlet.class);
+ holderPwd.setInitParameter("dirAllowed","true");
+ context.addServlet(holderPwd,"/");
+
+ try {
+ jettyServer.start();
+ jettyServer.join();
+ } catch (Throwable t) {
+ t.printStackTrace(System.err);
--- End diff --
Can we please just let the exception be thrown by main instead?
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by danny0405 <gi...@git.apache.org>.
Github user danny0405 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r202504021
--- Diff: storm-core/src/clj/org/apache/storm/ui/core.clj ---
@@ -1613,24 +1613,24 @@
https-want-client-auth (conf UI-HTTPS-WANT-CLIENT-AUTH)
https-need-client-auth (conf UI-HTTPS-NEED-CLIENT-AUTH)]
(StormMetricsRegistry/startMetricsReporters conf)
- (UIHelpers/stormRunJetty (int (conf UI-PORT))
- (conf UI-HOST)
- https-port
- header-buffer-size
- (reify IConfigurator
+ (UIHelpers/stormRunJetty (int (conf UI-PORT))
+ (conf UI-HOST)
+ https-port
+ header-buffer-size
+ (reify IConfigurator
(execute [this server]
(UIHelpers/configSsl server
- https-port
- https-ks-path
- https-ks-password
- https-ks-type
- https-key-password
- https-ts-path
- https-ts-password
- https-ts-type
- https-need-client-auth
- https-want-client-auth
- header-buffer-size)
+ https-port
+ https-ks-path
+ https-ks-password
+ https-ks-type
+ https-key-password
+ https-ts-path
+ https-ts-password
+ https-ts-type
+ https-need-client-auth
+ https-want-client-auth
+ header-buffer-size)
--- End diff --
This file has many nits whitespace.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r204259546
--- Diff: storm-core/src/jvm/org/apache/storm/command/UploadCredentials.java ---
@@ -34,6 +34,11 @@
private static final Logger LOG = LoggerFactory.getLogger(UploadCredentials.class);
+ /**
+ * Uploads credentials for a topology.
+ * @param args To accept topology name.
+ * @throws Exception Ignored.
--- End diff --
Same here.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r207551893
--- Diff: storm-webapp/src/main/java/org/apache/storm/daemon/ui/filters/AuthorizedUserFilter.java ---
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.ui.filters;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.security.Principal;
+import java.util.Map;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.ResourceInfo;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.ext.Provider;
+
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.StormCommon;
+import org.apache.storm.daemon.ui.resources.NimbusOp;
+import org.apache.storm.security.auth.IAuthorizer;
+import org.apache.storm.security.auth.ReqContext;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Provider
+public class AuthorizedUserFilter implements ContainerRequestFilter {
+
+ public static final Logger LOG = LoggerFactory.getLogger(AuthorizedUserFilter.class);
+ public static Map<String, Object> conf = Utils.readStormConfig();
+ public static IAuthorizer uiImpersonationHandler;
+ public static IAuthorizer uiAclHandler;
+
+ @Context private ResourceInfo resourceInfo;
+
+ static {
+ try {
+ uiImpersonationHandler = StormCommon.mkAuthorizationHandler(
+ (String) conf.get(DaemonConfig.NIMBUS_IMPERSONATION_AUTHORIZER), conf);
+ uiAclHandler = StormCommon.mkAuthorizationHandler(
+ (String) conf.get(DaemonConfig.NIMBUS_AUTHORIZER), conf);
+ } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
+ LOG.error("Error initializing AuthorizedUserFilter: ", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void filter(ContainerRequestContext containerRequestContext) throws IOException {
+ NimbusOp annotation = resourceInfo.getResourceMethod().getAnnotation(NimbusOp.class);
+ if (annotation == null) {
+ return;
+ }
+ String op = annotation.value();
+ if (op == null) {
+ return;
+ }
+
+ Map topoConf = null;
+ if (containerRequestContext.getUriInfo().getPathParameters().containsKey("id")) {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ try {
+ topoConf = (Map) JSONValue.parse(nimbusClient.getClient().getTopologyConf(
+ containerRequestContext.getUriInfo().getPathParameters().get("id").get(0)));
+ } catch (TException e) {
+ e.printStackTrace();
--- End diff --
This appears to be the part that is causing the issues. Printing the stacktrace is not logging it, so it will not show up where we want it. And if we are handling it properly it will not matter. An IOException is not going to be mapper properly into the result we want. Also all errors we don't expect 500 errors will be returned as HTML. Lets have a generic mapper that can take any Throwable and turn it into a JSON response? Also at the same time can we change this IOException so it has e as the cause, and in the Generic exception mapper can we have it look for a cause of Authroized, etc and map it to the proper response?
---
[GitHub] storm issue #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on the issue:
https://github.com/apache/storm/pull/2752
Please try running the DPRC server as well as the logviewer. I am getting some exceptions with DRPC.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r201739243
--- Diff: storm-core/src/jvm/org/apache/storm/ui/exceptionmappers/NotAliveExceptionMapper.java ---
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.exceptionmappers;
+
+import org.apache.storm.generated.NotAliveException;
+
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+import javax.ws.rs.ext.Provider;
+
+@Provider
+public class NotAliveExceptionMapper implements ExceptionMapper<NotAliveException> {
+ public Response toResponse(NotAliveException ex) {
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
--- End diff --
Could you file a follow on JIRA to handle this better in both the response and in the UI?
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200415886
--- Diff: storm-core/src/jvm/org/apache/storm/ui/filters/AuthorizedUserFilter.java ---
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.filters;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.security.Principal;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.StringJoiner;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.ResourceInfo;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.PathSegment;
+import javax.ws.rs.ext.Provider;
+
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.StormCommon;
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.security.auth.IAuthorizer;
+import org.apache.storm.security.auth.ReqContext;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Provider
+public class AuthorizedUserFilter implements ContainerRequestFilter {
+
+ public static final Logger LOG = LoggerFactory.getLogger(AuthorizedUserFilter.class);
+ public static Map<String, Object> conf = Utils.readStormConfig();
+ public static IAuthorizer uiImpersonationHandler;
+ public static IAuthorizer uiAclHandler;
+
+ public static Map<String, String> resourceMethodToNimbusOps = new HashMap();
+
+ @Context private ResourceInfo resourceInfo;
+
+ static {
+ resourceMethodToNimbusOps.put("getClusterSummary", "getClusterInfo");
+ resourceMethodToNimbusOps.put("getNimbusSummary", "getClusterInfo");
+ resourceMethodToNimbusOps.put("getOwnerResources", "getOwnerResourceSummaries");
+ resourceMethodToNimbusOps.put("getOwnerResource", "getOwnerResourceSummaries");
+ resourceMethodToNimbusOps.put("getSupervisorSummary", "getClusterInfo");
+ resourceMethodToNimbusOps.put("getSupervisor", "getSupervisorPageInfo");
+ resourceMethodToNimbusOps.put("getTopologySummary", "getClusterInfo");
+ resourceMethodToNimbusOps.put("getTopology", "getTopology");
+ resourceMethodToNimbusOps.put("getTopologyMetrics", "getTopology");
+ resourceMethodToNimbusOps.put("getTopologyVisializationInit", "getTopology");
+ resourceMethodToNimbusOps.put("getTopologyVisualization", "getTopology");
+ resourceMethodToNimbusOps.put("getTopologyLogconfig", "getTopology");
+ resourceMethodToNimbusOps.put("putTopologyLogconfig", "setLogConfig");
+ resourceMethodToNimbusOps.put("putTopologyActivate", "activate");
+ resourceMethodToNimbusOps.put("putTopologyDeactivate", "deactivate");
+ resourceMethodToNimbusOps.put("putTopologyDebugActionSpct", "debug");
+ resourceMethodToNimbusOps.put("putTopologyComponentDebugActionSpct", "debug");
+ resourceMethodToNimbusOps.put("putTopologyRebalance", "rebalance");
+ resourceMethodToNimbusOps.put("putTopologyKill", "killTopology");
+ resourceMethodToNimbusOps.put("getTopologyProfilingStart", "setWorkerProfiler");
+ resourceMethodToNimbusOps.put("getTopologyProfilingStop", "setWorkerProfiler");
+ resourceMethodToNimbusOps.put("getTopologyProfilingDumpProfile", "setWorkerProfiler");
+ resourceMethodToNimbusOps.put("getTopologyProfilingRestartWorker", "setWorkerProfiler");
+ resourceMethodToNimbusOps.put("getTopologyProfilingDumpheap", "setWorkerProfiler");
+ try {
+ uiImpersonationHandler = StormCommon.mkAuthorizationHandler(
+ (String) conf.get(DaemonConfig.NIMBUS_IMPERSONATION_AUTHORIZER), conf);
+ uiAclHandler = StormCommon.mkAuthorizationHandler(
+ (String) conf.get(DaemonConfig.NIMBUS_AUTHORIZER), conf);
+ } catch (IllegalAccessException e) {
+ e.printStackTrace();
+ } catch (InstantiationException e) {
+ e.printStackTrace();
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
--- End diff --
Please wrap all of these and throw them as a RuntimeException. If the UI is not configured properly I don't want it to fail open.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r204258845
--- Diff: pom.xml ---
@@ -329,7 +329,7 @@
<mavenVersion>3.1.0</mavenVersion>
<wagonVersion>1.0</wagonVersion>
<azure-eventhubs.version>0.13.1</azure-eventhubs.version>
- <jersey.version>2.24.1</jersey.version>
+ <jersey.version>2.27</jersey.version>
--- End diff --
Could you elaborate why we upgrade the version here?
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r204259623
--- Diff: storm-core/src/jvm/org/apache/storm/daemon/drpc/webapp/ReqContextFilter.java ---
@@ -28,10 +28,14 @@
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.container.PreMatching;
+import javax.ws.rs.ext.Provider;
import org.apache.storm.security.auth.IHttpCredentialsPlugin;
import org.apache.storm.security.auth.ReqContext;
+@Provider
+@PreMatching
public class ReqContextFilter implements Filter {
--- End diff --
Once UI also leverages this class, I feel we would be better to have common package on webservice, and move this class to there.
---
[GitHub] storm issue #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on the issue:
https://github.com/apache/storm/pull/2752
And I am getting a 404 response on the supervisor page from `/api/v1/supervisor` Specifically the full url it was going for is.
`http://localhost:8080/api/v1/supervisor?id=3223d08c-e1e6-4693-80c8-268097580ac7-172.130.98.13&sys=true`
Not sure if this is something we ported over properly or not.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by govind-menon <gi...@git.apache.org>.
Github user govind-menon commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r201717965
--- Diff: storm-core/src/jvm/org/apache/storm/ui/UIHelpers.java ---
@@ -295,9 +391,1104 @@ public static String getJsonResponseBody(Object data, String callback, boolean n
return callback != null ? wrapJsonInCallback(callback, serializedData) : serializedData;
}
+ /**
+ * Converts exception into json map.
+ * @param ex Exception to be converted.
+ * @param statusCode Status code to be returned.
+ * @return Map to be converted into json.
+ */
public static Map exceptionToJson(Exception ex, int statusCode) {
StringWriter sw = new StringWriter();
ex.printStackTrace(new PrintWriter(sw));
- return ImmutableMap.of("error", statusCode + " " + HttpStatus.getMessage(statusCode), "errorMessage", sw.toString());
+ return ImmutableMap.of(
+ "error", statusCode
+ + " "
+ + HttpStatus.getMessage(statusCode),
+ "errorMessage", sw.toString());
+ }
+
+ /**
+ * Converts thrift call result into map fit for UI/api.
+ * @param clusterSummary Obtained from Nimbus.
+ * @param user User Making request
+ * @param conf Storm Conf
+ * @return Cluster Summary for display on UI/monitoring purposes via API
+ */
+ public static Map<String, Object> getClusterSummary(ClusterSummary clusterSummary, String user,
+ Map<String, Object> conf) {
+ Map<String, Object> result = new HashMap();
+ List<SupervisorSummary> supervisorSummaries = clusterSummary.get_supervisors();
+ List<TopologySummary> topologySummaries = clusterSummary.get_topologies();
+
+ Integer usedSlots =
+ supervisorSummaries.stream().mapToInt(
+ SupervisorSummary::get_num_used_workers).sum();
+ Integer totalSlots =
+ supervisorSummaries.stream().mapToInt(
+ SupervisorSummary::get_num_workers).sum();
+
+ Integer totalTasks =
+ topologySummaries.stream().mapToInt(
+ TopologySummary::get_num_tasks).sum();
+ Integer totalExecutors =
+ topologySummaries.stream().mapToInt(
+ TopologySummary::get_num_executors).sum();
+
+ Double supervisorTotalMemory =
+ supervisorSummaries.stream().mapToDouble(x -> x.get_total_resources().getOrDefault(
+ Constants.COMMON_TOTAL_MEMORY_RESOURCE_NAME,
+ x.get_total_resources().get(Config.SUPERVISOR_MEMORY_CAPACITY_MB)
+ )
+ ).sum();
+
+ Double supervisorTotalCpu =
+ supervisorSummaries.stream().mapToDouble(x -> x.get_total_resources().getOrDefault(
+ Constants.COMMON_CPU_RESOURCE_NAME,
+ x.get_total_resources().get(Config.SUPERVISOR_CPU_CAPACITY)
+ )
+ ).sum();
+
+ Double supervisorUsedMemory =
+ supervisorSummaries.stream().mapToDouble(SupervisorSummary::get_used_mem).sum();
+ Double supervisorUsedCpu =
+ supervisorSummaries.stream().mapToDouble(SupervisorSummary::get_used_cpu).sum();
+ Double supervisorFragementedCpu =
+ supervisorSummaries.stream().mapToDouble(
+ SupervisorSummary::get_fragmented_cpu).sum();
+ Double supervisorFragmentedMem =
+ supervisorSummaries.stream().mapToDouble(
+ SupervisorSummary::get_fragmented_mem).sum();
+
+
+ result.put("user", user);
+ result.put("stormVersion", VersionInfo.getVersion());
+ result.put("supervisors", supervisorSummaries.size());
+ result.put("topologies", clusterSummary.get_topologies_size());
+ result.put("slotsUsed", usedSlots);
+ result.put("slotsTotal", totalSlots);
+ result.put("slotsFree", totalSlots - usedSlots);
+ result.put("tasksTotal", totalTasks);
+ result.put("totalExecutors", totalExecutors);
+
+ result.put("totalMem", supervisorTotalMemory);
+ result.put("totalCpu", supervisorTotalCpu);
+ result.put("availMem", supervisorTotalMemory - supervisorUsedMemory);
+ result.put("availCpu", supervisorTotalCpu - supervisorUsedCpu);
+ result.put("fragmentedMem", supervisorFragmentedMem);
+ result.put("fragmentedCpu", supervisorFragementedCpu);
+ result.put("schedulerDisplayResource",
+ conf.get(DaemonConfig.SCHEDULER_DISPLAY_RESOURCE));
+ result.put("memAssignedPercentUtil", supervisorTotalMemory > 0
+ ? ((supervisorTotalMemory - supervisorUsedMemory) * 100.0)
+ / supervisorTotalMemory : 0.0);
+ result.put("cpuAssignedPercentUtil", supervisorTotalCpu > 0
+ ? ((supervisorTotalCpu - supervisorUsedCpu) * 100.0)
+ / supervisorTotalCpu : 0.0);
+ result.put("bugtracker-url", conf.get(DaemonConfig.UI_PROJECT_BUGTRACKER_URL));
+ result.put("central-log-url", conf.get(DaemonConfig.UI_CENTRAL_LOGGING_URL));
+ return result;
+ }
+
+ /**
+ * Prettify OwnerResourceSummary.
+ * @param ownerResourceSummary
+ * @return Map of prettified OwnerResourceSummary.
+ */
+ public static Map<String, Object> unpackOwnerResourceSummary(OwnerResourceSummary ownerResourceSummary) {
+ Map<String, Object> result = new HashMap();
+
+ Double memoryGuarantee = Double.valueOf(-1);
--- End diff --
It now does that some lines below
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r204259473
--- Diff: storm-core/src/clj/org/apache/storm/ui/helpers.clj ---
@@ -56,7 +56,7 @@
;; TODO this function and its callings will be replace when ui.core and logviewer move to Java
(defnk json-response
- [data callback :need-serialize true :status 200 :headers {}]
- {:status status
+ [data callback :need-serialize true :status 200 :headers {}]
--- End diff --
Same here.
---
[GitHub] storm issue #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on the issue:
https://github.com/apache/storm/pull/2752
Sorry to revisit this in proper time. Thanks @govind-menon for doing this great work, and thanks @revans2 for the detailed review. I'll try to spend time to play with UI and report issues if any.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r204276808
--- Diff: storm-core/src/jvm/org/apache/storm/ui/filters/AuthorizedUserFilter.java ---
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.filters;
+
+import java.io.IOException;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Method;
+import java.net.InetAddress;
+import java.security.Principal;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.StringJoiner;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.ResourceInfo;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.PathSegment;
+import javax.ws.rs.ext.Provider;
+
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.StormCommon;
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.security.auth.IAuthorizer;
+import org.apache.storm.security.auth.ReqContext;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.ui.resources.NimbusOp;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Provider
+public class AuthorizedUserFilter implements ContainerRequestFilter {
+
+ public static final Logger LOG = LoggerFactory.getLogger(AuthorizedUserFilter.class);
+ public static Map<String, Object> conf = Utils.readStormConfig();
+ public static IAuthorizer uiImpersonationHandler;
+ public static IAuthorizer uiAclHandler;
+
+ @Context private ResourceInfo resourceInfo;
+
+ static {
+ try {
+ uiImpersonationHandler = StormCommon.mkAuthorizationHandler(
+ (String) conf.get(DaemonConfig.NIMBUS_IMPERSONATION_AUTHORIZER), conf);
+ uiAclHandler = StormCommon.mkAuthorizationHandler(
+ (String) conf.get(DaemonConfig.NIMBUS_AUTHORIZER), conf);
+ } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
+ LOG.error("Error initializing AuthorizedUserFilter: ", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void filter(ContainerRequestContext containerRequestContext) throws IOException {
+ NimbusOp annotation = resourceInfo.getResourceMethod().getAnnotation(NimbusOp.class);
+ if (annotation == null) {
+ return;
+ }
+ String op = annotation.value();
+ if (op == null) {
+ return;
+ }
+
+ Map topoConf = null;
+ if (containerRequestContext.getUriInfo().getPathParameters().containsKey("id")) {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ try {
+ topoConf = (Map) JSONValue.parse(nimbusClient.getClient().getTopologyConf(
+ containerRequestContext.getUriInfo().getPathParameters().get("id").get(0)));
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+ }
+
+ ReqContext reqContext = ReqContext.context();
+
+ if (reqContext.isImpersonating()) {
+ if (uiImpersonationHandler != null) {
+ if (!uiImpersonationHandler.permit(reqContext, op, topoConf)) {
+ Principal realPrincipal = reqContext.realPrincipal();
+ Principal principal = reqContext.principal();
+ String user = "unknown";
+ if (principal != null) {
+ user = principal.getName();
+ }
+ String realUser = "unknown";
+ if (realPrincipal != null) {
+ realUser = realPrincipal.getName();
+ }
+ InetAddress remoteAddress = reqContext.remoteAddress();
+
+ throw new IOException(
+ "user '" + realUser + "' is not authorized to impersonate user '"
+ + user + "' from host '" + remoteAddress.toString() + "'. Please"
+ + "see SECURITY.MD to learn how to configure impersonation ACL."
+ );
+ }
+
+ LOG.warn(" principal " + reqContext.realPrincipal().toString() + " is trying to impersonate "
+ + reqContext.principal().toString() + " but "
+ + conf.get(DaemonConfig.NIMBUS_IMPERSONATION_AUTHORIZER)
+ + " has no authorizer configured. This is a potential security hole."
+ + "Please see SECURITY.MD to learn how to configure an impersonation authorizer.");
+ }
+ }
+
+ if (uiAclHandler != null) {
+ if (!uiAclHandler.permit(reqContext, op, null)) {
+ Principal principal = reqContext.principal();
+ String user = "unknown";
+ if (principal != null) {
+ user = principal.getName();
+ }
+ throw new IOException("UI request '" + op + "' for '" + user + "' user is not authorized");
--- End diff --
Any reason to use IOException instead of AuthorizationException? The meaning of two exceptions are very different.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by HeartSaVioR <gi...@git.apache.org>.
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r204260283
--- Diff: storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java ---
@@ -37,7 +37,6 @@
import static java.util.stream.Collectors.toList;
import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
-import j2html.TagCreator;
--- End diff --
Are these for removing unused import?
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r206237782
--- Diff: storm-webapp/src/main/java/org/apache/storm/daemon/ui/filters/AuthorizedUserFilter.java ---
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.ui.filters;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.security.Principal;
+import java.util.Map;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.ResourceInfo;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.ext.Provider;
+
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.StormCommon;
+import org.apache.storm.daemon.ui.resources.NimbusOp;
+import org.apache.storm.security.auth.IAuthorizer;
+import org.apache.storm.security.auth.ReqContext;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Provider
+public class AuthorizedUserFilter implements ContainerRequestFilter {
+
+ public static final Logger LOG = LoggerFactory.getLogger(AuthorizedUserFilter.class);
+ public static Map<String, Object> conf = Utils.readStormConfig();
+ public static IAuthorizer uiImpersonationHandler;
+ public static IAuthorizer uiAclHandler;
+
+ @Context private ResourceInfo resourceInfo;
+
+ static {
+ try {
+ uiImpersonationHandler = StormCommon.mkAuthorizationHandler(
+ (String) conf.get(DaemonConfig.NIMBUS_IMPERSONATION_AUTHORIZER), conf);
+ uiAclHandler = StormCommon.mkAuthorizationHandler(
+ (String) conf.get(DaemonConfig.NIMBUS_AUTHORIZER), conf);
+ } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
+ LOG.error("Error initializing AuthorizedUserFilter: ", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void filter(ContainerRequestContext containerRequestContext) throws IOException {
+ NimbusOp annotation = resourceInfo.getResourceMethod().getAnnotation(NimbusOp.class);
+ if (annotation == null) {
+ return;
+ }
+ String op = annotation.value();
+ if (op == null) {
+ return;
+ }
+
+ Map topoConf = null;
+ if (containerRequestContext.getUriInfo().getPathParameters().containsKey("id")) {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ try {
+ topoConf = (Map) JSONValue.parse(nimbusClient.getClient().getTopologyConf(
+ containerRequestContext.getUriInfo().getPathParameters().get("id").get(0)));
+ } catch (TException e) {
+ e.printStackTrace();
--- End diff --
Can we turn this into an exception instead?
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200418105
--- Diff: storm-core/src/jvm/org/apache/storm/ui/filters/AuthorizedUserFilter.java ---
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.filters;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.security.Principal;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.StringJoiner;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.ResourceInfo;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.PathSegment;
+import javax.ws.rs.ext.Provider;
+
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.StormCommon;
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.security.auth.IAuthorizer;
+import org.apache.storm.security.auth.ReqContext;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Provider
+public class AuthorizedUserFilter implements ContainerRequestFilter {
+
+ public static final Logger LOG = LoggerFactory.getLogger(AuthorizedUserFilter.class);
+ public static Map<String, Object> conf = Utils.readStormConfig();
+ public static IAuthorizer uiImpersonationHandler;
+ public static IAuthorizer uiAclHandler;
+
+ public static Map<String, String> resourceMethodToNimbusOps = new HashMap();
+
+ @Context private ResourceInfo resourceInfo;
+
+ static {
+ resourceMethodToNimbusOps.put("getClusterSummary", "getClusterInfo");
+ resourceMethodToNimbusOps.put("getNimbusSummary", "getClusterInfo");
+ resourceMethodToNimbusOps.put("getOwnerResources", "getOwnerResourceSummaries");
+ resourceMethodToNimbusOps.put("getOwnerResource", "getOwnerResourceSummaries");
+ resourceMethodToNimbusOps.put("getSupervisorSummary", "getClusterInfo");
+ resourceMethodToNimbusOps.put("getSupervisor", "getSupervisorPageInfo");
+ resourceMethodToNimbusOps.put("getTopologySummary", "getClusterInfo");
+ resourceMethodToNimbusOps.put("getTopology", "getTopology");
+ resourceMethodToNimbusOps.put("getTopologyMetrics", "getTopology");
+ resourceMethodToNimbusOps.put("getTopologyVisializationInit", "getTopology");
+ resourceMethodToNimbusOps.put("getTopologyVisualization", "getTopology");
+ resourceMethodToNimbusOps.put("getTopologyLogconfig", "getTopology");
+ resourceMethodToNimbusOps.put("putTopologyLogconfig", "setLogConfig");
+ resourceMethodToNimbusOps.put("putTopologyActivate", "activate");
+ resourceMethodToNimbusOps.put("putTopologyDeactivate", "deactivate");
+ resourceMethodToNimbusOps.put("putTopologyDebugActionSpct", "debug");
+ resourceMethodToNimbusOps.put("putTopologyComponentDebugActionSpct", "debug");
+ resourceMethodToNimbusOps.put("putTopologyRebalance", "rebalance");
+ resourceMethodToNimbusOps.put("putTopologyKill", "killTopology");
+ resourceMethodToNimbusOps.put("getTopologyProfilingStart", "setWorkerProfiler");
+ resourceMethodToNimbusOps.put("getTopologyProfilingStop", "setWorkerProfiler");
+ resourceMethodToNimbusOps.put("getTopologyProfilingDumpProfile", "setWorkerProfiler");
+ resourceMethodToNimbusOps.put("getTopologyProfilingRestartWorker", "setWorkerProfiler");
+ resourceMethodToNimbusOps.put("getTopologyProfilingDumpheap", "setWorkerProfiler");
--- End diff --
Could we add in a custom annotation to the APIs instead of having a separate map here that is highly coupled to the code in a different file?
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r206240201
--- Diff: storm-webapp/src/main/java/org/apache/storm/daemon/ui/resources/StormApiResource.java ---
@@ -0,0 +1,702 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.ui.resources;
+
+import com.codahale.metrics.Meter;
+import java.net.URLDecoder;
+import java.util.Map;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+
+import org.apache.storm.daemon.ui.UIHelpers;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Root resource (exposed at "storm" path).
+ */
+@Path("/")
+public class StormApiResource {
+
+ public static final Logger LOG = LoggerFactory.getLogger(StormApiResource.class);
+
+ @Context
+ private HttpServletRequest servletRequest;
+
+ public static Map<String, Object> config = Utils.readStormConfig();
+
+ public static Meter clusterConfigurationRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-cluster-configuration-http-requests");
+
+ public static Meter clusterSummaryRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-cluster-summary-http-requests");
+
+ public static Meter nimbusSummaryRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-nimbus-summary-http-requests");
+
+ public static Meter supervisorRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-supervisor-http-requests");
+
+ public static Meter supervisorSummaryRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-supervisor-summary-http-requests");
+
+ public static Meter allTopologiesSummaryRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-all-topologies-summary-http-requests");
+
+ public static Meter topologyPageRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-topology-page-http-requests");
+
+ public static Meter topologyMetricRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-topology-metric-http-requests");
+
+ public static Meter buildVisualizationRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-build-visualization-http-requests");
+
+ public static Meter mkVisualizationDataRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-mk-visualization-data-http-requests");
+
+ public static Meter componentPageRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-component-page-http-requests");
+
+ public static Meter logConfigRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-log-config-http-requests");
+
+ public static Meter activateTopologyRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-activate-topology-http-requests");
+
+ public static Meter deactivateTopologyRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-deactivate-topology-http-requests");
+
+ public static Meter debugTopologyRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-debug-topology-http-requests");
+
+ public static Meter componentOpResponseRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-component-op-response-http-requests");
+
+ public static Meter topologyOpResponseMeter =
+ StormMetricsRegistry.registerMeter("ui:num-topology-op-response-http-requests");
+
+ public static Meter topologyLagRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-topology-lag-http-requests");
+
+ public static Meter getOwnerResourceSummariesMeter =
+ StormMetricsRegistry.registerMeter("ui:num-get-owner-resource-summaries-http-request");
+
+
+ /**
+ * /api/v1/cluster/configuration -> nimbus configuration.
+ */
+
+ @GET
+ @Path("/cluster/configuration")
+ @Produces("application/json")
+ public Response getClusterConfiguration(@QueryParam("callback") String callback) throws TException {
+ clusterConfigurationRequestMeter.mark();
+ try (NimbusClient nimbusClient = NimbusClient.getConfiguredClient(config)) {
+ return UIHelpers.makeStandardResponse(
+ nimbusClient.getClient().getNimbusConf(),
+ callback, false
+ );
+ }
+ }
+
+ /**
+ * /api/v1/cluster/summary -> cluster summary.
+ */
+ @GET
+ @Path("/cluster/summary")
+ @NimbusOp("getClusterInfo")
--- End diff --
If the user is not allowed to do a `getClusterInfo` and callback is set will the resulting JSON from the mapped authorization exception have the callback in it or not?
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200435146
--- Diff: storm-core/src/jvm/org/apache/storm/ui/resources/StormApiResource.java ---
@@ -0,0 +1,741 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.resources;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+import javax.ws.rs.core.UriInfo;
+
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.NotAliveException;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.ui.UIHelpers;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Root resource (exposed at "storm" path).
+ */
+@Path("/")
+public class StormApiResource {
+
+ public static final Logger LOG = LoggerFactory.getLogger(StormApiResource.class);
+
+ @Context
+ private UriInfo info;
+
+ @Context
+ private HttpServletRequest servletRequest;
+
+
+ @Context
+ private ServletContext servletContext;
+
+ public static Map<String, Object> config = Utils.readStormConfig();
+ /**
+ * /api/v1/cluster/configuration -> nimbus configuration.
+ */
+
+ @GET
+ @Path("/cluster/configuration")
+ @Produces("application/json")
+ public Response getClusterConfiguration() {
+ try {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ return Response.status(Response.Status.OK).entity(nimbusClient.getClient().getNimbusConf()).build();
+ } catch (AuthorizationException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.UNAUTHORIZED).build();
+ } catch (TException e) {
+ e.printStackTrace();
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ /**
+ * /api/v1/cluster/summary -> cluster summary.
+ */
+ @GET
+ @Path("/cluster/summary")
+ @Produces("application/json")
+ public Response getClusterSummary() {
+ try {
--- End diff --
Could you add back in the metrics for each of the http calls?
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by govind-menon <gi...@git.apache.org>.
Github user govind-menon commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r205816093
--- Diff: pom.xml ---
@@ -329,7 +329,7 @@
<mavenVersion>3.1.0</mavenVersion>
<wagonVersion>1.0</wagonVersion>
<azure-eventhubs.version>0.13.1</azure-eventhubs.version>
- <jersey.version>2.24.1</jersey.version>
+ <jersey.version>2.27</jersey.version>
--- End diff --
This is the latest stable version. Specifically it fixes a bug where a filter was not being called before a method
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r206240455
--- Diff: storm-webapp/src/main/java/org/apache/storm/daemon/ui/resources/StormApiResource.java ---
@@ -0,0 +1,702 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.ui.resources;
+
+import com.codahale.metrics.Meter;
+import java.net.URLDecoder;
+import java.util.Map;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+
+import org.apache.storm.daemon.ui.UIHelpers;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Root resource (exposed at "storm" path).
+ */
+@Path("/")
+public class StormApiResource {
+
+ public static final Logger LOG = LoggerFactory.getLogger(StormApiResource.class);
+
+ @Context
+ private HttpServletRequest servletRequest;
+
+ public static Map<String, Object> config = Utils.readStormConfig();
+
+ public static Meter clusterConfigurationRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-cluster-configuration-http-requests");
+
+ public static Meter clusterSummaryRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-cluster-summary-http-requests");
+
+ public static Meter nimbusSummaryRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-nimbus-summary-http-requests");
+
+ public static Meter supervisorRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-supervisor-http-requests");
+
+ public static Meter supervisorSummaryRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-supervisor-summary-http-requests");
+
+ public static Meter allTopologiesSummaryRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-all-topologies-summary-http-requests");
+
+ public static Meter topologyPageRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-topology-page-http-requests");
+
+ public static Meter topologyMetricRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-topology-metric-http-requests");
+
+ public static Meter buildVisualizationRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-build-visualization-http-requests");
+
+ public static Meter mkVisualizationDataRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-mk-visualization-data-http-requests");
+
+ public static Meter componentPageRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-component-page-http-requests");
+
+ public static Meter logConfigRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-log-config-http-requests");
+
+ public static Meter activateTopologyRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-activate-topology-http-requests");
+
+ public static Meter deactivateTopologyRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-deactivate-topology-http-requests");
+
+ public static Meter debugTopologyRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-debug-topology-http-requests");
+
+ public static Meter componentOpResponseRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-component-op-response-http-requests");
+
+ public static Meter topologyOpResponseMeter =
+ StormMetricsRegistry.registerMeter("ui:num-topology-op-response-http-requests");
+
+ public static Meter topologyLagRequestMeter =
+ StormMetricsRegistry.registerMeter("ui:num-topology-lag-http-requests");
+
+ public static Meter getOwnerResourceSummariesMeter =
+ StormMetricsRegistry.registerMeter("ui:num-get-owner-resource-summaries-http-request");
+
+
+ /**
+ * /api/v1/cluster/configuration -> nimbus configuration.
+ */
+
+ @GET
+ @Path("/cluster/configuration")
+ @Produces("application/json")
+ public Response getClusterConfiguration(@QueryParam("callback") String callback) throws TException {
+ clusterConfigurationRequestMeter.mark();
+ try (NimbusClient nimbusClient = NimbusClient.getConfiguredClient(config)) {
+ return UIHelpers.makeStandardResponse(
+ nimbusClient.getClient().getNimbusConf(),
+ callback, false
+ );
+ }
+ }
+
+ /**
+ * /api/v1/cluster/summary -> cluster summary.
+ */
+ @GET
+ @Path("/cluster/summary")
+ @NimbusOp("getClusterInfo")
+ @Produces("application/json")
+ public Response getClusterSummary(@QueryParam("callback") String callback) throws TException {
+ clusterSummaryRequestMeter.mark();
+ try (NimbusClient nimbusClient = NimbusClient.getConfiguredClient(config)) {
+ String user = servletRequest.getRemoteUser();
+ return UIHelpers.makeStandardResponse(
+ UIHelpers.getClusterSummary(
+ nimbusClient.getClient().getClusterInfo(), user, config),
+ callback
+ );
+ }
+ }
+
+ /**
+ * /api/v1/nimbus/summary -> nimbus summary.
+ */
+ @GET
+ @Path("/nimbus/summary")
+ @NimbusOp("getClusterInfo")
+ @Produces("application/json")
+ public Response getNimbusSummary(@QueryParam("callback") String callback) throws TException {
+ nimbusSummaryRequestMeter.mark();
+ try (NimbusClient nimbusClient = NimbusClient.getConfiguredClient(config)) {
+ return UIHelpers.makeStandardResponse(
+ UIHelpers.getNimbusSummary(
+ nimbusClient.getClient().getClusterInfo(), config),
+ callback
+ );
+ }
+ }
+
+ /**
+ * /api/v1/owner-resources -> owner resources.
+ */
+ @GET
+ @Path("/owner-resources")
+ @NimbusOp("getOwnerResourceSummaries")
+ @Produces("application/json")
+ public Response getOwnerResources(@QueryParam("callback") String callback) throws TException {
+ getOwnerResourceSummariesMeter.mark();
+ try (NimbusClient nimbusClient = NimbusClient.getConfiguredClient(config)) {
+ return UIHelpers.makeStandardResponse(
+ UIHelpers.getOwnerResourceSummaries(
+ nimbusClient.getClient().getOwnerResourceSummaries(null), config),
+ callback
+ );
+ }
+ }
+
+ /**
+ * /api/v1/owner-resources/:id -> owner resources.
+ */
+ @GET
+ @Path("/owner-resources/{id}")
+ @NimbusOp("getOwnerResourceSummaries")
--- End diff --
If someone forgets to set id, or sets it wrong, what will happen in the Authorization code when processing this NimbusOp?
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by govind-menon <gi...@git.apache.org>.
Github user govind-menon commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r205815985
--- Diff: storm-core/src/jvm/org/apache/storm/utils/Monitor.java ---
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF)
--- End diff --
I mistakenly did this for fixing checkstyle
---
[GitHub] storm issue #2752: Storm 1311 Migration of UI from clj to Java
Posted by govind-menon <gi...@git.apache.org>.
Github user govind-menon commented on the issue:
https://github.com/apache/storm/pull/2752
@HeartSaVioR @revans2 I've done the changes you requested. Please let me know of any other changes.
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by govind-menon <gi...@git.apache.org>.
Github user govind-menon commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r201717619
--- Diff: storm-core/src/jvm/org/apache/storm/ui/filters/AuthorizedUserFilter.java ---
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.filters;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.security.Principal;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.StringJoiner;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.ResourceInfo;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.PathSegment;
+import javax.ws.rs.ext.Provider;
+
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.StormCommon;
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.security.auth.IAuthorizer;
+import org.apache.storm.security.auth.ReqContext;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Provider
+public class AuthorizedUserFilter implements ContainerRequestFilter {
+
+ public static final Logger LOG = LoggerFactory.getLogger(AuthorizedUserFilter.class);
+ public static Map<String, Object> conf = Utils.readStormConfig();
+ public static IAuthorizer uiImpersonationHandler;
+ public static IAuthorizer uiAclHandler;
+
+ public static Map<String, String> resourceMethodToNimbusOps = new HashMap();
+
+ @Context private ResourceInfo resourceInfo;
+
+ static {
+ resourceMethodToNimbusOps.put("getClusterSummary", "getClusterInfo");
+ resourceMethodToNimbusOps.put("getNimbusSummary", "getClusterInfo");
+ resourceMethodToNimbusOps.put("getOwnerResources", "getOwnerResourceSummaries");
+ resourceMethodToNimbusOps.put("getOwnerResource", "getOwnerResourceSummaries");
+ resourceMethodToNimbusOps.put("getSupervisorSummary", "getClusterInfo");
+ resourceMethodToNimbusOps.put("getSupervisor", "getSupervisorPageInfo");
+ resourceMethodToNimbusOps.put("getTopologySummary", "getClusterInfo");
+ resourceMethodToNimbusOps.put("getTopology", "getTopology");
+ resourceMethodToNimbusOps.put("getTopologyMetrics", "getTopology");
+ resourceMethodToNimbusOps.put("getTopologyVisializationInit", "getTopology");
+ resourceMethodToNimbusOps.put("getTopologyVisualization", "getTopology");
+ resourceMethodToNimbusOps.put("getTopologyLogconfig", "getTopology");
+ resourceMethodToNimbusOps.put("putTopologyLogconfig", "setLogConfig");
+ resourceMethodToNimbusOps.put("putTopologyActivate", "activate");
+ resourceMethodToNimbusOps.put("putTopologyDeactivate", "deactivate");
+ resourceMethodToNimbusOps.put("putTopologyDebugActionSpct", "debug");
+ resourceMethodToNimbusOps.put("putTopologyComponentDebugActionSpct", "debug");
+ resourceMethodToNimbusOps.put("putTopologyRebalance", "rebalance");
+ resourceMethodToNimbusOps.put("putTopologyKill", "killTopology");
+ resourceMethodToNimbusOps.put("getTopologyProfilingStart", "setWorkerProfiler");
+ resourceMethodToNimbusOps.put("getTopologyProfilingStop", "setWorkerProfiler");
+ resourceMethodToNimbusOps.put("getTopologyProfilingDumpProfile", "setWorkerProfiler");
+ resourceMethodToNimbusOps.put("getTopologyProfilingRestartWorker", "setWorkerProfiler");
+ resourceMethodToNimbusOps.put("getTopologyProfilingDumpheap", "setWorkerProfiler");
--- End diff --
Still working on this
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by govind-menon <gi...@git.apache.org>.
Github user govind-menon commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r205815906
--- Diff: storm-core/src/jvm/org/apache/storm/ui/filters/AuthorizedUserFilter.java ---
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.ui.filters;
+
+import java.io.IOException;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Method;
+import java.net.InetAddress;
+import java.security.Principal;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.StringJoiner;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.ResourceInfo;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.PathSegment;
+import javax.ws.rs.ext.Provider;
+
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.StormCommon;
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.security.auth.IAuthorizer;
+import org.apache.storm.security.auth.ReqContext;
+import org.apache.storm.thrift.TException;
+import org.apache.storm.ui.resources.NimbusOp;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Provider
+public class AuthorizedUserFilter implements ContainerRequestFilter {
+
+ public static final Logger LOG = LoggerFactory.getLogger(AuthorizedUserFilter.class);
+ public static Map<String, Object> conf = Utils.readStormConfig();
+ public static IAuthorizer uiImpersonationHandler;
+ public static IAuthorizer uiAclHandler;
+
+ @Context private ResourceInfo resourceInfo;
+
+ static {
+ try {
+ uiImpersonationHandler = StormCommon.mkAuthorizationHandler(
+ (String) conf.get(DaemonConfig.NIMBUS_IMPERSONATION_AUTHORIZER), conf);
+ uiAclHandler = StormCommon.mkAuthorizationHandler(
+ (String) conf.get(DaemonConfig.NIMBUS_AUTHORIZER), conf);
+ } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
+ LOG.error("Error initializing AuthorizedUserFilter: ", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void filter(ContainerRequestContext containerRequestContext) throws IOException {
+ NimbusOp annotation = resourceInfo.getResourceMethod().getAnnotation(NimbusOp.class);
+ if (annotation == null) {
+ return;
+ }
+ String op = annotation.value();
+ if (op == null) {
+ return;
+ }
+
+ Map topoConf = null;
+ if (containerRequestContext.getUriInfo().getPathParameters().containsKey("id")) {
+ NimbusClient nimbusClient = NimbusClient.getConfiguredClient(Utils.readStormConfig());
+ try {
+ topoConf = (Map) JSONValue.parse(nimbusClient.getClient().getTopologyConf(
+ containerRequestContext.getUriInfo().getPathParameters().get("id").get(0)));
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+ }
+
+ ReqContext reqContext = ReqContext.context();
+
+ if (reqContext.isImpersonating()) {
+ if (uiImpersonationHandler != null) {
+ if (!uiImpersonationHandler.permit(reqContext, op, topoConf)) {
+ Principal realPrincipal = reqContext.realPrincipal();
+ Principal principal = reqContext.principal();
+ String user = "unknown";
+ if (principal != null) {
+ user = principal.getName();
+ }
+ String realUser = "unknown";
+ if (realPrincipal != null) {
+ realUser = realPrincipal.getName();
+ }
+ InetAddress remoteAddress = reqContext.remoteAddress();
+
+ throw new IOException(
+ "user '" + realUser + "' is not authorized to impersonate user '"
+ + user + "' from host '" + remoteAddress.toString() + "'. Please"
+ + "see SECURITY.MD to learn how to configure impersonation ACL."
+ );
+ }
+
+ LOG.warn(" principal " + reqContext.realPrincipal().toString() + " is trying to impersonate "
+ + reqContext.principal().toString() + " but "
+ + conf.get(DaemonConfig.NIMBUS_IMPERSONATION_AUTHORIZER)
+ + " has no authorizer configured. This is a potential security hole."
+ + "Please see SECURITY.MD to learn how to configure an impersonation authorizer.");
+ }
+ }
+
+ if (uiAclHandler != null) {
+ if (!uiAclHandler.permit(reqContext, op, null)) {
+ Principal principal = reqContext.principal();
+ String user = "unknown";
+ if (principal != null) {
+ user = principal.getName();
+ }
+ throw new IOException("UI request '" + op + "' for '" + user + "' user is not authorized");
--- End diff --
The filter only throws IOExceptions
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by govind-menon <gi...@git.apache.org>.
Github user govind-menon commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r201717748
--- Diff: storm-core/test/jvm/org/apache/storm/ui/StormApiResourceTest.java ---
@@ -0,0 +1,19 @@
+package org.apache.storm.ui;
+
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.WebTarget;
+
+import org.glassfish.grizzly.http.server.HttpServer;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+
+public class StormApiResourceTest {
+
+ private HttpServer server;
+ private WebTarget target;
--- End diff --
I will add a single test here
---
[GitHub] storm pull request #2752: Storm 1311 Migration of UI from clj to Java
Posted by revans2 <gi...@git.apache.org>.
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/2752#discussion_r200454747
--- Diff: storm-core/src/jvm/org/apache/storm/ui/UIHelpers.java ---
@@ -295,9 +391,1104 @@ public static String getJsonResponseBody(Object data, String callback, boolean n
return callback != null ? wrapJsonInCallback(callback, serializedData) : serializedData;
}
+ /**
+ * Converts exception into json map.
+ * @param ex Exception to be converted.
+ * @param statusCode Status code to be returned.
+ * @return Map to be converted into json.
+ */
public static Map exceptionToJson(Exception ex, int statusCode) {
StringWriter sw = new StringWriter();
ex.printStackTrace(new PrintWriter(sw));
- return ImmutableMap.of("error", statusCode + " " + HttpStatus.getMessage(statusCode), "errorMessage", sw.toString());
+ return ImmutableMap.of(
+ "error", statusCode
+ + " "
+ + HttpStatus.getMessage(statusCode),
+ "errorMessage", sw.toString());
+ }
+
+ /**
+ * Converts thrift call result into map fit for UI/api.
+ * @param clusterSummary Obtained from Nimbus.
+ * @param user User Making request
+ * @param conf Storm Conf
+ * @return Cluster Summary for display on UI/monitoring purposes via API
+ */
+ public static Map<String, Object> getClusterSummary(ClusterSummary clusterSummary, String user,
+ Map<String, Object> conf) {
+ Map<String, Object> result = new HashMap();
+ List<SupervisorSummary> supervisorSummaries = clusterSummary.get_supervisors();
+ List<TopologySummary> topologySummaries = clusterSummary.get_topologies();
+
+ Integer usedSlots =
+ supervisorSummaries.stream().mapToInt(
+ SupervisorSummary::get_num_used_workers).sum();
+ Integer totalSlots =
+ supervisorSummaries.stream().mapToInt(
+ SupervisorSummary::get_num_workers).sum();
+
+ Integer totalTasks =
+ topologySummaries.stream().mapToInt(
+ TopologySummary::get_num_tasks).sum();
+ Integer totalExecutors =
+ topologySummaries.stream().mapToInt(
+ TopologySummary::get_num_executors).sum();
+
+ Double supervisorTotalMemory =
+ supervisorSummaries.stream().mapToDouble(x -> x.get_total_resources().getOrDefault(
+ Constants.COMMON_TOTAL_MEMORY_RESOURCE_NAME,
+ x.get_total_resources().get(Config.SUPERVISOR_MEMORY_CAPACITY_MB)
+ )
+ ).sum();
+
+ Double supervisorTotalCpu =
+ supervisorSummaries.stream().mapToDouble(x -> x.get_total_resources().getOrDefault(
+ Constants.COMMON_CPU_RESOURCE_NAME,
+ x.get_total_resources().get(Config.SUPERVISOR_CPU_CAPACITY)
+ )
+ ).sum();
+
+ Double supervisorUsedMemory =
+ supervisorSummaries.stream().mapToDouble(SupervisorSummary::get_used_mem).sum();
+ Double supervisorUsedCpu =
+ supervisorSummaries.stream().mapToDouble(SupervisorSummary::get_used_cpu).sum();
+ Double supervisorFragementedCpu =
+ supervisorSummaries.stream().mapToDouble(
+ SupervisorSummary::get_fragmented_cpu).sum();
+ Double supervisorFragmentedMem =
+ supervisorSummaries.stream().mapToDouble(
+ SupervisorSummary::get_fragmented_mem).sum();
+
+
+ result.put("user", user);
+ result.put("stormVersion", VersionInfo.getVersion());
+ result.put("supervisors", supervisorSummaries.size());
+ result.put("topologies", clusterSummary.get_topologies_size());
+ result.put("slotsUsed", usedSlots);
+ result.put("slotsTotal", totalSlots);
+ result.put("slotsFree", totalSlots - usedSlots);
+ result.put("tasksTotal", totalTasks);
+ result.put("totalExecutors", totalExecutors);
+
+ result.put("totalMem", supervisorTotalMemory);
+ result.put("totalCpu", supervisorTotalCpu);
+ result.put("availMem", supervisorTotalMemory - supervisorUsedMemory);
+ result.put("availCpu", supervisorTotalCpu - supervisorUsedCpu);
+ result.put("fragmentedMem", supervisorFragmentedMem);
+ result.put("fragmentedCpu", supervisorFragementedCpu);
+ result.put("schedulerDisplayResource",
+ conf.get(DaemonConfig.SCHEDULER_DISPLAY_RESOURCE));
+ result.put("memAssignedPercentUtil", supervisorTotalMemory > 0
+ ? ((supervisorTotalMemory - supervisorUsedMemory) * 100.0)
+ / supervisorTotalMemory : 0.0);
+ result.put("cpuAssignedPercentUtil", supervisorTotalCpu > 0
+ ? ((supervisorTotalCpu - supervisorUsedCpu) * 100.0)
+ / supervisorTotalCpu : 0.0);
+ result.put("bugtracker-url", conf.get(DaemonConfig.UI_PROJECT_BUGTRACKER_URL));
+ result.put("central-log-url", conf.get(DaemonConfig.UI_CENTRAL_LOGGING_URL));
+ return result;
+ }
+
+ /**
+ * Prettify OwnerResourceSummary.
+ * @param ownerResourceSummary
+ * @return Map of prettified OwnerResourceSummary.
+ */
+ public static Map<String, Object> unpackOwnerResourceSummary(OwnerResourceSummary ownerResourceSummary) {
+ Map<String, Object> result = new HashMap();
+
+ Double memoryGuarantee = Double.valueOf(-1);
--- End diff --
Previously the values returned when this was not set were "N/A". Could we please change that back instead of a -1?
---