You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2014/11/13 20:36:56 UTC
[01/50] storm git commit: STORM-348: Netty SASL Authentication
Repository: storm
Updated Branches:
refs/heads/master 64f06c70b -> 413a6952c
STORM-348: Netty SASL Authentication
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3bce04ce
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3bce04ce
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3bce04ce
Branch: refs/heads/master
Commit: 3bce04ceab9abfd6e6da211cb5a792d9df6c96c6
Parents: 63e9082
Author: Raghavendra Nandagopal <sp...@gmail.com>
Authored: Mon Jul 28 16:28:45 2014 -0700
Committer: Raghavendra Nandagopal <sp...@gmail.com>
Committed: Mon Jul 28 16:28:45 2014 -0700
----------------------------------------------------------------------
.../backtype/storm/messaging/netty/Client.java | 2 +-
.../storm/messaging/netty/SaslMessageToken.java | 123 +++++-----
.../storm/messaging/netty/SaslNettyClient.java | 230 +++++++++---------
.../messaging/netty/SaslStormClientHandler.java | 243 ++++++++++---------
.../messaging/netty/SaslStormServerHandler.java | 240 +++++++++---------
.../storm/messaging/netty/SaslUtils.java | 75 +++---
.../netty/StormClientPipelineFactory.java | 32 ++-
.../netty/StormServerPipelineFactory.java | 43 ++--
8 files changed, 490 insertions(+), 498 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/3bce04ce/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java b/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java
index 64a1757..0cf1809 100644
--- a/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java
+++ b/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java
@@ -62,7 +62,7 @@ public class Client implements IConnection {
Map storm_conf;
- MessageBatch messageBatch = null;
+ private MessageBatch messageBatch = null;
private AtomicLong flushCheckTimer;
private int flushCheckInterval;
private ScheduledExecutorService scheduler;
http://git-wip-us.apache.org/repos/asf/storm/blob/3bce04ce/storm-core/src/jvm/backtype/storm/messaging/netty/SaslMessageToken.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/messaging/netty/SaslMessageToken.java b/storm-core/src/jvm/backtype/storm/messaging/netty/SaslMessageToken.java
index 8383d2c..d0d3ca1 100644
--- a/storm-core/src/jvm/backtype/storm/messaging/netty/SaslMessageToken.java
+++ b/storm-core/src/jvm/backtype/storm/messaging/netty/SaslMessageToken.java
@@ -27,74 +27,73 @@ import org.slf4j.LoggerFactory;
* Send and receive SASL tokens.
*/
public class SaslMessageToken {
- /** Class logger */
- private static final Logger LOG = LoggerFactory
- .getLogger(SaslMessageToken.class);
+ /** Class logger */
+ private static final Logger LOG = LoggerFactory
+ .getLogger(SaslMessageToken.class);
- /** Used for client or server's token to send or receive from each other. */
- private byte[] token;
+ /** Used for client or server's token to send or receive from each other. */
+ private byte[] token;
- /**
- * Constructor used for reflection only.
- */
- public SaslMessageToken() {
- }
+ /**
+ * Constructor used for reflection only.
+ */
+ public SaslMessageToken() {
+ }
- /**
- * Constructor used to send request.
- *
- * @param token
- * the SASL token, generated by a SaslClient or SaslServer.
- */
- public SaslMessageToken(byte[] token) {
- this.token = token;
- }
+ /**
+ * Constructor used to send request.
+ *
+ * @param token
+ * the SASL token, generated by a SaslClient or SaslServer.
+ */
+ public SaslMessageToken(byte[] token) {
+ this.token = token;
+ }
- /**
- * Read accessor for SASL token
- *
- * @return saslToken SASL token
- */
- public byte[] getSaslToken() {
- return token;
- }
+ /**
+ * Read accessor for SASL token
+ *
+ * @return saslToken SASL token
+ */
+ public byte[] getSaslToken() {
+ return token;
+ }
- /**
- * Write accessor for SASL token
- *
- * @param token
- * SASL token
- */
- public void setSaslToken(byte[] token) {
- this.token = token;
- }
+ /**
+ * Write accessor for SASL token
+ *
+ * @param token
+ * SASL token
+ */
+ public void setSaslToken(byte[] token) {
+ this.token = token;
+ }
- int encodeLength() {
- return 2+4+token.length;
- }
+ int encodeLength() {
+ return 2 + 4 + token.length;
+ }
- /**
- * encode the current SaslToken Message into a channel buffer
- * SaslTokenMessageRequest is encoded as:
- * identifier .... short(2) always it is -500
- * payload length .... int
- * payload .... byte[]
- * @throws Exception
- */
- ChannelBuffer buffer() throws Exception {
- ChannelBufferOutputStream bout = new ChannelBufferOutputStream(
- ChannelBuffers.directBuffer(encodeLength()));
- short identifier = -500;
- int payload_len = 0;
+ /**
+ * encode the current SaslToken Message into a channel buffer
+ * SaslTokenMessageRequest is encoded as: identifier .... short(2) always it
+ * is -500 payload length .... int payload .... byte[]
+ *
+ * @throws Exception
+ */
+ ChannelBuffer buffer() throws Exception {
+ ChannelBufferOutputStream bout = new ChannelBufferOutputStream(
+ ChannelBuffers.directBuffer(encodeLength()));
+ short identifier = -500;
+ int payload_len = 0;
if (token != null)
- payload_len = token.length;
-
- bout.writeShort((short)identifier);
- bout.writeInt((int)payload_len);
- if(payload_len>0) {
- bout.write(token);
- }
- bout.close();
- return bout.buffer();
- }
+ payload_len = token.length;
+
+ bout.writeShort((short) identifier);
+ bout.writeInt((int) payload_len);
+ if (payload_len > 0) {
+ bout.write(token);
+ }
+ bout.close();
+ return bout.buffer();
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/3bce04ce/storm-core/src/jvm/backtype/storm/messaging/netty/SaslNettyClient.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/messaging/netty/SaslNettyClient.java b/storm-core/src/jvm/backtype/storm/messaging/netty/SaslNettyClient.java
index fedcfff..023e950 100644
--- a/storm-core/src/jvm/backtype/storm/messaging/netty/SaslNettyClient.java
+++ b/storm-core/src/jvm/backtype/storm/messaging/netty/SaslNettyClient.java
@@ -38,129 +38,129 @@ import org.slf4j.LoggerFactory;
*/
public class SaslNettyClient {
- private static final Logger LOG = LoggerFactory
- .getLogger(SaslNettyClient.class);
+ private static final Logger LOG = LoggerFactory
+ .getLogger(SaslNettyClient.class);
- /**
- * Used to respond to server's counterpart, SaslServer with SASL tokens
- * represented as byte arrays.
- */
- private SaslClient saslClient;
+ /**
+ * Used to respond to server's counterpart, SaslServer with SASL tokens
+ * represented as byte arrays.
+ */
+ private SaslClient saslClient;
- /**
- * Create a SaslNettyClient for authentication with servers.
- */
- public SaslNettyClient(String topologyName, byte[] token) {
- try {
- LOG.debug("SaslNettyClient: Creating SASL "
- + SaslUtils.AUTH_DIGEST_MD5
- + " client to authenticate to server ");
+ /**
+ * Create a SaslNettyClient for authentication with servers.
+ */
+ public SaslNettyClient(String topologyName, byte[] token) {
+ try {
+ LOG.debug("SaslNettyClient: Creating SASL "
+ + SaslUtils.AUTH_DIGEST_MD5
+ + " client to authenticate to server ");
- saslClient = Sasl.createSaslClient(
- new String[] { SaslUtils.AUTH_DIGEST_MD5 }, null, null,
- SaslUtils.DEFAULT_REALM, SaslUtils.getSaslProps(),
- new SaslClientCallbackHandler(topologyName, token));
+ saslClient = Sasl.createSaslClient(
+ new String[] { SaslUtils.AUTH_DIGEST_MD5 }, null, null,
+ SaslUtils.DEFAULT_REALM, SaslUtils.getSaslProps(),
+ new SaslClientCallbackHandler(topologyName, token));
- } catch (IOException e) {
- LOG.error("SaslNettyClient: Could not obtain topology token for Netty "
- + "Client to use to authenticate with a Netty Server.");
- saslClient = null;
- }
- }
+ } catch (IOException e) {
+ LOG.error("SaslNettyClient: Could not obtain topology token for Netty "
+ + "Client to use to authenticate with a Netty Server.");
+ saslClient = null;
+ }
+ }
- public boolean isComplete() {
- return saslClient.isComplete();
- }
+ public boolean isComplete() {
+ return saslClient.isComplete();
+ }
- /**
- * Respond to server's SASL token.
- *
- * @param saslTokenMessage
- * contains server's SASL token
- * @return client's response SASL token
- */
- public byte[] saslResponse(SaslMessageToken saslTokenMessage) {
- try {
- byte[] retval = saslClient.evaluateChallenge(saslTokenMessage
- .getSaslToken());
- return retval;
- } catch (SaslException e) {
- LOG.error(
- "saslResponse: Failed to respond to SASL server's token:",
- e);
- return null;
- }
- }
+ /**
+ * Respond to server's SASL token.
+ *
+ * @param saslTokenMessage
+ * contains server's SASL token
+ * @return client's response SASL token
+ */
+ public byte[] saslResponse(SaslMessageToken saslTokenMessage) {
+ try {
+ byte[] retval = saslClient.evaluateChallenge(saslTokenMessage
+ .getSaslToken());
+ return retval;
+ } catch (SaslException e) {
+ LOG.error(
+ "saslResponse: Failed to respond to SASL server's token:",
+ e);
+ return null;
+ }
+ }
- /**
- * Implementation of javax.security.auth.callback.CallbackHandler that works
- * with Storm topology tokens.
- */
- private static class SaslClientCallbackHandler implements CallbackHandler {
- /** Generated username contained in TopologyToken */
- private final String userName;
- /** Generated password contained in TopologyToken */
- private final char[] userPassword;
+ /**
+ * Implementation of javax.security.auth.callback.CallbackHandler that works
+ * with Storm topology tokens.
+ */
+ private static class SaslClientCallbackHandler implements CallbackHandler {
+ /** Generated username contained in TopologyToken */
+ private final String userName;
+ /** Generated password contained in TopologyToken */
+ private final char[] userPassword;
- /**
- * Set private members using topology token.
- *
- * @param topologyToken
- */
- public SaslClientCallbackHandler(String topologyToken, byte[] token) {
- this.userName = SaslUtils
- .encodeIdentifier(topologyToken.getBytes());
- this.userPassword = SaslUtils.encodePassword(token);
- }
+ /**
+ * Set private members using topology token.
+ *
+ * @param topologyToken
+ */
+ public SaslClientCallbackHandler(String topologyToken, byte[] token) {
+ this.userName = SaslUtils
+ .encodeIdentifier(topologyToken.getBytes());
+ this.userPassword = SaslUtils.encodePassword(token);
+ }
- /**
- * Implementation used to respond to SASL tokens from server.
- *
- * @param callbacks
- * objects that indicate what credential information the
- * server's SaslServer requires from the client.
- * @throws UnsupportedCallbackException
- */
- public void handle(Callback[] callbacks)
- throws UnsupportedCallbackException {
- NameCallback nc = null;
- PasswordCallback pc = null;
- RealmCallback rc = null;
- for (Callback callback : callbacks) {
- if (callback instanceof RealmChoiceCallback) {
- continue;
- } else if (callback instanceof NameCallback) {
- nc = (NameCallback) callback;
- } else if (callback instanceof PasswordCallback) {
- pc = (PasswordCallback) callback;
- } else if (callback instanceof RealmCallback) {
- rc = (RealmCallback) callback;
- } else {
- throw new UnsupportedCallbackException(callback,
- "handle: Unrecognized SASL client callback");
- }
- }
- if (nc != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("handle: SASL client callback: setting username: "
- + userName);
- }
- nc.setName(userName);
- }
- if (pc != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("handle: SASL client callback: setting userPassword");
- }
- pc.setPassword(userPassword);
- }
- if (rc != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("handle: SASL client callback: setting realm: "
- + rc.getDefaultText());
- }
- rc.setText(rc.getDefaultText());
- }
- }
- }
+ /**
+ * Implementation used to respond to SASL tokens from server.
+ *
+ * @param callbacks
+ * objects that indicate what credential information the
+ * server's SaslServer requires from the client.
+ * @throws UnsupportedCallbackException
+ */
+ public void handle(Callback[] callbacks)
+ throws UnsupportedCallbackException {
+ NameCallback nc = null;
+ PasswordCallback pc = null;
+ RealmCallback rc = null;
+ for (Callback callback : callbacks) {
+ if (callback instanceof RealmChoiceCallback) {
+ continue;
+ } else if (callback instanceof NameCallback) {
+ nc = (NameCallback) callback;
+ } else if (callback instanceof PasswordCallback) {
+ pc = (PasswordCallback) callback;
+ } else if (callback instanceof RealmCallback) {
+ rc = (RealmCallback) callback;
+ } else {
+ throw new UnsupportedCallbackException(callback,
+ "handle: Unrecognized SASL client callback");
+ }
+ }
+ if (nc != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("handle: SASL client callback: setting username: "
+ + userName);
+ }
+ nc.setName(userName);
+ }
+ if (pc != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("handle: SASL client callback: setting userPassword");
+ }
+ pc.setPassword(userPassword);
+ }
+ if (rc != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("handle: SASL client callback: setting realm: "
+ + rc.getDefaultText());
+ }
+ rc.setText(rc.getDefaultText());
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/3bce04ce/storm-core/src/jvm/backtype/storm/messaging/netty/SaslStormClientHandler.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/messaging/netty/SaslStormClientHandler.java b/storm-core/src/jvm/backtype/storm/messaging/netty/SaslStormClientHandler.java
index 59c4abd..f94cbc3 100644
--- a/storm-core/src/jvm/backtype/storm/messaging/netty/SaslStormClientHandler.java
+++ b/storm-core/src/jvm/backtype/storm/messaging/netty/SaslStormClientHandler.java
@@ -32,124 +32,127 @@ import backtype.storm.Config;
public class SaslStormClientHandler extends SimpleChannelUpstreamHandler {
- private static final Logger LOG = LoggerFactory
- .getLogger(SaslStormClientHandler.class);
- private Client client;
- long start_time;
- /** Used for client or server's token to send or receive from each other. */
- private byte[] token;
- private String topologyName;
-
- public SaslStormClientHandler(Client client) throws IOException {
- this.client = client;
- start_time = System.currentTimeMillis();
- getSASLCredentials();
- }
-
- @Override
- public void channelConnected(ChannelHandlerContext ctx,
- ChannelStateEvent event) {
- // register the newly established channel
- Channel channel = ctx.getChannel();
-
- LOG.info("Connection established from " + channel.getLocalAddress()
- + " to " + channel.getRemoteAddress());
-
- try {
- SaslNettyClient saslNettyClient = SaslNettyClientState.getSaslNettyClient
- .get(channel);
-
- if (saslNettyClient == null) {
- LOG.debug("Creating saslNettyClient now " + "for channel: "
- + channel);
- saslNettyClient = new SaslNettyClient(topologyName, token);
- SaslNettyClientState.getSaslNettyClient.set(channel,
- saslNettyClient);
- }
- channel.write(ControlMessage.SASL_TOKEN_MESSAGE_REQUEST);
- } catch (Exception e) {
- LOG.error("Failed to authenticate with server " + "due to error: "
- + e);
- }
- return;
-
- }
-
- @Override
- public void messageReceived(ChannelHandlerContext ctx, MessageEvent event)
- throws Exception {
- LOG.debug("send/recv time (ms): {}",
- (System.currentTimeMillis() - start_time));
-
- Channel channel = ctx.getChannel();
-
- // Generate SASL response to server using Channel-local SASL client.
- SaslNettyClient saslNettyClient = SaslNettyClientState.getSaslNettyClient
- .get(channel);
- if (saslNettyClient == null) {
- throw new Exception("saslNettyClient was unexpectedly "
- + "null for channel: " + channel);
- }
-
- // examine the response message from server
- if (event.getMessage() instanceof ControlMessage) {
- ControlMessage msg = (ControlMessage) event.getMessage();
- if (msg == ControlMessage.SASL_COMPLETE_REQUEST) {
- LOG.debug("Server has sent us the SaslComplete "
- + "message. Allowing normal work to proceed.");
-
- if (!saslNettyClient.isComplete()) {
- LOG.error("Server returned a Sasl-complete message, "
- + "but as far as we can tell, we are not authenticated yet.");
- throw new Exception("Server returned a "
- + "Sasl-complete message, but as far as "
- + "we can tell, we are not authenticated yet.");
- }
- ctx.getPipeline().remove(this);
- // We call fireMessageReceived since the client is allowed to
- // perform this request. The client's request will now proceed
- // to the next pipeline component namely StormClientHandler.
- Channels.fireMessageReceived(ctx, msg);
- return;
- }
- }
- SaslMessageToken saslTokenMessage = (SaslMessageToken) event
- .getMessage();
- LOG.debug("Responding to server's token of length: "
- + saslTokenMessage.getSaslToken().length);
-
- // Generate SASL response (but we only actually send the response if
- // it's non-null.
- byte[] responseToServer = saslNettyClient
- .saslResponse(saslTokenMessage);
- if (responseToServer == null) {
- // If we generate a null response, then authentication has completed
- // (if not, warn), and return without sending a response back to the
- // server.
- LOG.debug("Response to server is null: "
- + "authentication should now be complete.");
- if (!saslNettyClient.isComplete()) {
- LOG.warn("Generated a null response, "
- + "but authentication is not complete.");
- }
- return;
- } else {
- LOG.debug("Response to server token has length:"
- + responseToServer.length);
- }
- // Construct a message containing the SASL response and send it to the
- // server.
- SaslMessageToken saslResponse = new SaslMessageToken(responseToServer);
- channel.write(saslResponse);
- }
-
- private void getSASLCredentials() throws IOException {
- topologyName = (String) this.client.storm_conf
- .get(Config.TOPOLOGY_NAME);
- String secretKey = SaslUtils.getSecretKey(this.client.storm_conf);
- if(secretKey!=null) {
- token = secretKey.getBytes();
- }
- LOG.debug("SASL credentials for storm topology "+topologyName+ " is "+secretKey);
- }
+ private static final Logger LOG = LoggerFactory
+ .getLogger(SaslStormClientHandler.class);
+ private Client client;
+ long start_time;
+ /** Used for client or server's token to send or receive from each other. */
+ private byte[] token;
+ private String topologyName;
+
+ public SaslStormClientHandler(Client client) throws IOException {
+ this.client = client;
+ start_time = System.currentTimeMillis();
+ getSASLCredentials();
+ }
+
+ @Override
+ public void channelConnected(ChannelHandlerContext ctx,
+ ChannelStateEvent event) {
+ // register the newly established channel
+ Channel channel = ctx.getChannel();
+
+ LOG.info("Connection established from " + channel.getLocalAddress()
+ + " to " + channel.getRemoteAddress());
+
+ try {
+ SaslNettyClient saslNettyClient = SaslNettyClientState.getSaslNettyClient
+ .get(channel);
+
+ if (saslNettyClient == null) {
+ LOG.debug("Creating saslNettyClient now " + "for channel: "
+ + channel);
+ saslNettyClient = new SaslNettyClient(topologyName, token);
+ SaslNettyClientState.getSaslNettyClient.set(channel,
+ saslNettyClient);
+ }
+ channel.write(ControlMessage.SASL_TOKEN_MESSAGE_REQUEST);
+ } catch (Exception e) {
+ LOG.error("Failed to authenticate with server " + "due to error: ",
+ e);
+ }
+ return;
+
+ }
+
+ @Override
+ public void messageReceived(ChannelHandlerContext ctx, MessageEvent event)
+ throws Exception {
+ LOG.debug("send/recv time (ms): {}",
+ (System.currentTimeMillis() - start_time));
+
+ Channel channel = ctx.getChannel();
+
+ // Generate SASL response to server using Channel-local SASL client.
+ SaslNettyClient saslNettyClient = SaslNettyClientState.getSaslNettyClient
+ .get(channel);
+ if (saslNettyClient == null) {
+ throw new Exception("saslNettyClient was unexpectedly "
+ + "null for channel: " + channel);
+ }
+
+ // examine the response message from server
+ if (event.getMessage() instanceof ControlMessage) {
+ ControlMessage msg = (ControlMessage) event.getMessage();
+ if (msg == ControlMessage.SASL_COMPLETE_REQUEST) {
+ LOG.debug("Server has sent us the SaslComplete "
+ + "message. Allowing normal work to proceed.");
+
+ if (!saslNettyClient.isComplete()) {
+ LOG.error("Server returned a Sasl-complete message, "
+ + "but as far as we can tell, we are not authenticated yet.");
+ throw new Exception("Server returned a "
+ + "Sasl-complete message, but as far as "
+ + "we can tell, we are not authenticated yet.");
+ }
+ ctx.getPipeline().remove(this);
+ // We call fireMessageReceived since the client is allowed to
+ // perform this request. The client's request will now proceed
+ // to the next pipeline component namely StormClientHandler.
+ Channels.fireMessageReceived(ctx, msg);
+ return;
+ }
+ }
+ SaslMessageToken saslTokenMessage = (SaslMessageToken) event
+ .getMessage();
+ LOG.debug("Responding to server's token of length: "
+ + saslTokenMessage.getSaslToken().length);
+
+ // Generate SASL response (but we only actually send the response if
+ // it's non-null.
+ byte[] responseToServer = saslNettyClient
+ .saslResponse(saslTokenMessage);
+ if (responseToServer == null) {
+ // If we generate a null response, then authentication has completed
+ // (if not, warn), and return without sending a response back to the
+ // server.
+ LOG.debug("Response to server is null: "
+ + "authentication should now be complete.");
+ if (!saslNettyClient.isComplete()) {
+ LOG.warn("Generated a null response, "
+ + "but authentication is not complete.");
+ throw new Exception("Server reponse is null, but as far as "
+ + "we can tell, we are not authenticated yet.");
+ }
+ return;
+ } else {
+ LOG.debug("Response to server token has length:"
+ + responseToServer.length);
+ }
+ // Construct a message containing the SASL response and send it to the
+ // server.
+ SaslMessageToken saslResponse = new SaslMessageToken(responseToServer);
+ channel.write(saslResponse);
+ }
+
+ private void getSASLCredentials() throws IOException {
+ topologyName = (String) this.client.storm_conf
+ .get(Config.TOPOLOGY_NAME);
+ String secretKey = SaslUtils.getSecretKey(this.client.storm_conf);
+ if (secretKey != null) {
+ token = secretKey.getBytes();
+ }
+ LOG.debug("SASL credentials for storm topology " + topologyName
+ + " is " + secretKey);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/3bce04ce/storm-core/src/jvm/backtype/storm/messaging/netty/SaslStormServerHandler.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/messaging/netty/SaslStormServerHandler.java b/storm-core/src/jvm/backtype/storm/messaging/netty/SaslStormServerHandler.java
index d06e960..02448e2 100644
--- a/storm-core/src/jvm/backtype/storm/messaging/netty/SaslStormServerHandler.java
+++ b/storm-core/src/jvm/backtype/storm/messaging/netty/SaslStormServerHandler.java
@@ -32,124 +32,124 @@ import backtype.storm.Config;
public class SaslStormServerHandler extends SimpleChannelUpstreamHandler {
- Server server;
- /** Used for client or server's token to send or receive from each other. */
- private byte[] token;
- private String topologyName;
-
- private static final Logger LOG = LoggerFactory
- .getLogger(SaslStormServerHandler.class);
-
- public SaslStormServerHandler(Server server) throws IOException {
- this.server = server;
- getSASLCredentials();
- }
-
- @Override
- public void messageReceived(ChannelHandlerContext ctx, MessageEvent e)
- throws Exception {
- Object msg = e.getMessage();
- if (msg == null)
- return;
-
- Channel channel = ctx.getChannel();
-
- if (msg instanceof ControlMessage
- && ((ControlMessage) e.getMessage()) == ControlMessage.SASL_TOKEN_MESSAGE_REQUEST) {
- // initialize server-side SASL functionality, if we haven't yet
- // (in which case we are looking at the first SASL message from the
- // client).
- SaslNettyServer saslNettyServer = SaslNettyServerState.getSaslNettyServer
- .get(channel);
- if (saslNettyServer == null) {
- LOG.debug("No saslNettyServer for " + channel
- + " yet; creating now, with topology token: ");
- try {
- saslNettyServer = new SaslNettyServer(topologyName, token);
- } catch (IOException ioe) {
- LOG.error("Error occurred while creating saslNettyServer on server "
- + channel.getLocalAddress()
- + " for client "
- + channel.getRemoteAddress());
- throw new IOException(ioe);
- }
-
- SaslNettyServerState.getSaslNettyServer.set(channel,
- saslNettyServer);
- } else {
- LOG.debug("Found existing saslNettyServer on server:"
- + channel.getLocalAddress() + " for client "
- + channel.getRemoteAddress());
- }
-
- LOG.debug("processToken: With nettyServer: " + saslNettyServer
- + " and token length: " + token.length);
-
- SaslMessageToken saslTokenMessageRequest = null;
- saslTokenMessageRequest = new SaslMessageToken(
- saslNettyServer.response(new byte[0]));
- // Send response to client.
- channel.write(saslTokenMessageRequest);
- // do not send upstream to other handlers: no further action needs
- // to be done for SASL_TOKEN_MESSAGE_REQUEST requests.
- return;
- }
-
- if (msg instanceof SaslMessageToken) {
- // initialize server-side SASL functionality, if we haven't yet
- // (in which case we are looking at the first SASL message from the
- // client).
- SaslNettyServer saslNettyServer = SaslNettyServerState.getSaslNettyServer
- .get(channel);
- if (saslNettyServer == null) {
- if (saslNettyServer == null) {
- throw new Exception("saslNettyServer was unexpectedly "
- + "null for channel: " + channel);
- }
- }
- SaslMessageToken saslTokenMessageRequest = new SaslMessageToken(
- saslNettyServer.response(((SaslMessageToken) msg)
- .getSaslToken()));
-
- // Send response to client.
- channel.write(saslTokenMessageRequest);
-
- if (saslNettyServer.isComplete()) {
- // If authentication of client is complete, we will also send a
- // SASL-Complete message to the client.
- LOG.debug("SASL authentication is complete for client with "
- + "username: " + saslNettyServer.getUserName());
- channel.write(ControlMessage.SASL_COMPLETE_REQUEST);
- LOG.debug("Removing SaslServerHandler from pipeline since SASL "
- + "authentication is complete.");
- ctx.getPipeline().remove(this);
- }
- return;
- } else {
- // Client should not be sending other-than-SASL messages before
- // SaslServerHandler has removed itself from the pipeline. Such
- // non-SASL requests will be denied by the Authorize channel handler
- // (the next handler upstream in the server pipeline) if SASL
- // authentication has not completed.
- LOG.warn("Sending upstream an unexpected non-SASL message : "
- + msg);
- Channels.fireMessageReceived(ctx, msg);
- }
- }
-
- @Override
- public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) {
- server.closeChannel(e.getChannel());
- }
-
- private void getSASLCredentials() throws IOException {
- topologyName = (String) this.server.storm_conf
- .get(Config.TOPOLOGY_NAME);
- String secretKey = SaslUtils.getSecretKey(this.server.storm_conf);
- if (secretKey != null) {
- token = secretKey.getBytes();
- }
- LOG.debug("SASL credentials for storm topology " + topologyName
- + " is " + secretKey);
- }
+ Server server;
+ /** Used for client or server's token to send or receive from each other. */
+ private byte[] token;
+ private String topologyName;
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(SaslStormServerHandler.class);
+
+ public SaslStormServerHandler(Server server) throws IOException {
+ this.server = server;
+ getSASLCredentials();
+ }
+
+ @Override
+ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e)
+ throws Exception {
+ Object msg = e.getMessage();
+ if (msg == null)
+ return;
+
+ Channel channel = ctx.getChannel();
+
+ if (msg instanceof ControlMessage
+ && ((ControlMessage) e.getMessage()) == ControlMessage.SASL_TOKEN_MESSAGE_REQUEST) {
+ // initialize server-side SASL functionality, if we haven't yet
+ // (in which case we are looking at the first SASL message from the
+ // client).
+ SaslNettyServer saslNettyServer = SaslNettyServerState.getSaslNettyServer
+ .get(channel);
+ if (saslNettyServer == null) {
+ LOG.debug("No saslNettyServer for " + channel
+ + " yet; creating now, with topology token: ");
+ try {
+ saslNettyServer = new SaslNettyServer(topologyName, token);
+ } catch (IOException ioe) {
+ LOG.error("Error occurred while creating saslNettyServer on server "
+ + channel.getLocalAddress()
+ + " for client "
+ + channel.getRemoteAddress());
+ saslNettyServer = null;
+ }
+
+ SaslNettyServerState.getSaslNettyServer.set(channel,
+ saslNettyServer);
+ } else {
+ LOG.debug("Found existing saslNettyServer on server:"
+ + channel.getLocalAddress() + " for client "
+ + channel.getRemoteAddress());
+ }
+
+ LOG.debug("processToken: With nettyServer: " + saslNettyServer
+ + " and token length: " + token.length);
+
+ SaslMessageToken saslTokenMessageRequest = null;
+ saslTokenMessageRequest = new SaslMessageToken(
+ saslNettyServer.response(new byte[0]));
+ // Send response to client.
+ channel.write(saslTokenMessageRequest);
+ // do not send upstream to other handlers: no further action needs
+ // to be done for SASL_TOKEN_MESSAGE_REQUEST requests.
+ return;
+ }
+
+ if (msg instanceof SaslMessageToken) {
+ // initialize server-side SASL functionality, if we haven't yet
+ // (in which case we are looking at the first SASL message from the
+ // client).
+ SaslNettyServer saslNettyServer = SaslNettyServerState.getSaslNettyServer
+ .get(channel);
+ if (saslNettyServer == null) {
+ if (saslNettyServer == null) {
+ throw new Exception("saslNettyServer was unexpectedly "
+ + "null for channel: " + channel);
+ }
+ }
+ SaslMessageToken saslTokenMessageRequest = new SaslMessageToken(
+ saslNettyServer.response(((SaslMessageToken) msg)
+ .getSaslToken()));
+
+ // Send response to client.
+ channel.write(saslTokenMessageRequest);
+
+ if (saslNettyServer.isComplete()) {
+ // If authentication of client is complete, we will also send a
+ // SASL-Complete message to the client.
+ LOG.debug("SASL authentication is complete for client with "
+ + "username: " + saslNettyServer.getUserName());
+ channel.write(ControlMessage.SASL_COMPLETE_REQUEST);
+ LOG.debug("Removing SaslServerHandler from pipeline since SASL "
+ + "authentication is complete.");
+ ctx.getPipeline().remove(this);
+ }
+ return;
+ } else {
+ // Client should not be sending other-than-SASL messages before
+ // SaslServerHandler has removed itself from the pipeline. Such
+ // non-SASL requests will be denied by the Authorize channel handler
+ // (the next handler upstream in the server pipeline) if SASL
+ // authentication has not completed.
+ LOG.warn("Sending upstream an unexpected non-SASL message : "
+ + msg);
+ Channels.fireMessageReceived(ctx, msg);
+ }
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) {
+ server.closeChannel(e.getChannel());
+ }
+
+ private void getSASLCredentials() throws IOException {
+ topologyName = (String) this.server.storm_conf
+ .get(Config.TOPOLOGY_NAME);
+ String secretKey = SaslUtils.getSecretKey(this.server.storm_conf);
+ if (secretKey != null) {
+ token = secretKey.getBytes();
+ }
+ LOG.debug("SASL credentials for storm topology " + topologyName
+ + " is " + secretKey);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/3bce04ce/storm-core/src/jvm/backtype/storm/messaging/netty/SaslUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/messaging/netty/SaslUtils.java b/storm-core/src/jvm/backtype/storm/messaging/netty/SaslUtils.java
index 0f96233..a4cc0ba 100644
--- a/storm-core/src/jvm/backtype/storm/messaging/netty/SaslUtils.java
+++ b/storm-core/src/jvm/backtype/storm/messaging/netty/SaslUtils.java
@@ -28,48 +28,47 @@ import org.apache.commons.codec.binary.Base64;
import backtype.storm.Config;
class SaslUtils {
- public static final String AUTH_DIGEST_MD5 = "DIGEST-MD5";
- public static final String DEFAULT_REALM = "default";
+ public static final String AUTH_DIGEST_MD5 = "DIGEST-MD5";
+ public static final String DEFAULT_REALM = "default";
- static Map<String, String> getSaslProps() {
- Map<String, String> props = new HashMap<String, String>();
- props.put(Sasl.POLICY_NOPLAINTEXT, "true");
- return props;
- }
+ static Map<String, String> getSaslProps() {
+ Map<String, String> props = new HashMap<String, String>();
+ props.put(Sasl.POLICY_NOPLAINTEXT, "true");
+ return props;
+ }
- /**
- * Encode a password as a base64-encoded char[] array.
- *
- * @param password
- * as a byte array.
- * @return password as a char array.
- */
- static char[] encodePassword(byte[] password) {
- return new String(Base64.encodeBase64(password),
- Charset.defaultCharset()).toCharArray();
- }
+ /**
+ * Encode a password as a base64-encoded char[] array.
+ *
+ * @param password
+ * as a byte array.
+ * @return password as a char array.
+ */
+ static char[] encodePassword(byte[] password) {
+ return new String(Base64.encodeBase64(password),
+ Charset.defaultCharset()).toCharArray();
+ }
- /**
- * Encode a identifier as a base64-encoded char[] array.
- *
- * @param identifier
- * as a byte array.
- * @return identifier as a char array.
- */
- static String encodeIdentifier(byte[] identifier) {
- return new String(Base64.encodeBase64(identifier),
- Charset.defaultCharset());
- }
+ /**
+ * Encode a identifier as a base64-encoded char[] array.
+ *
+ * @param identifier
+ * as a byte array.
+ * @return identifier as a char array.
+ */
+ static String encodeIdentifier(byte[] identifier) {
+ return new String(Base64.encodeBase64(identifier),
+ Charset.defaultCharset());
+ }
- static String getSecretKey(Map conf) {
- if (conf == null || conf.isEmpty())
- return null;
+ static String getSecretKey(Map conf) {
+ if (conf == null || conf.isEmpty())
+ return null;
- String secretPayLoad = (String) conf
- .get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD);
+ String secretPayLoad = (String) conf
+ .get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD);
+
+ return secretPayLoad;
+ }
- return secretPayLoad;
- }
-
-
}
http://git-wip-us.apache.org/repos/asf/storm/blob/3bce04ce/storm-core/src/jvm/backtype/storm/messaging/netty/StormClientPipelineFactory.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/messaging/netty/StormClientPipelineFactory.java b/storm-core/src/jvm/backtype/storm/messaging/netty/StormClientPipelineFactory.java
index 4fdaee9..1ea382b 100644
--- a/storm-core/src/jvm/backtype/storm/messaging/netty/StormClientPipelineFactory.java
+++ b/storm-core/src/jvm/backtype/storm/messaging/netty/StormClientPipelineFactory.java
@@ -27,31 +27,27 @@ class StormClientPipelineFactory implements ChannelPipelineFactory {
private Client client;
StormClientPipelineFactory(Client client) {
- this.client = client;
+ this.client = client;
}
public ChannelPipeline getPipeline() throws Exception {
// Create a default pipeline implementation.
ChannelPipeline pipeline = Channels.pipeline();
- boolean isNettyAuth = (Boolean) this.client.storm_conf.get(Config.STORM_MESSAGING_NETTY_AUTHENTICATION);
- if(isNettyAuth) {
- // Decoder
- pipeline.addLast("decoder", new MessageDecoder());
- // Encoder
- pipeline.addLast("encoder", new MessageEncoder());
- // Authenticate: Removed after authentication completes
- pipeline.addLast("saslClientHandler", new SaslStormClientHandler(client));
- // business logic.
- pipeline.addLast("handler", new StormClientErrorHandler(client.name()));
- } else {
- // Decoder
- pipeline.addLast("decoder", new MessageDecoder());
- // Encoder
- pipeline.addLast("encoder", new MessageEncoder());
- // business logic.
- pipeline.addLast("handler", new StormClientErrorHandler(client.name()));
+ // Decoder
+ pipeline.addLast("decoder", new MessageDecoder());
+ // Encoder
+ pipeline.addLast("encoder", new MessageEncoder());
+
+ boolean isNettyAuth = (Boolean) this.client.storm_conf
+ .get(Config.STORM_MESSAGING_NETTY_AUTHENTICATION);
+ if (isNettyAuth) {
+ // Authenticate: Removed after authentication completes
+ pipeline.addLast("saslClientHandler", new SaslStormClientHandler(
+ client));
}
+ // business logic.
+ pipeline.addLast("handler", new StormClientErrorHandler(client.name()));
return pipeline;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/3bce04ce/storm-core/src/jvm/backtype/storm/messaging/netty/StormServerPipelineFactory.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/messaging/netty/StormServerPipelineFactory.java b/storm-core/src/jvm/backtype/storm/messaging/netty/StormServerPipelineFactory.java
index c2b4c53..f6e20c5 100644
--- a/storm-core/src/jvm/backtype/storm/messaging/netty/StormServerPipelineFactory.java
+++ b/storm-core/src/jvm/backtype/storm/messaging/netty/StormServerPipelineFactory.java
@@ -22,41 +22,36 @@ import org.jboss.netty.channel.ChannelPipelineFactory;
import org.jboss.netty.channel.Channels;
import backtype.storm.Config;
-import backtype.storm.utils.Utils;
-
-class StormServerPipelineFactory implements ChannelPipelineFactory {
+class StormServerPipelineFactory implements ChannelPipelineFactory {
private Server server;
-
+
StormServerPipelineFactory(Server server) {
- this.server = server;
+ this.server = server;
}
-
+
public ChannelPipeline getPipeline() throws Exception {
// Create a default pipeline implementation.
ChannelPipeline pipeline = Channels.pipeline();
- boolean isNettyAuth = (Boolean) this.server.storm_conf.get(Config.STORM_MESSAGING_NETTY_AUTHENTICATION);
- if(isNettyAuth) {
- // Decoder
- pipeline.addLast("decoder", new MessageDecoder());
- // Encoder
- pipeline.addLast("encoder", new MessageEncoder());
+ // Decoder
+ pipeline.addLast("decoder", new MessageDecoder());
+ // Encoder
+ pipeline.addLast("encoder", new MessageEncoder());
+
+ boolean isNettyAuth = (Boolean) this.server.storm_conf
+ .get(Config.STORM_MESSAGING_NETTY_AUTHENTICATION);
+ if (isNettyAuth) {
// Authenticate: Removed after authentication completes
- pipeline.addLast("saslServerHandler", new SaslStormServerHandler(server));
+ pipeline.addLast("saslServerHandler", new SaslStormServerHandler(
+ server));
// Authorize
- pipeline.addLast("authorizeServerHandler", new SaslStormServerAuthorizeHandler());
- // business logic.
- pipeline.addLast("handler", new StormServerHandler(server));
- } else {
- // Decoder
- pipeline.addLast("decoder", new MessageDecoder());
- // Encoder
- pipeline.addLast("encoder", new MessageEncoder());
- // business logic.
- pipeline.addLast("handler", new StormServerHandler(server));
+ pipeline.addLast("authorizeServerHandler",
+ new SaslStormServerAuthorizeHandler());
}
-
+ // business logic.
+ pipeline.addLast("handler", new StormServerHandler(server));
+
return pipeline;
}
}
[44/50] storm git commit: STORM-509. Make groups checking specific
for SimpleACLAuthorizer.
Posted by bo...@apache.org.
STORM-509. Make groups checking specific for SimpleACLAuthorizer.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/0f6719d4
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/0f6719d4
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/0f6719d4
Branch: refs/heads/master
Commit: 0f6719d4303cae6fc8521b802bf86f077278e160
Parents: 11e6fc9
Author: Sriharsha Chintalapani <ma...@harsha.io>
Authored: Tue Sep 30 11:43:37 2014 -0700
Committer: Sriharsha Chintalapani <ma...@harsha.io>
Committed: Tue Sep 30 11:43:37 2014 -0700
----------------------------------------------------------------------
.../storm/security/auth/authorizer/SimpleACLAuthorizer.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/0f6719d4/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleACLAuthorizer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleACLAuthorizer.java b/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleACLAuthorizer.java
index daef956..e8425bf 100644
--- a/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleACLAuthorizer.java
+++ b/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleACLAuthorizer.java
@@ -115,11 +115,9 @@ public class SimpleACLAuthorizer implements IAuthorizer {
}
if(_groupMappingProvider != null && topoGroups.size() > 0) {
- System.out.println("hello in if");
try {
Set<String> userGroups = _groupMappingProvider.getGroups(user);
for (String tgroup : topoGroups) {
- System.out.println("tgroup "+tgroup);
if(userGroups.contains(tgroup))
return true;
}
[46/50] storm git commit: Merge branch 'STORM-502' of
https://github.com/harshach/incubator-storm into STORM-502-merge
Posted by bo...@apache.org.
Merge branch 'STORM-502' of https://github.com/harshach/incubator-storm into STORM-502-merge
STORM-502. storm native worker-launcher tests fail.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/4788c7ec
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/4788c7ec
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/4788c7ec
Branch: refs/heads/master
Commit: 4788c7ecebefb670c1dbe85a32ed26bd85c4e807
Parents: fce22a7 02f6db5
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Tue Oct 7 16:27:56 2014 -0500
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Tue Oct 7 16:27:56 2014 -0500
----------------------------------------------------------------------
storm-core/src/native/worker-launcher/test/test-worker-launcher.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[06/50] storm git commit: STORM-348: Netty SASL Authentication
Posted by bo...@apache.org.
STORM-348: Netty SASL Authentication
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/90cf0e6c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/90cf0e6c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/90cf0e6c
Branch: refs/heads/master
Commit: 90cf0e6c21df978462d29fe906ad0f2a6adcdb1f
Parents: 02e7949
Author: Raghavendra Nandagopal <sp...@gmail.com>
Authored: Tue Jul 29 17:32:38 2014 -0700
Committer: Raghavendra Nandagopal <sp...@gmail.com>
Committed: Tue Jul 29 17:32:38 2014 -0700
----------------------------------------------------------------------
.../test/clj/backtype/storm/messaging/netty_integration_test.clj | 1 +
storm-core/test/clj/backtype/storm/messaging/netty_unit_test.clj | 4 ++++
2 files changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/90cf0e6c/storm-core/test/clj/backtype/storm/messaging/netty_integration_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/messaging/netty_integration_test.clj b/storm-core/test/clj/backtype/storm/messaging/netty_integration_test.clj
index 8534c82..dea4abe 100644
--- a/storm-core/test/clj/backtype/storm/messaging/netty_integration_test.clj
+++ b/storm-core/test/clj/backtype/storm/messaging/netty_integration_test.clj
@@ -25,6 +25,7 @@
(with-simulated-time-local-cluster [cluster :supervisors 4 :supervisor-slot-port-min 6710
:daemon-conf {STORM-LOCAL-MODE-ZMQ true
STORM-MESSAGING-TRANSPORT "backtype.storm.messaging.netty.Context"
+ STORM_MESSAGING_NETTY_AUTHENTICATION false
STORM-MESSAGING-NETTY-BUFFER-SIZE 1024000
STORM-MESSAGING-NETTY-MAX-RETRIES 10
STORM-MESSAGING-NETTY-MIN-SLEEP-MS 1000
http://git-wip-us.apache.org/repos/asf/storm/blob/90cf0e6c/storm-core/test/clj/backtype/storm/messaging/netty_unit_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/messaging/netty_unit_test.clj b/storm-core/test/clj/backtype/storm/messaging/netty_unit_test.clj
index ea7b8dc..04d25ec 100644
--- a/storm-core/test/clj/backtype/storm/messaging/netty_unit_test.clj
+++ b/storm-core/test/clj/backtype/storm/messaging/netty_unit_test.clj
@@ -26,6 +26,7 @@
(deftest test-basic
(let [req_msg (String. "0123456789abcdefghijklmnopqrstuvwxyz")
storm-conf {STORM-MESSAGING-TRANSPORT "backtype.storm.messaging.netty.Context"
+ STORM_MESSAGING_NETTY_AUTHENTICATION false
STORM-MESSAGING-NETTY-BUFFER-SIZE 1024
STORM-MESSAGING-NETTY-MAX-RETRIES 10
STORM-MESSAGING-NETTY-MIN-SLEEP-MS 1000
@@ -48,6 +49,7 @@
(deftest test-large-msg
(let [req_msg (apply str (repeat 2048000 'c'))
storm-conf {STORM-MESSAGING-TRANSPORT "backtype.storm.messaging.netty.Context"
+ STORM_MESSAGING_NETTY_AUTHENTICATION false
STORM-MESSAGING-NETTY-BUFFER-SIZE 102400
STORM-MESSAGING-NETTY-MAX-RETRIES 10
STORM-MESSAGING-NETTY-MIN-SLEEP-MS 1000
@@ -70,6 +72,7 @@
(deftest test-server-delayed
(let [req_msg (String. "0123456789abcdefghijklmnopqrstuvwxyz")
storm-conf {STORM-MESSAGING-TRANSPORT "backtype.storm.messaging.netty.Context"
+ STORM_MESSAGING_NETTY_AUTHENTICATION false
STORM-MESSAGING-NETTY-BUFFER-SIZE 1024
STORM-MESSAGING-NETTY-MAX-RETRIES 10
STORM-MESSAGING-NETTY-MIN-SLEEP-MS 1000
@@ -99,6 +102,7 @@
(deftest test-batch
(let [storm-conf {STORM-MESSAGING-TRANSPORT "backtype.storm.messaging.netty.Context"
+ STORM_MESSAGING_NETTY_AUTHENTICATION false
STORM-MESSAGING-NETTY-BUFFER-SIZE 1024000
STORM-MESSAGING-NETTY-MAX-RETRIES 10
STORM-MESSAGING-NETTY-MIN-SLEEP-MS 1000
[40/50] storm git commit: Merge branch 'master' into security-upmerge
Posted by bo...@apache.org.
Merge branch 'master' into security-upmerge
Conflicts:
storm-core/src/jvm/backtype/storm/messaging/netty/Client.java
storm-core/src/jvm/backtype/storm/utils/Utils.java
storm-core/src/ui/public/templates/component-page-template.html
storm-core/src/ui/public/templates/index-page-template.html
storm-core/src/ui/public/templates/topology-page-template.html
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/154ec7b6
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/154ec7b6
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/154ec7b6
Branch: refs/heads/master
Commit: 154ec7b6daec34c6289eaed1846d6bfba809f6ab
Parents: aa6e7ed cfcedcf
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Wed Sep 24 12:55:04 2014 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Wed Sep 24 12:55:04 2014 -0500
----------------------------------------------------------------------
CHANGELOG.md | 14 +
DEVELOPER.md | 2 +-
LICENSE | 25 +
README.markdown | 5 +-
STORM-UI-REST-API.md | 4 +-
dev-tools/jira-github-join.py | 2 +-
.../multilang/resources/asyncSplitsentence.js | 32 +
.../multilang/resources/randomsentence.js | 75 +++
.../multilang/resources/splitsentence.js | 26 +
.../storm-starter/multilang/resources/storm.js | 349 +++++++++++
.../storm/starter/WordCountTopologyNode.java | 121 ++++
.../src/jvm/storm/kafka/KafkaUtils.java | 5 +-
.../src/jvm/storm/kafka/PartitionManager.java | 5 +-
pom.xml | 2 +-
storm-core/pom.xml | 53 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 4 +-
storm-core/src/clj/backtype/storm/util.clj | 2 +-
storm-core/src/dev/resources/storm.js | 349 +++++++++++
storm-core/src/dev/resources/tester_bolt.js | 27 +
storm-core/src/dev/resources/tester_spout.js | 58 ++
.../src/jvm/backtype/storm/StormSubmitter.java | 17 +
.../backtype/storm/messaging/netty/Client.java | 27 +-
.../storm/security/auth/TBackoffConnect.java | 10 +-
.../StormBoundedExponentialBackoffRetry.java | 74 +++
.../src/jvm/backtype/storm/utils/Utils.java | 33 +-
storm-core/src/ui/public/component.html | 6 +
storm-core/src/ui/public/images/spinner.gif | Bin 0 -> 23470 bytes
storm-core/src/ui/public/index.html | 6 +
.../src/ui/public/js/jquery.blockUI.min.js | 6 +
storm-core/src/ui/public/js/script.js | 11 +
storm-core/src/ui/public/js/visualization.js | 16 +-
.../templates/component-page-template.html | 593 ++++++++++++++-----
.../public/templates/index-page-template.html | 243 ++++++--
.../templates/topology-page-template.html | 208 ++++++-
storm-core/src/ui/public/topology.html | 6 +
.../test/clj/backtype/storm/multilang_test.clj | 55 +-
.../test/clj/backtype/storm/utils_test.clj | 3 +-
...StormBoundedExponentialBackoffRetryTest.java | 101 ++++
38 files changed, 2259 insertions(+), 316 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/154ec7b6/README.markdown
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/154ec7b6/STORM-UI-REST-API.md
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/154ec7b6/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/154ec7b6/storm-core/pom.xml
----------------------------------------------------------------------
diff --cc storm-core/pom.xml
index 649b183,4c72498..95b634f
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@@ -58,22 -53,14 +58,28 @@@
<dependency>
<groupId>ring</groupId>
<artifactId>ring-jetty-adapter</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
+ <groupId>ring</groupId>
+ <artifactId>ring-anti-forgery</artifactId>
+ <version>1.0.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlets</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.clojure</groupId>
<artifactId>tools.logging</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/storm/blob/154ec7b6/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/154ec7b6/storm-core/src/clj/backtype/storm/util.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/154ec7b6/storm-core/src/jvm/backtype/storm/StormSubmitter.java
----------------------------------------------------------------------
diff --cc storm-core/src/jvm/backtype/storm/StormSubmitter.java
index d5da103,282c828..54cc701
--- a/storm-core/src/jvm/backtype/storm/StormSubmitter.java
+++ b/storm-core/src/jvm/backtype/storm/StormSubmitter.java
@@@ -158,6 -71,22 +158,23 @@@ public class StormSubmitter
}
/**
+ * Submits a topology to run on the cluster. A topology runs forever or until
+ * explicitly killed.
+ *
+ * @param name the name of the storm.
+ * @param stormConf the topology-specific configuration. See {@link Config}.
+ * @param topology the processing to execute.
+ * @param opts to manipulate the starting of the topology.
+ * @throws AlreadyAliveException if a topology with this name is already running
+ * @throws InvalidTopologyException if an invalid topology was submitted
++ * @throws AuthorizationException if authorization is failed
+ */
+ public static void submitTopology(String name, Map stormConf, StormTopology topology, SubmitOptions opts)
- throws AlreadyAliveException, InvalidTopologyException {
++ throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {
+ submitTopology(name, stormConf, topology, opts, null);
+ }
+
+ /**
* Submits a topology to run on the cluster. A topology runs forever or until
* explicitly killed.
*
http://git-wip-us.apache.org/repos/asf/storm/blob/154ec7b6/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java
----------------------------------------------------------------------
diff --cc storm-core/src/jvm/backtype/storm/messaging/netty/Client.java
index 5440259,fed684e..4c4b092
--- a/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java
+++ b/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java
@@@ -28,19 -40,6 +28,20 @@@ import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
+import org.jboss.netty.bootstrap.ClientBootstrap;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelFactory;
+import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelFutureListener;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.Config;
+import backtype.storm.messaging.IConnection;
+import backtype.storm.messaging.TaskMessage;
++import backtype.storm.utils.StormBoundedExponentialBackoffRetry;
+import backtype.storm.utils.Utils;
+
public class Client implements IConnection {
private static final Logger LOG = LoggerFactory.getLogger(Client.class);
private static final String PREFIX = "Netty-Client-";
http://git-wip-us.apache.org/repos/asf/storm/blob/154ec7b6/storm-core/src/jvm/backtype/storm/security/auth/TBackoffConnect.java
----------------------------------------------------------------------
diff --cc storm-core/src/jvm/backtype/storm/security/auth/TBackoffConnect.java
index b50ae6e,0000000..eaed61f
mode 100644,000000..100644
--- a/storm-core/src/jvm/backtype/storm/security/auth/TBackoffConnect.java
+++ b/storm-core/src/jvm/backtype/storm/security/auth/TBackoffConnect.java
@@@ -1,77 -1,0 +1,77 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package backtype.storm.security.auth;
+
+import java.io.IOException;
+import java.util.Random;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
- import backtype.storm.utils.Utils.BoundedExponentialBackoffRetry;
++import backtype.storm.utils.StormBoundedExponentialBackoffRetry;
+
+public class TBackoffConnect {
+ private static final Logger LOG = LoggerFactory.getLogger(TBackoffConnect.class);
+ private int _completedRetries = 0;
+ private int _retryTimes;
- private BoundedExponentialBackoffRetry waitGrabber;
++ private StormBoundedExponentialBackoffRetry waitGrabber;
+
+ public TBackoffConnect(int retryTimes, int retryInterval, int retryIntervalCeiling) {
+
+ _retryTimes = retryTimes;
- waitGrabber = new BoundedExponentialBackoffRetry(retryInterval,
- retryTimes,
- retryIntervalCeiling);
++ waitGrabber = new StormBoundedExponentialBackoffRetry(retryInterval,
++ retryIntervalCeiling,
++ retryTimes);
+ }
+
+ public TTransport doConnectWithRetry(ITransportPlugin transportPlugin, TTransport underlyingTransport, String host) throws IOException {
+ boolean connected = false;
+ TTransport transportResult = null;
+ while(!connected) {
+ try {
+ transportResult = transportPlugin.connect(underlyingTransport, host);
+ connected = true;
+ } catch (TTransportException ex) {
+ retryNext(ex);
+ }
+ }
+ return transportResult;
+ }
+
+ private void retryNext(TTransportException ex) {
+ if(!canRetry()) {
+ throw new RuntimeException(ex);
+ }
+ try {
+ int sleeptime = waitGrabber.getSleepTimeMs(_completedRetries, 0);
+
+ LOG.debug("Failed to connect. Retrying... (" + Integer.toString( _completedRetries) + ") in " + Integer.toString(sleeptime) + "ms");
+
+ Thread.sleep(sleeptime);
+ } catch (InterruptedException e) {
+ LOG.info("Nimbus connection retry interrupted.");
+ }
+
+ _completedRetries++;
+ }
+
+ private boolean canRetry() {
+ return (_completedRetries < _retryTimes);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/154ec7b6/storm-core/src/jvm/backtype/storm/utils/Utils.java
----------------------------------------------------------------------
diff --cc storm-core/src/jvm/backtype/storm/utils/Utils.java
index 6d31d28,0e44479..6e8458a
--- a/storm-core/src/jvm/backtype/storm/utils/Utils.java
+++ b/storm-core/src/jvm/backtype/storm/utils/Utils.java
@@@ -47,15 -46,11 +47,14 @@@ import backtype.storm.serialization.Def
import backtype.storm.serialization.SerializationDelegate;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
- import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.commons.lang.StringUtils;
import org.apache.thrift.TException;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+import org.json.simple.JSONValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.json.simple.JSONValue;
import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.constructor.SafeConstructor;
@@@ -347,27 -338,9 +346,8 @@@ public class Utils
return UUID.randomUUID().getLeastSignificantBits();
}
- public static class BoundedExponentialBackoffRetry extends ExponentialBackoffRetry {
-
- protected final int maxRetryInterval;
-
- public BoundedExponentialBackoffRetry(int baseSleepTimeMs,
- int maxRetries, int maxSleepTimeMs) {
- super(baseSleepTimeMs, maxRetries);
- this.maxRetryInterval = maxSleepTimeMs;
- }
-
- public int getMaxRetryInterval() {
- return this.maxRetryInterval;
- }
-
- @Override
- public int getSleepTimeMs(int count, long elapsedMs)
- {
- return Math.min(maxRetryInterval,
- super.getSleepTimeMs(count, elapsedMs));
- }
-
-
+ public static CuratorFramework newCurator(Map conf, List<String> servers, Object port, String root) {
+ return newCurator(conf, servers, port, root, null);
}
public static CuratorFramework newCurator(Map conf, List<String> servers, Object port, String root, ZookeeperAuthInfo auth) {
@@@ -376,33 -349,26 +356,34 @@@
serverPorts.add(zkServer + ":" + Utils.getInt(port));
}
String zkStr = StringUtils.join(serverPorts, ",") + root;
- CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder()
- .connectString(zkStr)
- .connectionTimeoutMs(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT)))
- .sessionTimeoutMs(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)))
- .retryPolicy(new StormBoundedExponentialBackoffRetry(
- Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)),
- Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING)),
- Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES))));
- if(auth!=null && auth.scheme!=null) {
+ CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder();
+
+ setupBuilder(builder, zkStr, conf, auth);
+
+ return builder.build();
+ }
+
+ protected static void setupBuilder(CuratorFrameworkFactory.Builder builder, String zkStr, Map conf, ZookeeperAuthInfo auth)
+ {
+ builder.connectString(zkStr)
+ .connectionTimeoutMs(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT)))
+ .sessionTimeoutMs(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)))
- .retryPolicy(new BoundedExponentialBackoffRetry(
- Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)),
- Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)),
- Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING))));
++ .retryPolicy(new StormBoundedExponentialBackoffRetry(
++ Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)),
++ Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING)),
++ Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES))));
++
+ if(auth!=null && auth.scheme!=null && auth.payload!=null) {
builder = builder.authorization(auth.scheme, auth.payload);
}
- return builder.build();
}
- public static CuratorFramework newCurator(Map conf, List<String> servers, Object port) {
- return newCurator(conf, servers, port, "");
+ public static CuratorFramework newCurator(Map conf, List<String> servers, Object port, ZookeeperAuthInfo auth) {
+ return newCurator(conf, servers, port, "", auth);
}
- public static CuratorFramework newCuratorStarted(Map conf, List<String> servers, Object port, String root) {
- CuratorFramework ret = newCurator(conf, servers, port, root);
+ public static CuratorFramework newCuratorStarted(Map conf, List<String> servers, Object port, String root, ZookeeperAuthInfo auth) {
+ CuratorFramework ret = newCurator(conf, servers, port, root, auth);
ret.start();
return ret;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/154ec7b6/storm-core/src/ui/public/component.html
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/154ec7b6/storm-core/src/ui/public/index.html
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/154ec7b6/storm-core/src/ui/public/js/script.js
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/154ec7b6/storm-core/src/ui/public/templates/index-page-template.html
----------------------------------------------------------------------
diff --cc storm-core/src/ui/public/templates/index-page-template.html
index dfe9446,d249fac..800b579
--- a/storm-core/src/ui/public/templates/index-page-template.html
+++ b/storm-core/src/ui/public/templates/index-page-template.html
@@@ -31,48 -75,103 +75,115 @@@
</table>
</script>
<script id="topology-summary-template" type="text/html">
- <table class="zebra-striped" id="topology-summary-table">
- <thead><tr>
- <th><span class="tip right" title="The name given to the topology by when it was submitted. Click the name to view the Topology's information.">Name</span></th>
- <th><span class="tip right" title="The unique ID given to a Topology each time it is launched.">Id</span></th>
- <th><span class="tip above" title="The user that submitted the Topology, if authentication is enabled.">Owner</span></th>
- <th><span class="tip above" title="The status can be one of ACTIVE, INACTIVE, KILLED, or REBALANCING.">Status</span></th>
- <th><span class="tip above" title="The time since the Topology was submitted.">Uptime</span></th>
- <th><span class="tip above" title="The number of Workers (processes).">Num workers</span></th>
- <th><span class="tip above" title="Executors are threads in a Worker process.">Num executors</span></th>
- <th><span class="tip above" title="A Task is an instance of a Bolt or Spout. The number of Tasks is almost always equal to the number of Executors.">Num tasks</span></th>
- <th><span class="tip left" title="This shows information from the scheduler about the latest attempt to schedule the Topology on the cluster.">Scheduler Info</span></th>
- </tr></thead>
- <tbody>
- {{#topologies}}
- <tr>
- <td><a href="/topology.html?id={{id}}">{{name}}</a></td>
- <td>{{id}}</td>
- <td>{{owner}}</td>
- <td>{{status}}</td>
- <td>{{uptime}}</td>
- <td>{{workersTotal}}</td>
- <td>{{executorsTotal}}</td>
- <td>{{tasksTotal}}</td>
- <td>{{schedulerInfo}}</td>
- </tr>
- {{/topologies}}
- </tbody>
- </table>
+ <table class="zebra-striped" id="topology-summary-table">
+ <thead>
+ <tr>
+ <th>
+ <span class="tip right" title="The name given to the topology by when it was submitted. Click the name to view the Topology's information.">
+ Name
+ </span>
+ </th>
+ <th>
+ <span class="tip right" title="The unique ID given to a Topology each time it is launched.">
+ Id
+ </span>
+ </th>
+ <th>
++ <span class="tip above" title="The user that submitted the Topology, if authentication is enabled.">
++ Owner
++ </span>
++ </th>
++ <th>
+ <span class="tip above" title="The status can be one of ACTIVE, INACTIVE, KILLED, or REBALANCING.">
+ Status
+ </span>
+ </th>
+ <th>
+ <span class="tip above" title="The time since the Topology was submitted.">
+ Uptime
+ </span>
+ </th>
+ <th>
+ <span class="tip above" title="The number of Workers (processes).">
+ Num workers
+ </span>
+ </th>
+ <th>
+ <span class="tip above" title="Executors are threads in a Worker process.">
+ Num executors
+ </span>
+ </th>
+ <th>
+ <span class="tip above" title="A Task is an instance of a Bolt or Spout. The number of Tasks is almost always equal to the number of Executors.">
+ Num tasks
+ </span>
+ </th>
++ <th>
++ <span class="tip left" title="This shows information from the scheduler about the latest attempt to schedule the Topology on the cluster.">
++ Scheduler Info
++ </span>
++ </th>
+ </tr>
+ </thead>
+ <tbody>
+ {{#topologies}}
+ <tr>
+ <td><a href="/topology.html?id={{id}}">{{name}}</a></td>
+ <td>{{id}}</td>
++ <td>{{owner}}</td>
+ <td>{{status}}</td>
+ <td>{{uptime}}</td>
+ <td>{{workersTotal}}</td>
+ <td>{{executorsTotal}}</td>
+ <td>{{tasksTotal}}</td>
++ <td>{{schedulerInfo}}</td>
+ </tr>
+ {{/topologies}}
+ </tbody>
+ </table>
</script>
<script id="supervisor-summary-template" type="text/html">
- <table class="zebra-striped" id="supervisor-summary-table"><thead><tr><th><span class="tip right" title="A unique identifier given to a Supervisor when it joins the cluster.">Id</span></th><th><span class="tip above" title="The hostname reported by the remote host. (Note that this hostname is not the result of a reverse lookup at the Nimbus node.)">Host</span></th><th><span class="tip above" title="The length of time a Supervisor has been registered to the cluster.">Uptime</span></th><th><span class="tip above" title="Slots are Workers (processes).">Slots</span></th><th><span class="tip left" title="Slots are Workers (processes).">Used slots</span></th></tr></thead>
- <tbody>
- {{#supervisors}}
- <tr>
- <td>{{id}}</td>
- <td>{{host}}</td>
- <td>{{uptime}}</td>
- <td>{{slotsTotal}}</td>
- <td>{{slotsUsed}}</td>
- </tr>
- {{/supervisors}}
- </tbody>
+ <table class="zebra-striped" id="supervisor-summary-table">
+ <thead>
+ <tr>
+ <th>
+ <span class="tip right" title="A unique identifier given to a Supervisor when it joins the cluster.">
+ Id
+ </span>
+ </th>
+ <th>
+ <span class="tip above" title="The hostname reported by the remote host. (Note that this hostname is not the result of a reverse lookup at the Nimbus node.)">
+ Host
+ </span>
+ </th>
+ <th>
+ <span class="tip above" title="The length of time a Supervisor has been registered to the cluster.">
+ Uptime
+ </span>
+ </th>
+ <th>
+ <span class="tip above" title="Slots are Workers (processes).">
+ Slots
+ </span>
+ </th>
+ <th>
+ <span class="tip left" title="Slots are Workers (processes).">
+ Used slots
+ </span>
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+ {{#supervisors}}
+ <tr>
+ <td>{{id}}</td>
+ <td>{{host}}</td>
+ <td>{{uptime}}</td>
+ <td>{{slotsTotal}}</td>
+ <td>{{slotsUsed}}</td>
+ </tr>
+ {{/supervisors}}
+ </tbody>
</table>
</script>
http://git-wip-us.apache.org/repos/asf/storm/blob/154ec7b6/storm-core/src/ui/public/templates/topology-page-template.html
----------------------------------------------------------------------
diff --cc storm-core/src/ui/public/templates/topology-page-template.html
index d442107,b977071..f8676db
--- a/storm-core/src/ui/public/templates/topology-page-template.html
+++ b/storm-core/src/ui/public/templates/topology-page-template.html
@@@ -16,17 -16,45 +16,55 @@@
-->
<script id="topology-summary-template" type="text/html">
<table id="topology-summary-table">
- <thead><tr>
- <th><span class="tip right" title="The name given to the topology by when it was submitted.">Name</span></th>
- <th><span class="tip right" title="The unique ID given to a Topology each time it is launched.">Id</span></th>
- <th><span class="tip above" title="The user that submitted the Topology, if authentication is enabled.">Owner</span></th>
- <th><span class="tip above" title="The status can be one of ACTIVE, INACTIVE, KILLED, or REBALANCING.">Status</span></th>
- <th><span class="tip above" title="The time since the Topology was submitted.">Uptime</span></th>
- <th><span class="tip above" title="The number of Workers (processes).">Num workers</span></th>
- <th><span class="tip above" title="Executors are threads in a Worker process.">Num executors</span></th>
- <th><span class="tip above" title="A Task is an instance of a Bolt or Spout. The number of Tasks is almost always equal to the number of Executors.">Num tasks</span></th>
- <th><span class="tip left" title="This shows information from the scheduler about the latest attempt to schedule the Topology on the cluster.">Scheduler Info</span></th>
- </tr></thead>
+ <thead>
+ <tr>
+ <th>
+ <span class="tip right" title="The name given to the topology by when it was submitted.">
+ Name
+ </span>
+ </th>
+ <th>
+ <span class="tip right" title="The unique ID given to a Topology each time it is launched.">
+ Id
+ </span>
+ </th>
+ <th>
++ <span class="tip above" title="The user that submitted the Topology, if authentication is enabled.">
++ Owner
++ </span>
++ </th>
++ <th>
+ <span class="tip above" title="The status can be one of ACTIVE, INACTIVE, KILLED, or REBALANCING.">
+ Status
+ </span>
+ </th>
+ <th>
+ <span class="tip above" title="The time since the Topology was submitted.">
+ Uptime
+ </span>
+ </th>
+ <th>
+ <span class="tip above" title="The number of Workers (processes).">
+ Num workers
+ </span>
+ </th>
+ <th>
+ <span class="tip above" title="Executors are threads in a Worker process.">
+ Num executors
+ </span>
+ </th>
+ <th>
+ <span class="tip above" title="A Task is an instance of a Bolt or Spout. The number of Tasks is almost always equal to the number of Executors.">
+ Num tasks
+ </span>
+ </th>
++ <th>
++ <span class="tip left" title="This shows information from the scheduler about the latest attempt to schedule the Topology on the cluster.">
++ Scheduler Info
++ </span>
++ </th>
+ </tr>
+ </thead>
<tbody>
<tr>
<td>{{name}}</td>
@@@ -36,8 -63,7 +74,8 @@@
<td>{{uptime}}</td>
<td>{{workersTotal}}</td>
<td>{{executorsTotal}}</td>
- <td>{{schedulerInfo}}</td>
<td>{{tasksTotal}}</td>
++ <td>{{schedulerInfo}}</td>
</tr>
</tbody>
</table>
http://git-wip-us.apache.org/repos/asf/storm/blob/154ec7b6/storm-core/src/ui/public/topology.html
----------------------------------------------------------------------
diff --cc storm-core/src/ui/public/topology.html
index 805b6b4,9d53144..f1e6b0c
--- a/storm-core/src/ui/public/topology.html
+++ b/storm-core/src/ui/public/topology.html
@@@ -52,10 -55,12 +54,14 @@@
<p id="toggle-switch" style="display: block;" class="js-only"></p>
<div id="json-response-error">
</div>
+<div id="anti-forgery-token" class="anti-forgery-token">
+</div>
</body>
<script>
+ $(document).ajaxStop($.unblockUI);
+ $(document).ajaxStart(function(){
+ $.blockUI({ message: '<img src="images/spinner.gif" /> <h3>Loading topology summary...</h3>'});
+ });
$(document).ready(function() {
var topologyId = $.url().param("id");
var window = $.url().param("window");
http://git-wip-us.apache.org/repos/asf/storm/blob/154ec7b6/storm-core/test/clj/backtype/storm/utils_test.clj
----------------------------------------------------------------------
[29/50] storm git commit: add new logback config to binary
distribution
Posted by bo...@apache.org.
add new logback config to binary distribution
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d07da44a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d07da44a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d07da44a
Branch: refs/heads/master
Commit: d07da44a76e1365ed8ea5cb44e7efab30f15aafa
Parents: af61cca
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Thu Sep 4 12:46:12 2014 -0500
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Thu Sep 4 12:46:12 2014 -0500
----------------------------------------------------------------------
storm-dist/binary/src/main/assembly/binary.xml | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d07da44a/storm-dist/binary/src/main/assembly/binary.xml
----------------------------------------------------------------------
diff --git a/storm-dist/binary/src/main/assembly/binary.xml b/storm-dist/binary/src/main/assembly/binary.xml
index adc1d80..e612894 100644
--- a/storm-dist/binary/src/main/assembly/binary.xml
+++ b/storm-dist/binary/src/main/assembly/binary.xml
@@ -128,6 +128,11 @@
<outputDirectory>/logback</outputDirectory>
</file>
+ <file>
+ <source>${project.basedir}/../../logback/worker.xml</source>
+ <outputDirectory>/logback</outputDirectory>
+ </file>
+
<file>
<source>${project.basedir}/LICENSE</source>
[23/50] storm git commit: Merge branch 'master' into security-upmerge
Posted by bo...@apache.org.
Merge branch 'master' into security-upmerge
Conflicts:
README.markdown
conf/defaults.yaml
pom.xml
storm-core/src/clj/backtype/storm/daemon/common.clj
storm-core/src/clj/backtype/storm/daemon/supervisor.clj
storm-core/src/clj/backtype/storm/ui/core.clj
storm-core/src/jvm/backtype/storm/Config.java
storm-core/src/jvm/backtype/storm/utils/Utils.java
storm-core/test/clj/backtype/storm/supervisor_test.clj
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/84b7deaa
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/84b7deaa
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/84b7deaa
Branch: refs/heads/master
Commit: 84b7deaa5badc07b7ea8a88fef589577c837f8ee
Parents: 7a06ef7 d9b68ab
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Thu Aug 28 13:48:29 2014 -0500
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Thu Aug 28 13:48:29 2014 -0500
----------------------------------------------------------------------
CHANGELOG.md | 15 +++
README.markdown | 7 ++
STORM-UI-REST-API.md | 1 +
bin/storm.cmd | 10 +-
conf/defaults.yaml | 1 +
.../jvm/storm/starter/PrintSampleStream.java | 4 +-
pom.xml | 8 +-
.../src/clj/backtype/storm/LocalCluster.clj | 4 +-
.../src/clj/backtype/storm/daemon/common.clj | 3 +
.../clj/backtype/storm/daemon/supervisor.clj | 32 +++--
.../src/clj/backtype/storm/daemon/worker.clj | 1 -
storm-core/src/clj/backtype/storm/event.clj | 3 +
.../backtype/storm/scheduler/EvenScheduler.clj | 2 +-
storm-core/src/clj/backtype/storm/testing.clj | 9 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 40 ++++---
storm-core/src/jvm/backtype/storm/Config.java | 42 +++++--
.../storm/multilang/JsonSerializer.java | 14 ++-
.../DefaultSerializationDelegate.java | 61 ++++++++++
.../GzipBridgeSerializationDelegate.java | 64 ++++++++++
.../GzipSerializationDelegate.java | 68 +++++++++++
.../serialization/SerializationDelegate.java | 35 ++++++
.../src/jvm/backtype/storm/task/ShellBolt.java | 9 +-
.../src/jvm/backtype/storm/utils/Utils.java | 60 ++++++----
.../test/clj/backtype/storm/scheduler_test.clj | 21 ++++
.../test/clj/backtype/storm/supervisor_test.clj | 118 ++++++++-----------
.../GzipBridgeSerializationDelegateTest.java | 82 +++++++++++++
26 files changed, 555 insertions(+), 159 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/84b7deaa/README.markdown
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/84b7deaa/conf/defaults.yaml
----------------------------------------------------------------------
diff --cc conf/defaults.yaml
index 3e00b3d,0050227..2601bee
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@@ -37,16 -35,8 +37,17 @@@ storm.zookeeper.auth.password: nul
storm.cluster.mode: "distributed" # can be distributed or local
storm.local.mode.zmq: false
storm.thrift.transport: "backtype.storm.security.auth.SimpleTransportPlugin"
+storm.principal.tolocal: "backtype.storm.security.auth.DefaultPrincipalToLocal"
+storm.group.mapping.service: "backtype.storm.security.auth.ShellBasedGroupsMapping"
storm.messaging.transport: "backtype.storm.messaging.netty.Context"
+storm.nimbus.retry.times: 5
+storm.nimbus.retry.interval.millis: 2000
+storm.nimbus.retry.intervalceiling.millis: 60000
+storm.auth.simple-white-list.users: []
+storm.auth.simple-acl.users: []
+storm.auth.simple-acl.users.commands: []
+storm.auth.simple-acl.admins: []
+ storm.meta.serialization.delegate: "backtype.storm.serialization.DefaultSerializationDelegate"
### nimbus.* configs are for the master
nimbus.host: "localhost"
http://git-wip-us.apache.org/repos/asf/storm/blob/84b7deaa/pom.xml
----------------------------------------------------------------------
diff --cc pom.xml
index ba3d1ce,3db8b41..7d31916
--- a/pom.xml
+++ b/pom.xml
@@@ -186,10 -186,9 +186,10 @@@
<commons-lang.version>2.5</commons-lang.version>
<commons-exec.version>1.1</commons-exec.version>
<clj-time.version>0.4.1</clj-time.version>
- <curator.version>2.4.0</curator.version>
+ <curator.version>2.5.0</curator.version>
<json-simple.version>1.1</json-simple.version>
- <ring.version>0.3.11</ring.version>
+ <ring.version>1.3.0</ring.version>
+ <jetty.version>7.6.13.v20130916</jetty.version>
<clojure.tools.logging.version>0.2.3</clojure.tools.logging.version>
<clojure.math.numeric-tower.version>0.0.1</clojure.math.numeric-tower.version>
<carbonite.version>1.4.0</carbonite.version>
@@@ -207,10 -206,7 +207,10 @@@
<mockito.version>1.9.5</mockito.version>
<reply.version>0.3.0</reply.version>
<conjure.version>2.1.3</conjure.version>
- <zookeeper.version>3.4.5</zookeeper.version>
+ <zookeeper.version>3.4.6</zookeeper.version>
+ <conjure.version>2.1.3</conjure.version>
+ <clojure-data-codec.version>0.1.0</clojure-data-codec.version>
+ <clojure-contrib.version>1.2.0</clojure-contrib.version>
</properties>
http://git-wip-us.apache.org/repos/asf/storm/blob/84b7deaa/storm-core/src/clj/backtype/storm/LocalCluster.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/84b7deaa/storm-core/src/clj/backtype/storm/daemon/common.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/common.clj
index cf0549c,0b45535..f091dfb
--- a/storm-core/src/clj/backtype/storm/daemon/common.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/common.clj
@@@ -21,7 -21,7 +21,8 @@@
(:import [backtype.storm.task WorkerTopologyContext])
(:import [backtype.storm Constants])
(:import [backtype.storm.metric SystemBolt])
+ (:import [backtype.storm.security.auth IAuthorizer])
+ (:import [java.io InterruptedIOException])
(:require [clojure.set :as set])
(:require [backtype.storm.daemon.acker :as acker])
(:require [backtype.storm.thrift :as thrift])
http://git-wip-us.apache.org/repos/asf/storm/blob/84b7deaa/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/84b7deaa/storm-core/src/clj/backtype/storm/daemon/worker.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/84b7deaa/storm-core/src/clj/backtype/storm/testing.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/84b7deaa/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/ui/core.clj
index 0aea7ec,9937607..d0d71c3
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@@ -850,42 -816,38 +850,50 @@@
[sys?]
(if (or (nil? sys?) (= "false" sys?)) false true))
- (defn json-response [data & [status]]
- {:status (or status 200)
- :headers {"Content-Type" "application/json"}
- :body (to-json data)})
+ (defn wrap-json-in-callback [callback response]
+ (str callback "(" response ");"))
+
+ (defnk json-response
+ [data callback :serialize-fn to-json :status 200]
+ {:status status
+ :headers (if (not-nil? callback) {"Content-Type" "application/javascript"}
+ {"Content-Type" "application/json"})
+ :body (if (not-nil? callback)
+ (wrap-json-in-callback callback (serialize-fn data))
+ (serialize-fn data))})
+(def http-creds-handler (AuthUtils/GetUiHttpCredentialsPlugin *STORM-CONF*))
+
(defroutes main-routes
- (GET "/api/v1/cluster/configuration" []
- (cluster-configuration))
- (GET "/api/v1/cluster/summary" [:as {:keys [cookies servlet-request]}]
+ (GET "/api/v1/cluster/configuration" [& m]
+ (json-response (cluster-configuration)
+ (:callback m) :serialize-fn identity))
- (GET "/api/v1/cluster/summary" [& m]
- (json-response (cluster-summary) (:callback m)))
- (GET "/api/v1/supervisor/summary" [& m]
++ (GET "/api/v1/cluster/summary" [:as {:keys [cookies servlet-request]} & m]
+ (let [user (.getUserName http-creds-handler servlet-request)]
+ (assert-authorized-user servlet-request "getClusterInfo")
- (json-response (cluster-summary user))))
- (GET "/api/v1/supervisor/summary" [:as {:keys [cookies servlet-request]}]
++ (json-response (cluster-summary user) (:callback m))))
++ (GET "/api/v1/supervisor/summary" [:as {:keys [cookies servlet-request]} & m]
+ (assert-authorized-user servlet-request "getClusterInfo")
- (json-response (supervisor-summary)))
- (GET "/api/v1/topology/summary" [:as {:keys [cookies servlet-request]}]
+ (json-response (supervisor-summary) (:callback m)))
- (GET "/api/v1/topology/summary" [& m]
++ (GET "/api/v1/topology/summary" [:as {:keys [cookies servlet-request]} & m]
+ (assert-authorized-user servlet-request "getClusterInfo")
- (json-response (all-topologies-summary)))
+ (json-response (all-topologies-summary) (:callback m)))
- (GET "/api/v1/topology/:id" [id & m]
- (let [id (url-decode id)]
- (json-response (topology-page id (:window m) (check-include-sys? (:sys m))) (:callback m))))
+ (GET "/api/v1/topology/:id" [:as {:keys [cookies servlet-request]} id & m]
+ (let [id (url-decode id)
+ user (.getUserName http-creds-handler servlet-request)]
+ (assert-authorized-user servlet-request "getTopology" (topology-config id))
- (json-response (topology-page id (:window m) (check-include-sys? (:sys m)) user))))
++ (json-response (topology-page id (:window m) (check-include-sys? (:sys m)) user) (:callback m))))
(GET "/api/v1/topology/:id/visualization" [:as {:keys [cookies servlet-request]} id & m]
- (json-response (mk-visualization-data id (:window m) (check-include-sys? (:sys m))) (:callback m)))
- (GET "/api/v1/topology/:id/component/:component" [id component & m]
+ (let [id (url-decode id)]
+ (assert-authorized-user servlet-request "getTopology" (topology-config id))
- (json-response (mk-visualization-data id (:window m) (check-include-sys? (:sys m))))))
++ (json-response (mk-visualization-data id (:window m) (check-include-sys? (:sys m))) (:callback m))))
+ (GET "/api/v1/topology/:id/component/:component" [:as {:keys [cookies servlet-request]} id component & m]
(let [id (url-decode id)
- component (url-decode component)]
- (json-response (component-page id component (:window m) (check-include-sys? (:sys m))) (:callback m))))
- (POST "/api/v1/topology/:id/activate" [id]
+ component (url-decode component)
+ user (.getUserName http-creds-handler servlet-request)]
+ (assert-authorized-user servlet-request "getTopology" (topology-config id))
- (json-response (component-page id component (:window m) (check-include-sys? (:sys m)) user))))
++ (json-response (component-page id component (:window m) (check-include-sys? (:sys m)) user) (:callback m))))
+ (POST "/api/v1/topology/:id/activate" [:as {:keys [cookies servlet-request]} id]
(with-nimbus nimbus
(let [id (url-decode id)
tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
http://git-wip-us.apache.org/repos/asf/storm/blob/84b7deaa/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --cc storm-core/src/jvm/backtype/storm/Config.java
index c15c45d,8407010..a38b51e
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@@ -102,14 -95,15 +102,21 @@@ public class Config extends HashMap<Str
*/
public static final String STORM_NETTY_FLUSH_CHECK_INTERVAL_MS = "storm.messaging.netty.flush.check.interval.ms";
public static final Object STORM_NETTY_FLUSH_CHECK_INTERVAL_MS_SCHEMA = ConfigValidation.IntegerValidator;
-
+
+ /**
+ * Netty based messaging: Is authentication required for Netty messaging from client worker process to server worker process.
+ */
+ public static final String STORM_MESSAGING_NETTY_AUTHENTICATION = "storm.messaging.netty.authentication";
+ public static final Object STORM_MESSAGING_NETTY_AUTHENTICATION_SCHEMA = Boolean.class;
+
/**
+ * The delegate for serializing metadata, should be used for serialized objects stored in zookeeper and on disk.
+ * This is NOT used for compressing serialized tuples sent between topologies.
+ */
+ public static final String STORM_META_SERIALIZATION_DELEGATE = "storm.meta.serialization.delegate";
+ public static final Object STORM_META_SERIALIZATION_DELEGATE_SCHEMA = String.class;
+
+ /**
* A list of hosts of ZooKeeper servers used to manage the cluster.
*/
public static final String STORM_ZOOKEEPER_SERVERS = "storm.zookeeper.servers";
@@@ -746,22 -473,12 +753,25 @@@
public static final Object SUPERVISOR_MONITOR_FREQUENCY_SECS_SCHEMA = ConfigValidation.IntegerValidator;
/**
+ * Should the supervior try to run the worker as the lauching user or not. Defaults to false.
+ */
+ public static final String SUPERVISOR_RUN_WORKER_AS_USER = "supervisor.run.worker.as.user";
+ public static final Object SUPERVISOR_RUN_WORKER_AS_USER_SCHEMA = Boolean.class;
+
+ /**
+ * Full path to the worker-laucher executable that will be used to lauch workers when
+ * SUPERVISOR_RUN_WORKER_AS_USER is set to true.
+ */
+ public static final String SUPERVISOR_WORKER_LAUNCHER = "supervisor.worker.launcher";
+ public static final Object SUPERVISOR_WORKER_LAUNCHER_SCHEMA = String.class;
+
+ /**
- * The jvm opts provided to workers launched by this supervisor. All "%ID%" substrings are replaced
- * with an identifier for this worker. Also, "%WORKER-ID%", "%STORM-ID%" and "%WORKER-PORT%" are
- * replaced with appropriate runtime values for this worker.
+ * The jvm opts provided to workers launched by this supervisor. All "%ID%", "%WORKER-ID%", "%TOPOLOGY-ID%"
+ * and "%WORKER-PORT%" substrings are replaced with:
+ * %ID% -> port (for backward compatibility),
+ * %WORKER-ID% -> worker-id,
+ * %TOPOLOGY-ID% -> topology-id,
+ * %WORKER-PORT% -> port.
*/
public static final String WORKER_CHILDOPTS = "worker.childopts";
public static final Object WORKER_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
http://git-wip-us.apache.org/repos/asf/storm/blob/84b7deaa/storm-core/src/jvm/backtype/storm/utils/Utils.java
----------------------------------------------------------------------
diff --cc storm-core/src/jvm/backtype/storm/utils/Utils.java
index fff91e6,a657d2b..6d31d28
--- a/storm-core/src/jvm/backtype/storm/utils/Utils.java
+++ b/storm-core/src/jvm/backtype/storm/utils/Utils.java
@@@ -478,66 -447,24 +469,87 @@@ public class Utils
return false;
}
+ /**
+ * Is the cluster configured to interact with ZooKeeper in a secure way?
+ * This only works when called from within Nimbus or a Supervisor process.
+ * @param conf the storm configuration, not the topology configuration
+ * @return true if it is configured else false.
+ */
+ public static boolean isZkAuthenticationConfiguredStormServer(Map conf) {
+ return null != System.getProperty("java.security.auth.login.config")
+ || (conf != null
+ && conf.get(Config.STORM_ZOOKEEPER_AUTH_SCHEME) != null
+ && ! ((String)conf.get(Config.STORM_ZOOKEEPER_AUTH_SCHEME)).isEmpty());
+ }
+
+ /**
+ * Is the topology configured to have ZooKeeper authentication.
+ * @param conf the topology configuration
+ * @return true if ZK is configured else false
+ */
+ public static boolean isZkAuthenticationConfiguredTopology(Map conf) {
+ return (conf != null
+ && conf.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_SCHEME) != null
+ && ! ((String)conf.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_SCHEME)).isEmpty());
+ }
+
+ public static List<ACL> getWorkerACL(Map conf) {
+ //This is a work around to an issue with ZK where a sasl super user is not super unless there is an open SASL ACL so we are trying to give the correct perms
+ if (!isZkAuthenticationConfiguredTopology(conf)) {
+ return null;
+ }
+ String stormZKUser = (String)conf.get(Config.STORM_ZOOKEEPER_SUPERACL);
+ if (stormZKUser == null) {
+ throw new IllegalArgumentException("Authentication is enabled but "+Config.STORM_ZOOKEEPER_SUPERACL+" is not set");
+ }
+ String[] split = stormZKUser.split(":",2);
+ if (split.length != 2) {
+ throw new IllegalArgumentException(Config.STORM_ZOOKEEPER_SUPERACL+" does not appear to be in the form scheme:acl, i.e. sasl:storm-user");
+ }
+ ArrayList<ACL> ret = new ArrayList<ACL>(ZooDefs.Ids.CREATOR_ALL_ACL);
+ ret.add(new ACL(ZooDefs.Perms.ALL, new Id(split[0], split[1])));
+ return ret;
+ }
+
+ public static String threadDump() {
+ final StringBuilder dump = new StringBuilder();
+ final java.lang.management.ThreadMXBean threadMXBean = java.lang.management.ManagementFactory.getThreadMXBean();
+ final java.lang.management.ThreadInfo[] threadInfos = threadMXBean.getThreadInfo(threadMXBean.getAllThreadIds(), 100);
+ for (java.lang.management.ThreadInfo threadInfo : threadInfos) {
+ dump.append('"');
+ dump.append(threadInfo.getThreadName());
+ dump.append("\" ");
+ final Thread.State state = threadInfo.getThreadState();
+ dump.append("\n java.lang.Thread.State: ");
+ dump.append(state);
+ final StackTraceElement[] stackTraceElements = threadInfo.getStackTrace();
+ for (final StackTraceElement stackTraceElement : stackTraceElements) {
+ dump.append("\n at ");
+ dump.append(stackTraceElement);
+ }
+ dump.append("\n\n");
+ }
+ return dump.toString();
+ }
++
+ // Assumes caller is synchronizing
+ private static SerializationDelegate getSerializationDelegate(Map stormConf) {
+ String delegateClassName = (String)stormConf.get(Config.STORM_META_SERIALIZATION_DELEGATE);
+ SerializationDelegate delegate;
+ try {
+ Class delegateClass = Class.forName(delegateClassName);
+ delegate = (SerializationDelegate) delegateClass.newInstance();
+ } catch (ClassNotFoundException e) {
+ LOG.error("Failed to construct serialization delegate, falling back to default", e);
+ delegate = new DefaultSerializationDelegate();
+ } catch (InstantiationException e) {
+ LOG.error("Failed to construct serialization delegate, falling back to default", e);
+ delegate = new DefaultSerializationDelegate();
+ } catch (IllegalAccessException e) {
+ LOG.error("Failed to construct serialization delegate, falling back to default", e);
+ delegate = new DefaultSerializationDelegate();
+ }
+ delegate.prepare(stormConf);
+ return delegate;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/84b7deaa/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --cc storm-core/test/clj/backtype/storm/supervisor_test.clj
index 6f02688,5abfa89..c94688f
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@@ -463,94 -354,35 +463,90 @@@
;; TODO just do reassign, and check that cleans up worker states after killing but doesn't get rid of downloaded code
)
+(deftest test-supervisor-data-acls
+ (testing "supervisor-data uses correct ACLs"
+ (let [scheme "digest"
+ digest "storm:thisisapoorpassword"
+ auth-conf {STORM-ZOOKEEPER-AUTH-SCHEME scheme
+ STORM-ZOOKEEPER-AUTH-PAYLOAD digest}
+ expected-acls supervisor/SUPERVISOR-ZK-ACLS
+ fake-isupervisor (reify ISupervisor
+ (getSupervisorId [this] nil)
+ (getAssignmentId [this] nil))]
+ (stubbing [uptime-computer nil
+ cluster/mk-storm-cluster-state nil
+ supervisor-state nil
+ local-hostname nil
+ mk-timer nil]
+ (supervisor/supervisor-data auth-conf nil fake-isupervisor)
+ (verify-call-times-for cluster/mk-storm-cluster-state 1)
+ (verify-first-call-args-for-indices cluster/mk-storm-cluster-state [2]
+ expected-acls)))))
+
+(deftest test-write-log-metadata
+ (testing "supervisor writes correct data to logs metadata file"
+ (let [exp-owner "alice"
+ exp-worker-id "42"
+ exp-storm-id "0123456789"
+ exp-port 4242
+ exp-logs-users ["bob" "charlie" "daryl"]
+ storm-conf {TOPOLOGY-SUBMITTER-USER "alice"
+ TOPOLOGY-USERS ["charlie" "bob"]
+ LOGS-USERS ["daryl"]}
+ exp-data {TOPOLOGY-SUBMITTER-USER exp-owner
+ "worker-id" exp-worker-id
+ LOGS-USERS exp-logs-users}
+ conf {}]
+ (mocking [supervisor/write-log-metadata-to-yaml-file!]
+ (supervisor/write-log-metadata! storm-conf exp-owner exp-worker-id
+ exp-storm-id exp-port conf)
+ (verify-called-once-with-args supervisor/write-log-metadata-to-yaml-file!
+ exp-storm-id exp-port exp-data conf)))))
+
+(deftest test-worker-launcher-requires-user
+ (testing "worker-launcher throws on blank user"
+ (mocking [launch-process]
+ (is (thrown-cause-with-msg? java.lang.IllegalArgumentException
+ #"(?i).*user cannot be blank.*"
+ (supervisor/worker-launcher {} nil ""))))))
+
+(defn found? [sub-str input-str]
+ (if (string? input-str)
+ (contrib-str/substring? sub-str (str input-str))
+ (some? #(contrib-str/substring? sub-str %) input-str)))
+
+(defn not-found? [sub-str input-str]
+ (complement (found? sub-str input-str)))
+
- (deftest test-substitute-childopts-happy-path
+ (deftest test-substitute-childopts-happy-path-string
(testing "worker-launcher replaces ids in childopts"
- (let [ worker-id "w-01"
- storm-id "s-01"
- port 9999
- childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log"
- ]
- (def childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port))
- (is (not-found? "%WORKER-ID%" childopts-with-ids))
- (is (found? "w-01" childopts-with-ids))
- (is (not-found? "%STORM-ID%" childopts-with-ids))
- (is (found? "s-01" childopts-with-ids))
- (is (not-found? "%WORKER-PORT%" childopts-with-ids))
- (is (found? "-9999." childopts-with-ids))
- (is (not-found? "%ID%" childopts-with-ids))
- (is (found? "worker-9999" childopts-with-ids) (str childopts-with-ids))
- )))
+ (let [worker-id "w-01"
+ topology-id "s-01"
+ port 9999
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log -Xms256m"
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m")
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port)]
+ (is (= expected-childopts childopts-with-ids)))))
- (deftest test-substitute-childopts-storm-id-alone
+ (deftest test-substitute-childopts-happy-path-list
(testing "worker-launcher replaces ids in childopts"
- (let [ worker-id "w-01"
- storm-id "s-01"
- port 9999
- childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%STORM-ID%.log"]
- (def childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port))
- (is (not-found? "%WORKER-ID%" childopts-with-ids))
- (is (not-found? "w-01" childopts-with-ids))
- (is (not-found? "%STORM-ID%" childopts-with-ids))
- (is (found? "s-01" childopts-with-ids))
- (is (not-found? "%WORKER-PORT%" childopts-with-ids))
- (is (not-found? "-9999." childopts-with-ids))
- (is (not-found? "%ID%" childopts-with-ids))
- (is (not-found? "worker-9999" childopts-with-ids) (str childopts-with-ids)) )))
+ (let [worker-id "w-01"
+ topology-id "s-01"
+ port 9999
+ childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log" "-Xms256m")
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m")
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port)]
+ (is (= expected-childopts childopts-with-ids)))))
+
+ (deftest test-substitute-childopts-topology-id-alone
+ (testing "worker-launcher replaces ids in childopts"
+ (let [worker-id "w-01"
+ topology-id "s-01"
+ port 9999
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%TOPOLOGY-ID%.log"
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-s-01.log")
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port)]
+ (is (= expected-childopts childopts-with-ids)))))
(deftest test-substitute-childopts-no-keys
(testing "worker-launcher has no ids to replace in childopts"
[31/50] storm git commit: avoid adding multiple jetty handlers
Posted by bo...@apache.org.
avoid adding multiple jetty handlers
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f8cb7b59
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f8cb7b59
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f8cb7b59
Branch: refs/heads/master
Commit: f8cb7b594aaad68e9298914610f200e970f4eb11
Parents: b19fdbb
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Thu Sep 4 15:29:31 2014 -0500
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Thu Sep 4 15:29:31 2014 -0500
----------------------------------------------------------------------
.../src/clj/backtype/storm/daemon/drpc.clj | 25 +++++++++---------
.../src/clj/backtype/storm/daemon/logviewer.clj | 9 +++----
storm-core/src/clj/backtype/storm/ui/core.clj | 12 ++++-----
.../src/clj/backtype/storm/ui/helpers.clj | 27 +++++++++++++++++++-
4 files changed, 46 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f8cb7b59/storm-core/src/clj/backtype/storm/daemon/drpc.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/drpc.clj b/storm-core/src/clj/backtype/storm/daemon/drpc.clj
index 68128c3..95fa1be 100644
--- a/storm-core/src/clj/backtype/storm/daemon/drpc.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/drpc.clj
@@ -30,7 +30,6 @@
(:use [backtype.storm.ui helpers])
(:use compojure.core)
(:use ring.middleware.reload)
- (:use [ring.adapter.jetty :only [run-jetty]])
(:require [compojure.handler :as handler])
(:gen-class))
@@ -190,9 +189,9 @@
(if http-creds-handler
(.populateContext http-creds-handler (ReqContext/context)
servlet-request))
- (.execute handler func "")))
- (wrap-reload '[backtype.storm.daemon.drpc])
- handle-request))
+ (.execute handler func "")))
+ (wrap-reload '[backtype.storm.daemon.drpc])
+ handle-request))
(defn launch-server!
([]
@@ -230,15 +229,15 @@
https-ks-password (conf DRPC-HTTPS-KEYSTORE-PASSWORD)
https-ks-type (conf DRPC-HTTPS-KEYSTORE-TYPE)]
- (run-jetty app
- {:port drpc-http-port :join? false
- :configurator (fn [server]
- (config-ssl server
- https-port
- https-ks-path
- https-ks-password
- https-ks-type)
- (config-filter server app filters-confs))})))
+ (storm-run-jetty
+ {:port drpc-http-port
+ :configurator (fn [server]
+ (config-ssl server
+ https-port
+ https-ks-path
+ https-ks-password
+ https-ks-type)
+ (config-filter server app filters-confs))})))
(when handler-server
(.serve handler-server)))))
http://git-wip-us.apache.org/repos/asf/storm/blob/f8cb7b59/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/logviewer.clj b/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
index c3868eb..0745d5d 100644
--- a/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
@@ -20,7 +20,6 @@
(:use [hiccup core page-helpers])
(:use [backtype.storm config util log timer])
(:use [backtype.storm.ui helpers])
- (:use [ring.adapter.jetty :only [run-jetty]])
(:import [org.slf4j LoggerFactory])
(:import [ch.qos.logback.classic Logger])
(:import [ch.qos.logback.core FileAppender])
@@ -351,11 +350,9 @@ Note that if anything goes wrong, this will throw an Error and exit."
[{:filter-class "org.eclipse.jetty.servlets.GzipFilter"
:filter-name "Gzipper"
:filter-params {}}])]
- (run-jetty middle
- {:port (int (conf LOGVIEWER-PORT))
- :join? false
- :configurator (fn [server]
- (config-filter server middle filters-confs))}))
+ (storm-run-jetty {:port (int (conf LOGVIEWER-PORT))
+ :configurator (fn [server]
+ (config-filter server middle filters-confs))}))
(catch Exception ex
(log-error ex))))
http://git-wip-us.apache.org/repos/asf/storm/blob/f8cb7b59/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/ui/core.clj b/storm-core/src/clj/backtype/storm/ui/core.clj
index d0d71c3..043f123 100644
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@ -22,7 +22,6 @@
(:use [backtype.storm.ui helpers])
(:use [backtype.storm.daemon [common :only [ACKER-COMPONENT-ID ACKER-INIT-STREAM-ID ACKER-ACK-STREAM-ID
ACKER-FAIL-STREAM-ID system-id? mk-authorization-handler]]])
- (:use [ring.adapter.jetty :only [run-jetty]])
(:use [clojure.string :only [blank? lower-case trim]])
(:import [backtype.storm.utils Utils])
(:import [backtype.storm.generated ExecutorSpecificStats
@@ -967,12 +966,11 @@
header-buffer-size (int (.get conf UI-HEADER-BUFFER-BYTES))
filters-confs [{:filter-class (conf UI-FILTER)
:filter-params (conf UI-FILTER-PARAMS)}]]
- (run-jetty app {:port (conf UI-PORT)
- :join? false
- :configurator (fn [server]
- (doseq [connector (.getConnectors server)]
- (.setRequestHeaderSize connector header-buffer-size))
- (config-filter server app filters-confs))}))
+ (storm-run-jetty {:port (conf UI-PORT)
+ :configurator (fn [server]
+ (doseq [connector (.getConnectors server)]
+ (.setRequestHeaderSize connector header-buffer-size))
+ (config-filter server app filters-confs))}))
(catch Exception ex
(log-error ex))))
http://git-wip-us.apache.org/repos/asf/storm/blob/f8cb7b59/storm-core/src/clj/backtype/storm/ui/helpers.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/ui/helpers.clj b/storm-core/src/clj/backtype/storm/ui/helpers.clj
index 8033c50..413a68d 100644
--- a/storm-core/src/clj/backtype/storm/ui/helpers.clj
+++ b/storm-core/src/clj/backtype/storm/ui/helpers.clj
@@ -23,7 +23,9 @@
(:use [backtype.storm.util :only [clojurify-structure uuid defnk url-encode]])
(:use [clj-time coerce format])
(:import [backtype.storm.generated ExecutorInfo ExecutorSummary])
- (:import [org.eclipse.jetty.server.ssl SslSocketConnector]
+ (:import [org.eclipse.jetty.server Server]
+ [org.eclipse.jetty.server.nio SelectChannelConnector]
+ [org.eclipse.jetty.server.ssl SslSocketConnector]
[org.eclipse.jetty.servlet ServletHolder FilterMapping])
(:require [ring.util servlet])
(:require [compojure.route :as route]
@@ -184,3 +186,26 @@ $(\"table#%s\").each(function(i) { $(this).tablesorter({ sortList: %s, headers:
{:headers {}
:status 400
:body (.getMessage ex)})
+
+;; Modified from ring.adapter.jetty 1.3.0
+(defn- jetty-create-server
+ "Construct a Jetty Server instance."
+ [options]
+ (let [connector (doto (SelectChannelConnector.)
+ (.setPort (options :port 80))
+ (.setHost (options :host))
+ (.setMaxIdleTime (options :max-idle-time 200000)))
+ server (doto (Server.)
+ (.addConnector connector)
+ (.setSendDateHeader true))]
+ server))
+
+(defn storm-run-jetty
+ "Modified version of run-jetty
+ Assumes configurator sets handler."
+ [config]
+ {:pre [(:configurator config)]}
+ (let [#^Server s (jetty-create-server (dissoc config :configurator))
+ configurator (:configurator config)]
+ (configurator s)
+ (.start s)))
[34/50] storm git commit: STORM-489. remove uiActionsEnabled from js
and templates.
Posted by bo...@apache.org.
STORM-489. remove uiActionsEnabled from js and templates.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ce0b18a3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ce0b18a3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ce0b18a3
Branch: refs/heads/master
Commit: ce0b18a3915f1020042b90ca3e981e586175732b
Parents: 5f3a3a9
Author: Sriharsha Chintalapani <ma...@harsha.io>
Authored: Sat Sep 6 20:11:24 2014 -0700
Committer: Sriharsha Chintalapani <ma...@harsha.io>
Committed: Sat Sep 6 20:11:24 2014 -0700
----------------------------------------------------------------------
storm-core/src/ui/public/js/script.js | 3 +--
storm-core/src/ui/public/topology.html | 2 +-
2 files changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ce0b18a3/storm-core/src/ui/public/js/script.js
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/js/script.js b/storm-core/src/ui/public/js/script.js
index 09ecd57..9d75322 100644
--- a/storm-core/src/ui/public/js/script.js
+++ b/storm-core/src/ui/public/js/script.js
@@ -141,9 +141,8 @@ function renderToggleSys(div) {
}
}
-function topologyActionJson(id,name,status,msgTimeout, uiActionsEnabled) {
+function topologyActionJson(id,name,status,msgTimeout) {
var jsonData = {};
- jsonData["uiActionsEnabled"] = uiActionsEnabled;
jsonData["id"] = id;
jsonData["name"] = name;
jsonData["msgTimeout"] = msgTimeout;
http://git-wip-us.apache.org/repos/asf/storm/blob/ce0b18a3/storm-core/src/ui/public/topology.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/topology.html b/storm-core/src/ui/public/topology.html
index 805b6b4..cadec38 100644
--- a/storm-core/src/ui/public/topology.html
+++ b/storm-core/src/ui/public/topology.html
@@ -91,7 +91,7 @@ $(document).ready(function() {
var topologyActions = $("#topology-actions");
var topologyVisualization = $("#topology-visualization")
var formattedConfig = formatConfigData(response["configuration"]);
- var buttonJsonData = topologyActionJson(response["id"],response["name"],response["status"],response["msgTimeout"], response["uiActionsEnabled"]);
+ var buttonJsonData = topologyActionJson(response["id"],response["name"],response["status"],response["msgTimeout"]);
$.get("/templates/topology-page-template.html", function(template) {
topologySummary.append(Mustache.render($(template).filter("#topology-summary-template").html(),response));
topologyActions.append(Mustache.render($(template).filter("#topology-actions-template").html(),buttonJsonData));
[28/50] storm git commit: use separate logback config for workers
Posted by bo...@apache.org.
use separate logback config for workers
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/af61cca7
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/af61cca7
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/af61cca7
Branch: refs/heads/master
Commit: af61cca75c6215d5a0db2d6fb338deca03e55095
Parents: b19fdbb
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Wed Sep 3 10:05:25 2014 -0500
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Wed Sep 3 10:05:25 2014 -0500
----------------------------------------------------------------------
logback/worker.xml | 41 ++++++++++++++++++++
.../clj/backtype/storm/daemon/supervisor.clj | 2 +-
.../test/clj/backtype/storm/supervisor_test.clj | 4 +-
3 files changed, 44 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/af61cca7/logback/worker.xml
----------------------------------------------------------------------
diff --git a/logback/worker.xml b/logback/worker.xml
new file mode 100644
index 0000000..3d9009e
--- /dev/null
+++ b/logback/worker.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration scan="true" scanPeriod="60 seconds">
+ <appender name="A1" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${storm.log.dir}/${logfile.name}</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+ <fileNamePattern>${storm.log.dir}/${logfile.name}.%i</fileNamePattern>
+ <minIndex>1</minIndex>
+ <maxIndex>9</maxIndex>
+ </rollingPolicy>
+
+ <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+ <maxFileSize>100MB</maxFileSize>
+ </triggeringPolicy>
+
+ <encoder>
+ <pattern>%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n</pattern>
+ </encoder>
+ </appender>
+
+ <root level="INFO">
+ <appender-ref ref="A1"/>
+ </root>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/storm/blob/af61cca7/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index 7b242e5..1fe5b2e 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -638,7 +638,7 @@
(str "-Dlogfile.name=" logfilename)
(str "-Dstorm.home=" storm-home)
(str "-Dstorm.log.dir=" storm-log-dir)
- (str "-Dlogback.configurationFile=" storm-home "/logback/cluster.xml")
+ (str "-Dlogback.configurationFile=" storm-home "/logback/worker.xml")
(str "-Dstorm.id=" storm-id)
(str "-Dworker.id=" worker-id)
(str "-Dworker.port=" port)
http://git-wip-us.apache.org/repos/asf/storm/blob/af61cca7/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/supervisor_test.clj b/storm-core/test/clj/backtype/storm/supervisor_test.clj
index c94688f..4261b4d 100644
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@ -262,7 +262,7 @@
(str "-Dlogfile.name=" mock-storm-id "-worker-" mock-port ".log")
"-Dstorm.home="
"-Dstorm.log.dir=/logs"
- "-Dlogback.configurationFile=/logback/cluster.xml"
+ "-Dlogback.configurationFile=/logback/worker.xml"
(str "-Dstorm.id=" mock-storm-id)
(str "-Dworker.id=" mock-worker-id)
(str "-Dworker.port=" mock-port)
@@ -380,7 +380,7 @@
" '-Dlogfile.name=" mock-storm-id "-worker-" mock-port ".log'"
" '-Dstorm.home='"
" '-Dstorm.log.dir=/logs'"
- " '-Dlogback.configurationFile=/logback/cluster.xml'"
+ " '-Dlogback.configurationFile=/logback/worker.xml'"
" '-Dstorm.id=" mock-storm-id "'"
" '-Dworker.id=" mock-worker-id "'"
" '-Dworker.port=" mock-port "'"
[10/50] storm git commit: Merge branch 'master' into STORM-366
Posted by bo...@apache.org.
Merge branch 'master' into STORM-366
Conflicts:
storm-core/src/clj/backtype/storm/ui/core.clj
storm-core/src/ui/public/component.html
storm-core/src/ui/public/templates/component-page-template.html
storm-core/src/ui/public/templates/topology-page-template.html
storm-core/src/ui/public/topology.html
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/fa40fbcc
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/fa40fbcc
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/fa40fbcc
Branch: refs/heads/master
Commit: fa40fbcc8953f95e6a5f0f147f84170fae026c62
Parents: 9d0b618 d65c913
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Wed Jul 30 19:03:25 2014 -0500
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Wed Jul 30 19:03:25 2014 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
storm-core/src/ui/public/templates/component-page-template.html | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/fa40fbcc/storm-core/src/ui/public/templates/component-page-template.html
----------------------------------------------------------------------
[33/50] storm git commit: Merge branch 'mt-scheduler-used-node-check'
of https://github.com/kishorvpatil/incubator-storm into STORM-479-merge
Posted by bo...@apache.org.
Merge branch 'mt-scheduler-used-node-check' of https://github.com/kishorvpatil/incubator-storm into STORM-479-merge
STORM-479 (Security) Multi-tenant scheduler needs extra check for used slots
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/5f3a3a97
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/5f3a3a97
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/5f3a3a97
Branch: refs/heads/master
Commit: 5f3a3a97959770d7374e78f9e415e91b5091a560
Parents: a7b5d68 6e3c77a
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Fri Sep 5 11:20:07 2014 -0500
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Fri Sep 5 11:20:07 2014 -0500
----------------------------------------------------------------------
.../storm/scheduler/multitenant/Node.java | 20 ++-
.../scheduler/multitenant_scheduler_test.clj | 163 +++++++++++++------
2 files changed, 126 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
[36/50] storm git commit: Merge branch 'STORM-427' of
http://github.com/revans2/incubator-storm into STORM-427
Posted by bo...@apache.org.
Merge branch 'STORM-427' of http://github.com/revans2/incubator-storm into STORM-427
STORM-427: AutoTGT and HBase can expose JVM kerberos bug.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/5a277f83
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/5a277f83
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/5a277f83
Branch: refs/heads/master
Commit: 5a277f8378460e4d9622dc8cfb09ddf088f5a3f9
Parents: 5f3a3a9 ee5bb17
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Sep 23 13:30:37 2014 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Tue Sep 23 13:30:37 2014 -0500
----------------------------------------------------------------------
.../storm/security/auth/kerberos/AutoTGT.java | 22 ++++++++++++++++----
1 file changed, 18 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/5a277f83/storm-core/src/jvm/backtype/storm/security/auth/kerberos/AutoTGT.java
----------------------------------------------------------------------
[39/50] storm git commit: Merge branch 'STORM-486' of
http://github.com/d2r/incubator-storm into STORM-486
Posted by bo...@apache.org.
Merge branch 'STORM-486' of http://github.com/d2r/incubator-storm into STORM-486
Conflicts:
storm-core/src/clj/backtype/storm/ui/core.clj
STORM-486: avoid adding multiple jetty handlers
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/aa6e7ed7
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/aa6e7ed7
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/aa6e7ed7
Branch: refs/heads/master
Commit: aa6e7ed768797a3cb1674de61bb5a7c23f221bc1
Parents: e08bf4b f8cb7b5
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Sep 23 16:34:01 2014 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Tue Sep 23 16:34:01 2014 -0500
----------------------------------------------------------------------
.../src/clj/backtype/storm/daemon/drpc.clj | 25 +++++++++---------
.../src/clj/backtype/storm/daemon/logviewer.clj | 9 +++----
storm-core/src/clj/backtype/storm/ui/core.clj | 12 ++++-----
.../src/clj/backtype/storm/ui/helpers.clj | 27 +++++++++++++++++++-
4 files changed, 46 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/aa6e7ed7/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/ui/core.clj
index 2181b48,043f123..600c96a
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@@ -22,8 -22,6 +22,7 @@@
(:use [backtype.storm.ui helpers])
(:use [backtype.storm.daemon [common :only [ACKER-COMPONENT-ID ACKER-INIT-STREAM-ID ACKER-ACK-STREAM-ID
ACKER-FAIL-STREAM-ID system-id? mk-authorization-handler]]])
- (:use [ring.adapter.jetty :only [run-jetty]])
+ (:use [ring.middleware.anti-forgery])
(:use [clojure.string :only [blank? lower-case trim]])
(:import [backtype.storm.utils Utils])
(:import [backtype.storm.generated ExecutorSpecificStats
[19/50] storm git commit: Fixed tests after upmerge
Posted by bo...@apache.org.
Fixed tests after upmerge
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7a06ef72
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7a06ef72
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7a06ef72
Branch: refs/heads/master
Commit: 7a06ef722a7bbf5abae42a6c4b1f6762b2293773
Parents: d20c101
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Mon Aug 18 14:40:45 2014 -0500
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Mon Aug 18 14:40:45 2014 -0500
----------------------------------------------------------------------
storm-core/test/clj/backtype/storm/supervisor_test.clj | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/7a06ef72/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/supervisor_test.clj b/storm-core/test/clj/backtype/storm/supervisor_test.clj
index 3eb5de6..6f02688 100644
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@ -379,6 +379,7 @@
" '-Djava.library.path='"
" '-Dlogfile.name=" mock-storm-id "-worker-" mock-port ".log'"
" '-Dstorm.home='"
+ " '-Dstorm.log.dir=/logs'"
" '-Dlogback.configurationFile=/logback/cluster.xml'"
" '-Dstorm.id=" mock-storm-id "'"
" '-Dworker.id=" mock-worker-id "'"
[05/50] storm git commit: Merge remote-tracking branch
'upstream/security' into security
Posted by bo...@apache.org.
Merge remote-tracking branch 'upstream/security' into security
Conflicts:
conf/defaults.yaml
storm-core/src/jvm/backtype/storm/Config.java
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/02e79499
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/02e79499
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/02e79499
Branch: refs/heads/master
Commit: 02e79499fa381b56f8a5333e00cf0cc0bd7aa252
Parents: 19ad135 559c883
Author: Raghavendra Nandagopal <sp...@gmail.com>
Authored: Tue Jul 29 16:37:50 2014 -0700
Committer: Raghavendra Nandagopal <sp...@gmail.com>
Committed: Tue Jul 29 16:37:50 2014 -0700
----------------------------------------------------------------------
BYLAWS.md | 96 ++++
CHANGELOG.md | 31 +
LICENSE | 30 +-
README.markdown | 14 +
STORM-UI-REST-API.md | 567 +++++++++++++++++++
bin/storm | 77 ++-
conf/defaults.yaml | 11 +-
conf/storm_env.ini | 2 +-
dev-tools/github/__init__.py | 109 ++++
dev-tools/jira-github-join.py | 80 +++
dev-tools/jira/__init__.py | 232 ++++++++
examples/storm-starter/README.markdown | 30 +-
.../storm-starter/multilang/resources/storm.py | 2 +-
.../src/jvm/storm/starter/RollingTopWords.java | 62 +-
.../src/jvm/storm/starter/util/StormRunner.java | 9 +
.../storm-kafka/src/jvm/storm/kafka/Broker.java | 9 +-
.../src/jvm/storm/kafka/KafkaConfig.java | 2 +-
.../src/jvm/storm/kafka/Partition.java | 9 +-
pom.xml | 6 +-
.../src/clj/backtype/storm/LocalCluster.clj | 7 +-
storm-core/src/clj/backtype/storm/cluster.clj | 31 +-
.../src/clj/backtype/storm/command/monitor.clj | 37 ++
.../src/clj/backtype/storm/daemon/common.clj | 2 +-
.../src/clj/backtype/storm/daemon/drpc.clj | 6 +-
.../src/clj/backtype/storm/daemon/executor.clj | 4 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 7 +-
.../clj/backtype/storm/daemon/supervisor.clj | 104 +++-
.../src/clj/backtype/storm/daemon/worker.clj | 72 +--
storm-core/src/clj/backtype/storm/disruptor.clj | 2 +-
storm-core/src/clj/backtype/storm/event.clj | 2 +-
storm-core/src/clj/backtype/storm/testing.clj | 49 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 2 +-
.../src/clj/backtype/storm/ui/helpers.clj | 5 -
storm-core/src/clj/backtype/storm/util.clj | 52 +-
storm-core/src/clj/backtype/storm/zookeeper.clj | 25 +
.../src/dev/resources/tester_bolt_metrics.py | 35 ++
.../src/dev/resources/tester_spout_metrics.py | 51 ++
storm-core/src/jvm/backtype/storm/Config.java | 306 +++++-----
.../jvm/backtype/storm/ConfigValidation.java | 70 +++
.../backtype/storm/messaging/netty/Client.java | 13 +-
.../metric/api/rpc/AssignableShellMetric.java | 30 +
.../metric/api/rpc/CombinedShellMetric.java | 31 +
.../storm/metric/api/rpc/CountShellMetric.java | 38 ++
.../storm/metric/api/rpc/IShellMetric.java | 31 +
.../metric/api/rpc/ReducedShellMetric.java | 32 ++
.../storm/multilang/JsonSerializer.java | 15 +
.../jvm/backtype/storm/multilang/ShellMsg.java | 46 ++
.../backtype/storm/security/auth/AuthUtils.java | 49 +-
.../auth/IGroupMappingServiceProvider.java | 42 ++
.../security/auth/ShellBasedGroupsMapping.java | 94 +++
.../auth/authorizer/SimpleACLAuthorizer.java | 29 +-
.../src/jvm/backtype/storm/spout/ISpout.java | 2 +-
.../jvm/backtype/storm/spout/ShellSpout.java | 72 ++-
.../src/jvm/backtype/storm/task/ShellBolt.java | 75 ++-
.../backtype/storm/task/TopologyContext.java | 28 +
.../storm/testing/PythonShellMetricsBolt.java | 32 ++
.../storm/testing/PythonShellMetricsSpout.java | 35 ++
.../src/jvm/backtype/storm/utils/Monitor.java | 249 ++++++++
.../jvm/backtype/storm/utils/ShellProcess.java | 46 +-
.../jvm/backtype/storm/utils/ShellUtils.java | 498 ++++++++++++++++
.../src/jvm/backtype/storm/utils/Utils.java | 23 +-
storm-core/src/multilang/py/storm.py | 30 +-
storm-core/src/multilang/rb/storm.rb | 24 +-
storm-core/src/ui/public/component.html | 3 +-
.../src/ui/public/js/jquery.tablesorter.min.js | 9 +-
storm-core/src/ui/public/js/moment.min.js | 6 +
storm-core/src/ui/public/js/script.js | 9 +
.../test/clj/backtype/storm/cluster_test.clj | 3 +-
.../test/clj/backtype/storm/config_test.clj | 41 +-
.../test/clj/backtype/storm/metrics_test.clj | 206 ++++---
.../backtype/storm/security/auth/auth_test.clj | 11 +-
.../test/clj/backtype/storm/supervisor_test.clj | 135 ++++-
72 files changed, 3780 insertions(+), 454 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/02e79499/conf/defaults.yaml
----------------------------------------------------------------------
diff --cc conf/defaults.yaml
index ee66717,05948e1..3e00b3d
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@@ -153,9 -155,9 +155,12 @@@ storm.messaging.netty.transfer.batch.si
# We check with this interval that whether the Netty channel is writable and try to write pending messages if it is.
storm.messaging.netty.flush.check.interval.ms: 10
+# By default, the Netty SASL authentication is set to false. Users can override and set it true for a specific topology.
+storm.messaging.netty.authentication: false
+
+ # default number of seconds group mapping service will cache user group
+ storm.group.mapping.service.cache.duration.secs: 120
+
### topology.* configs are for specific executing storms
topology.enable.message.timeouts: true
topology.debug: false
http://git-wip-us.apache.org/repos/asf/storm/blob/02e79499/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --cc storm-core/src/jvm/backtype/storm/Config.java
index 46b120c,d6b45ea..94d1cb3
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@@ -103,13 -102,7 +102,13 @@@ public class Config extends HashMap<Str
* We check with this interval that whether the Netty channel is writable and try to write pending messages
*/
public static final String STORM_NETTY_FLUSH_CHECK_INTERVAL_MS = "storm.messaging.netty.flush.check.interval.ms";
- public static final Object STORM_NETTY_FLUSH_CHECK_INTERVAL_MS_SCHEMA = Number.class;
-
+ public static final Object STORM_NETTY_FLUSH_CHECK_INTERVAL_MS_SCHEMA = ConfigValidation.IntegerValidator;
++
+ /**
+ * Netty based messaging: Is authentication required for Netty messaging from client worker process to server worker process.
+ */
+ public static final String STORM_MESSAGING_NETTY_AUTHENTICATION = "storm.messaging.netty.authentication";
+ public static final Object STORM_MESSAGING_NETTY_AUTHENTICATION_SCHEMA = Boolean.class;
/**
* A list of hosts of ZooKeeper servers used to manage the cluster.
http://git-wip-us.apache.org/repos/asf/storm/blob/02e79499/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java
----------------------------------------------------------------------
[30/50] storm git commit: remove empty line
Posted by bo...@apache.org.
remove empty line
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b0cdf65d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b0cdf65d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b0cdf65d
Branch: refs/heads/master
Commit: b0cdf65da8542ffffca8b3caffbd7921866a5649
Parents: d07da44
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Thu Sep 4 12:48:19 2014 -0500
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Thu Sep 4 12:48:19 2014 -0500
----------------------------------------------------------------------
storm-dist/binary/src/main/assembly/binary.xml | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b0cdf65d/storm-dist/binary/src/main/assembly/binary.xml
----------------------------------------------------------------------
diff --git a/storm-dist/binary/src/main/assembly/binary.xml b/storm-dist/binary/src/main/assembly/binary.xml
index e612894..05720f9 100644
--- a/storm-dist/binary/src/main/assembly/binary.xml
+++ b/storm-dist/binary/src/main/assembly/binary.xml
@@ -133,7 +133,6 @@
<outputDirectory>/logback</outputDirectory>
</file>
-
<file>
<source>${project.basedir}/LICENSE</source>
<outputDirectory>/</outputDirectory>
[12/50] storm git commit: Merge branch 'STORM-346' of
https://github.com/Parth-Brahmbhatt/incubator-storm into STORM-346
Posted by bo...@apache.org.
Merge branch 'STORM-346' of https://github.com/Parth-Brahmbhatt/incubator-storm into STORM-346
Conflicts:
storm-core/src/jvm/backtype/storm/Config.java
STORM-346: Oozie style delegation tokens for HDFS
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/1a1df087
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/1a1df087
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/1a1df087
Branch: refs/heads/master
Commit: 1a1df087c1a0f2e15c5e670d8192d66d12d1bae8
Parents: fa40fbc 1fedcd5
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Thu Aug 7 10:09:12 2014 -0500
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Thu Aug 7 10:09:12 2014 -0500
----------------------------------------------------------------------
SECURITY.md | 12 +
.../src/clj/backtype/storm/MockAutoCred.clj | 58 ++++
.../src/clj/backtype/storm/daemon/nimbus.clj | 44 ++--
storm-core/src/jvm/backtype/storm/Config.java | 24 +-
.../storm/security/INimbusCredentialPlugin.java | 47 ++++
.../backtype/storm/security/auth/AuthUtils.java | 24 ++
.../security/auth/ICredentialsRenewer.java | 3 +-
.../storm/security/auth/hadoop/AutoHDFS.java | 262 +++++++++++++++++++
.../storm/security/auth/kerberos/AutoTGT.java | 2 +-
.../test/clj/backtype/storm/nimbus_test.clj | 37 ++-
10 files changed, 490 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/1a1df087/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/1a1df087/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --cc storm-core/src/jvm/backtype/storm/Config.java
index 94d1cb3,03bc438..6ddc98a
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@@ -1216,25 -1192,25 +1222,41 @@@ public class Config extends HashMap<Str
public static final String TOPOLOGY_ISOLATED_MACHINES = "topology.isolate.machines";
public static final Object TOPOLOGY_ISOLATED_MACHINES_SCHEMA = Number.class;
+ /**
+ * HDFS information, used to get the delegation token on behalf of the topology
+ * submitter user and renew the tokens. see {@link backtype.storm.security.auth.hadoop.AutoHDFS}
+ * kerberos principal name with realm should be provided.
+ */
+ public static final Object TOPOLOGY_HDFS_PRINCIPAL = "topology.hdfs.user";
+ public static final Object TOPOLOGY_HDFS_PRINCIPAL_SCHEMA = String.class;
+
+ /**
+ * The HDFS URI to be used by AutoHDFS.java to grab the delegation token on topology
+ * submitter user's behalf by the nimbus. If this is not provided the default URI provided
+ * in the hdfs configuration files will be used.
+ */
+ public static final Object TOPOLOGY_HDFS_URI = "topology.hdfs.uri";
+ public static final Object TOPOLOGY_HDFS_URI_SCHEMA = String.class;
+
+ public static void setClasspath(Map conf, String cp) {
+ conf.put(Config.TOPOLOGY_CLASSPATH, cp);
+ }
+
+ public void setClasspath(String cp) {
+ setClasspath(this, cp);
+ }
+
+ public static void setEnvironment(Map conf, Map env) {
+ conf.put(Config.TOPOLOGY_ENVIRONMENT, env);
+ }
+
+ public void setEnvironment(Map env) {
+ setEnvironment(this, env);
+ }
+
public static void setDebug(Map conf, boolean isOn) {
conf.put(Config.TOPOLOGY_DEBUG, isOn);
- }
+ }
public void setDebug(boolean isOn) {
setDebug(this, isOn);
http://git-wip-us.apache.org/repos/asf/storm/blob/1a1df087/storm-core/src/jvm/backtype/storm/security/auth/AuthUtils.java
----------------------------------------------------------------------
[38/50] storm git commit: Merge branch
'STORM-485-separate-worker-logback' of http://github.com/d2r/incubator-storm
into STORM-485
Posted by bo...@apache.org.
Merge branch 'STORM-485-separate-worker-logback' of http://github.com/d2r/incubator-storm into STORM-485
STORM-485: use separate logback config for workers
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e08bf4b5
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e08bf4b5
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e08bf4b5
Branch: refs/heads/master
Commit: e08bf4b502db76132424d15d1e9540d113bf19ee
Parents: 429ead7 b0cdf65
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Sep 23 16:11:48 2014 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Tue Sep 23 16:11:48 2014 -0500
----------------------------------------------------------------------
logback/worker.xml | 41 ++++++++++++++++++++
.../clj/backtype/storm/daemon/supervisor.clj | 2 +-
.../test/clj/backtype/storm/supervisor_test.clj | 4 +-
storm-dist/binary/src/main/assembly/binary.xml | 4 ++
4 files changed, 48 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[09/50] storm git commit: Merge branch 'show-recent-error-security'
of https://github.com/kishorvpatil/incubator-storm into STORM-366
Posted by bo...@apache.org.
Merge branch 'show-recent-error-security' of https://github.com/kishorvpatil/incubator-storm into STORM-366
Conflicts:
STORM-UI-REST-API.md
STORM-366: Add color span to most recent error and fix ui templates for security.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/9d0b6180
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/9d0b6180
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/9d0b6180
Branch: refs/heads/master
Commit: 9d0b61804a17c3edb6d9b12f64655533ee77b38c
Parents: cf5fc0c 6d0311f
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Wed Jul 30 18:55:51 2014 -0500
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Wed Jul 30 18:55:51 2014 -0500
----------------------------------------------------------------------
STORM-UI-REST-API.md | 46 ++++++++++----------
storm-core/src/clj/backtype/storm/ui/core.clj | 11 ++---
storm-core/src/ui/public/component.html | 10 +++++
.../templates/component-page-template.html | 4 +-
.../templates/topology-page-template.html | 8 +++-
storm-core/src/ui/public/topology.html | 10 +++++
6 files changed, 57 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/9d0b6180/STORM-UI-REST-API.md
----------------------------------------------------------------------
diff --cc STORM-UI-REST-API.md
index e8a412d,72b5ea0..6dda435
--- a/STORM-UI-REST-API.md
+++ b/STORM-UI-REST-API.md
@@@ -341,9 -348,13 +348,13 @@@ Response Fields
|componentType | String | component's type SPOUT or BOLT|
|windowHint| String | window param value in "hh mm ss" format. Default value is "All Time"|
|executors| Integer |Number of executor tasks in the component|
- |componentErrors| Array of Strings | List of component errors|
+ |componentErrors| Array of Errors | List of component errors|
-|componentErrors.time| String | Date/time timezone for the error|
+|componentErrors.time| Long | Timestamp when the exception occurred |
- |componentErrors.error| String | The stack trace of an exception info |
+ |componentErrors.errorHost| String | host name for the error|
+ |componentErrors.errorPort| String | port for the error|
+ |componentErrors.error| String |Shows the error happened in a component|
+ |componentErrors.errorLapsedSecs| Integer | Number of seconds elapsed since the error happened in a component |
+ |componentErrors.errorWorkerLogLink| String | Link to the worker log that reported the exception |
|topologyId| String | Topology's Id|
|tasks| Integer |Number of instances of component|
|window |String. Default value "All Time" | window duration for metrics in seconds|
@@@ -377,26 -388,13 +388,13 @@@ Sample Response
"componentType": "spout",
"windowHint": "10m 0s",
"executors": 5,
- "componentErrors": [
- {
- "time":1406006074000,
- "error":"java.lang.RuntimeException: java.lang.NullPointerException
- at backtype.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:84)
- at backtype.storm.utils.DisruptorQueue.consumeBatchWhenAvailable(DisruptorQueue.java:55)
- at backtype.storm.disruptor$consume_batch_when_available.invoke(disruptor.clj:56)
- at backtype.storm.disruptor$consume_loop_STAR_$fn__1597.invoke(disruptor.clj:67)
- at backtype.storm.util$async_loop$fn__465.invoke(util.clj:377)
- at clojure.lang.AFn.run(AFn.java:24)
- at java.lang.Thread.run(Thread.java:662)
- Caused by: java.lang.NullPointerException
- at backtype.storm.serialization.SerializationFactory$IdDictionary.getStreamName(SerializationFactory.java:137)
- at backtype.storm.serialization.KryoTupleDeserializer.deserialize(KryoTupleDeserializer.java:34)
- at backtype.storm.daemon.executor$mk_task_receiver$fn__3967.invoke(executor.clj:311)
- at backtype.storm.disruptor$clojure_handler$reify__1585.onEvent(disruptor.clj:43)
- at backtype.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:81)
- ... 6 more"
- }
- ],
- "componentErrors":[{"time": "Fri, 18 Jul 2014 19:00:16 +0000",
++ "componentErrors":[{"time": 1406006074000,
+ "errorHost": "10.11.1.70",
+ "errorPort": 6701,
+ "errorWorkerLogLink": "http://10.11.1.7:8000/log?file=worker-6701.log",
+ "errorLapsedSecs": 16,
+ "error": "java.lang.RuntimeException: java.lang.StringIndexOutOfBoundsException: Some Error\n\tat backtype.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:128)\n\tat backtype.storm.utils.DisruptorQueue.consumeBatchWhenAvailable(DisruptorQueue.java:99)\n\tat backtype.storm.disruptor$consume_batch_when_available.invoke(disruptor.clj:80)\n\tat backtype...more.."
+ }],
"topologyId": "WordCount3-1-1402960825",
"tasks": 5,
"window": "600",
http://git-wip-us.apache.org/repos/asf/storm/blob/9d0b6180/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/9d0b6180/storm-core/src/ui/public/component.html
----------------------------------------------------------------------
[27/50] storm git commit: Changed plugin to clear out old credentials.
Posted by bo...@apache.org.
Changed plugin to clear out old credentials.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b07eff3f
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b07eff3f
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b07eff3f
Branch: refs/heads/master
Commit: b07eff3fd86eed1fd67c1159e03d52e583b655e8
Parents: b19fdbb
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Fri Aug 29 19:19:47 2014 +0000
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Fri Aug 29 19:46:54 2014 +0000
----------------------------------------------------------------------
.../security/auth/DefaultHttpCredentialsPlugin.java | 12 ++++++++----
.../security/auth/DefaultHttpCredentialsPlugin_test.clj | 10 +++++-----
2 files changed, 13 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b07eff3f/storm-core/src/jvm/backtype/storm/security/auth/DefaultHttpCredentialsPlugin.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/security/auth/DefaultHttpCredentialsPlugin.java b/storm-core/src/jvm/backtype/storm/security/auth/DefaultHttpCredentialsPlugin.java
index 7d8e0fb..8645558 100644
--- a/storm-core/src/jvm/backtype/storm/security/auth/DefaultHttpCredentialsPlugin.java
+++ b/storm-core/src/jvm/backtype/storm/security/auth/DefaultHttpCredentialsPlugin.java
@@ -72,12 +72,16 @@ public class DefaultHttpCredentialsPlugin implements IHttpCredentialsPlugin {
public ReqContext populateContext(ReqContext context,
HttpServletRequest req) {
String userName = getUserName(req);
+ Principal p = null;
if (userName != null) {
- Set<SingleUserPrincipal> principals = new HashSet<SingleUserPrincipal>(1);
- principals.add(new SingleUserPrincipal(userName));
- Subject s = new Subject(true, principals, new HashSet(), new HashSet());
- context.setSubject(s);
+ p = new SingleUserPrincipal(userName);
}
+ Set<Principal> principals = new HashSet<Principal>(1);
+ if (p != null) {
+ principals.add(p);
+ }
+ Subject s = new Subject(true, principals, new HashSet(), new HashSet());
+ context.setSubject(s);
return context;
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/b07eff3f/storm-core/test/clj/backtype/storm/security/auth/DefaultHttpCredentialsPlugin_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/security/auth/DefaultHttpCredentialsPlugin_test.clj b/storm-core/test/clj/backtype/storm/security/auth/DefaultHttpCredentialsPlugin_test.clj
index ab54d82..bf62a6b 100644
--- a/storm-core/test/clj/backtype/storm/security/auth/DefaultHttpCredentialsPlugin_test.clj
+++ b/storm-core/test/clj/backtype/storm/security/auth/DefaultHttpCredentialsPlugin_test.clj
@@ -2,6 +2,7 @@
(:use [clojure test])
(:import [javax.security.auth Subject])
(:import [javax.servlet.http HttpServletRequest])
+ (:import [backtype.storm.security.auth SingleUserPrincipal])
(:import [org.mockito Mockito])
(:import [backtype.storm.security.auth DefaultHttpCredentialsPlugin
ReqContext SingleUserPrincipal])
@@ -31,10 +32,9 @@
thenReturn princ)
(is (.equals exp-name (.getUserName handler req)))))))
-(deftest test-populate-req-context-noop-on-null-user
+(deftest test-populate-req-context-on-null-user
(let [req (Mockito/mock HttpServletRequest)
handler (doto (DefaultHttpCredentialsPlugin.) (.prepare {}))
- expected-subj (Subject.)
- context (ReqContext. expected-subj)]
- (is (.equals expected-subj
- (-> handler (.populateContext context req) (.subject))))))
+ subj (Subject. false (set [(SingleUserPrincipal. "test")]) (set []) (set []))
+ context (ReqContext. subj)]
+ (is (= 0 (-> handler (.populateContext context req) (.subject) (.getPrincipals) (.size))))))
[32/50] storm git commit: fix indentation
Posted by bo...@apache.org.
fix indentation
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a7b5d68e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a7b5d68e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a7b5d68e
Branch: refs/heads/master
Commit: a7b5d68e65d196c5b6330320b412fbaef6335c1e
Parents: c7fe9ef
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Fri Sep 5 10:37:34 2014 -0500
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Fri Sep 5 10:37:34 2014 -0500
----------------------------------------------------------------------
storm-core/src/ui/public/css/style.css | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/a7b5d68e/storm-core/src/ui/public/css/style.css
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/css/style.css b/storm-core/src/ui/public/css/style.css
index 6790a69..2346f68 100644
--- a/storm-core/src/ui/public/css/style.css
+++ b/storm-core/src/ui/public/css/style.css
@@ -25,7 +25,7 @@
}
.anti-forgery-token {
-visibility:hidden;
+ visibility:hidden;
}
body {
[35/50] storm git commit: STORM-502. storm native worker-launcher
tests fail.
Posted by bo...@apache.org.
STORM-502. storm native worker-launcher tests fail.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/02f6db57
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/02f6db57
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/02f6db57
Branch: refs/heads/master
Commit: 02f6db5783f75a88ba305430991f6fda0bfc7b1a
Parents: 5f3a3a9
Author: Sriharsha Chintalapani <ma...@harsha.io>
Authored: Mon Sep 22 11:16:45 2014 -0700
Committer: Sriharsha Chintalapani <ma...@harsha.io>
Committed: Mon Sep 22 11:16:45 2014 -0700
----------------------------------------------------------------------
storm-core/src/native/worker-launcher/test/test-worker-launcher.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/02f6db57/storm-core/src/native/worker-launcher/test/test-worker-launcher.c
----------------------------------------------------------------------
diff --git a/storm-core/src/native/worker-launcher/test/test-worker-launcher.c b/storm-core/src/native/worker-launcher/test/test-worker-launcher.c
index 412e922..7294511 100644
--- a/storm-core/src/native/worker-launcher/test/test-worker-launcher.c
+++ b/storm-core/src/native/worker-launcher/test/test-worker-launcher.c
@@ -97,7 +97,7 @@ int write_config_file(char *file_name) {
return EXIT_FAILURE;
}
fprintf(file, "banned.users=bannedUser\n");
- fprintf(file, "min.user.id=1000\n");
+ fprintf(file, "min.user.id=%d\n",getuid());
fclose(file);
return 0;
}
[15/50] storm git commit: STORM-460: Fix for storm CSRF vulnerability
using ring-anti-forgery.
Posted by bo...@apache.org.
STORM-460: Fix for storm CSRF vulnerability using ring-anti-forgery.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/4ee22b13
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/4ee22b13
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/4ee22b13
Branch: refs/heads/master
Commit: 4ee22b1383fe14bfc852a0bcd8bf7b391ed47f27
Parents: 1a1df08
Author: Parth Brahmbhatt <br...@gmail.com>
Authored: Sun Aug 17 14:56:27 2014 -0700
Committer: Parth Brahmbhatt <br...@gmail.com>
Committed: Mon Aug 18 08:53:52 2014 -0700
----------------------------------------------------------------------
storm-core/pom.xml | 7 ++++++-
storm-core/src/clj/backtype/storm/ui/core.clj | 14 +++++++++++---
storm-core/src/ui/public/css/style.css | 4 ++++
storm-core/src/ui/public/js/script.js | 3 ++-
.../public/templates/anti-forgery-template.html | 19 +++++++++++++++++++
storm-core/src/ui/public/topology.html | 7 +++++++
6 files changed, 49 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/4ee22b13/storm-core/pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/pom.xml b/storm-core/pom.xml
index c677b7c..44bca86 100644
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@ -60,6 +60,11 @@
<artifactId>ring-jetty-adapter</artifactId>
</dependency>
<dependency>
+ <groupId>ring</groupId>
+ <artifactId>ring-anti-forgery</artifactId>
+ <version>1.0.0</version>
+ </dependency>
+ <dependency>
<groupId>org.clojure</groupId>
<artifactId>tools.logging</artifactId>
</dependency>
@@ -96,7 +101,7 @@
<artifactId>data.codec</artifactId>
<scope>test</scope>
</dependency>
-
+
<!--java-->
<dependency>
<groupId>commons-io</groupId>
http://git-wip-us.apache.org/repos/asf/storm/blob/4ee22b13/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/ui/core.clj b/storm-core/src/clj/backtype/storm/ui/core.clj
index 8dd301e..f771ee6 100644
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@ -23,6 +23,7 @@
(:use [backtype.storm.daemon [common :only [ACKER-COMPONENT-ID ACKER-INIT-STREAM-ID
ACKER-ACK-STREAM-ID ACKER-FAIL-STREAM-ID system-id?]]])
(:use [ring.adapter.jetty :only [run-jetty]])
+ (:use [ring.middleware.anti-forgery])
(:use [clojure.string :only [blank? lower-case trim]])
(:import [backtype.storm.utils Utils])
(:import [backtype.storm.generated ExecutorSpecificStats
@@ -687,7 +688,8 @@
"bolts" (bolt-comp id bolt-comp-summs (.get_errors summ) window include-sys?)
"configuration" topology-conf
"visualizationTable" (stream-boxes visualizer-data)
- "uiActionsEnabled" (ui-actions-enabled?)}))))
+ "uiActionsEnabled" (ui-actions-enabled?)
+ "anti-forgery-token" *anti-forgery-token*}))))
(defn spout-output-stats
[stream-summary window]
@@ -956,10 +958,16 @@
(catch Exception ex
(json-response (exception->json ex) 500)))))
+
+(def csrf-error-response
+ (json-response {"error" "Forbidden action."
+ "errorMessage" "missing CSRF token."} 403))
+
(def app
(handler/site (-> main-routes
- (wrap-reload '[backtype.storm.ui.core])
- catch-errors)))
+ (wrap-reload '[backtype.storm.ui.core])
+ (wrap-anti-forgery {:error-response csrf-error-response})
+ catch-errors)))
(defn start-server!
[]
http://git-wip-us.apache.org/repos/asf/storm/blob/4ee22b13/storm-core/src/ui/public/css/style.css
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/css/style.css b/storm-core/src/ui/public/css/style.css
index 29a45eb..85a8961 100644
--- a/storm-core/src/ui/public/css/style.css
+++ b/storm-core/src/ui/public/css/style.css
@@ -24,6 +24,10 @@
padding: 0.5em;
}
+.anti-forgery-token {
+visibility:hidden;
+}
+
body {
color: #808080;
padding: 0.2em;
http://git-wip-us.apache.org/repos/asf/storm/blob/4ee22b13/storm-core/src/ui/public/js/script.js
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/js/script.js b/storm-core/src/ui/public/js/script.js
index fef3b59..09ecd57 100644
--- a/storm-core/src/ui/public/js/script.js
+++ b/storm-core/src/ui/public/js/script.js
@@ -71,7 +71,8 @@ function ensureInt(n) {
function confirmAction(id, name, action, wait, defaultWait) {
var opts = {
type:'POST',
- url:'/api/v1/topology/' + id + '/' + action
+ url:'/api/v1/topology/' + id + '/' + action,
+ headers: { 'x-csrf-token': $.trim($('#anti-forgery-token').text()) }
};
if (wait) {
var waitSecs = prompt('Do you really want to ' + action + ' topology "' + name + '"? ' +
http://git-wip-us.apache.org/repos/asf/storm/blob/4ee22b13/storm-core/src/ui/public/templates/anti-forgery-template.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/templates/anti-forgery-template.html b/storm-core/src/ui/public/templates/anti-forgery-template.html
new file mode 100644
index 0000000..1a63ace
--- /dev/null
+++ b/storm-core/src/ui/public/templates/anti-forgery-template.html
@@ -0,0 +1,19 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<script id="anti-forgery-template" type="text/html">
+{{#anti-forgery-token}}{{anti-forgery-token}}{{/anti-forgery-token}}
+</script>
http://git-wip-us.apache.org/repos/asf/storm/blob/4ee22b13/storm-core/src/ui/public/topology.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/topology.html b/storm-core/src/ui/public/topology.html
index eb74522..805b6b4 100644
--- a/storm-core/src/ui/public/topology.html
+++ b/storm-core/src/ui/public/topology.html
@@ -52,6 +52,8 @@
<p id="toggle-switch" style="display: block;" class="js-only"></p>
<div id="json-response-error">
</div>
+<div id="anti-forgery-token" class="anti-forgery-token">
+</div>
</body>
<script>
$(document).ready(function() {
@@ -76,6 +78,11 @@ $(document).ready(function() {
uiUser.append(Mustache.render($(template).filter("#user-template").html(),response));
});
+ var antiForgeryToken = $("#anti-forgery-token");
+ $.get("/templates/anti-forgery-template.html", function(template) {
+ antiForgeryToken.append(Mustache.render($(template).filter("#anti-forgery-template").html(),response));
+ });
+
var topologySummary = $("#topology-summary");
var topologyStats = $("#topology-stats");
var spoutStats = $("#spout-stats");
[24/50] storm git commit: Fixed some merge issues.
Posted by bo...@apache.org.
Fixed some merge issues.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b19fdbb7
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b19fdbb7
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b19fdbb7
Branch: refs/heads/master
Commit: b19fdbb7444e9d3ea38ede9d186055319df4751d
Parents: 84b7dea
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Thu Aug 28 15:02:27 2014 -0500
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Thu Aug 28 15:02:27 2014 -0500
----------------------------------------------------------------------
pom.xml | 5 +++++
storm-core/pom.xml | 4 ++++
storm-core/src/clj/backtype/storm/daemon/logviewer.clj | 4 +---
storm-core/src/ui/public/templates/index-page-template.html | 2 +-
storm-core/src/ui/public/templates/topology-page-template.html | 2 --
5 files changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b19fdbb7/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7d31916..2ee56fc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -397,6 +397,11 @@
<version>${jetty.version}</version>
</dependency>
<dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlets</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.clojure</groupId>
<artifactId>tools.logging</artifactId>
<version>${clojure.tools.logging.version}</version>
http://git-wip-us.apache.org/repos/asf/storm/blob/b19fdbb7/storm-core/pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/pom.xml b/storm-core/pom.xml
index 1a676f1..3785858 100644
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@ -64,6 +64,10 @@
<artifactId>jetty-servlet</artifactId>
</dependency>
<dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlets</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.clojure</groupId>
<artifactId>tools.logging</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/storm/blob/b19fdbb7/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/logviewer.clj b/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
index 36a737f..c3868eb 100644
--- a/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
@@ -348,15 +348,13 @@ Note that if anything goes wrong, this will throw an Error and exit."
:filter-params (or (conf UI-FILTER-PARAMS) {})}]
[])
filters-confs (concat filters-confs
- [{:filter-class "org.mortbay.servlet.GzipFilter"
+ [{:filter-class "org.eclipse.jetty.servlets.GzipFilter"
:filter-name "Gzipper"
:filter-params {}}])]
(run-jetty middle
{:port (int (conf LOGVIEWER-PORT))
:join? false
:configurator (fn [server]
- (doseq [connector (.getConnectors server)]
- (.setHeaderBufferSize connector header-buffer-size))
(config-filter server middle filters-confs))}))
(catch Exception ex
(log-error ex))))
http://git-wip-us.apache.org/repos/asf/storm/blob/b19fdbb7/storm-core/src/ui/public/templates/index-page-template.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/templates/index-page-template.html b/storm-core/src/ui/public/templates/index-page-template.html
index 8f1c29b..dfe9446 100644
--- a/storm-core/src/ui/public/templates/index-page-template.html
+++ b/storm-core/src/ui/public/templates/index-page-template.html
@@ -53,8 +53,8 @@
<td>{{uptime}}</td>
<td>{{workersTotal}}</td>
<td>{{executorsTotal}}</td>
- <td>{{schedulerInfo}}</td>
<td>{{tasksTotal}}</td>
+ <td>{{schedulerInfo}}</td>
</tr>
{{/topologies}}
</tbody>
http://git-wip-us.apache.org/repos/asf/storm/blob/b19fdbb7/storm-core/src/ui/public/templates/topology-page-template.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/templates/topology-page-template.html b/storm-core/src/ui/public/templates/topology-page-template.html
index ddf3c17..d442107 100644
--- a/storm-core/src/ui/public/templates/topology-page-template.html
+++ b/storm-core/src/ui/public/templates/topology-page-template.html
@@ -160,7 +160,6 @@
</script>
<script id="topology-actions-template" type="text/html">
- {{#uiActionsEnabled}}
<h2>Topology actions</h2>
<p id="topology-actions">
<input {{activateStatus}} onclick="confirmAction('{{id}}', '{{name}}', 'activate', false, 0)" type="button" value="Activate">
@@ -168,5 +167,4 @@
<input {{rebalanceStatus}} onclick="confirmAction('{{id}}', '{{name}}', 'rebalance', true, {{msgTimeout}})" type="button" value="Rebalance">
<input {{killStatus}} onclick="confirmAction('{{id}}', '{{name}}', 'kill', true, 30)" type="button" value="Kill">
</p>
- {{/uiActionsEnabled}}
</script>
[08/50] storm git commit: Updated README for STORM-348
Posted by bo...@apache.org.
Updated README for STORM-348
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/cf5fc0c3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/cf5fc0c3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/cf5fc0c3
Branch: refs/heads/master
Commit: cf5fc0c3f0a71d8e81d344f247a477694e83a2a7
Parents: 00f7d63
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Wed Jul 30 11:15:43 2014 -0500
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Wed Jul 30 11:15:43 2014 -0500
----------------------------------------------------------------------
README.markdown | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/cf5fc0c3/README.markdown
----------------------------------------------------------------------
diff --git a/README.markdown b/README.markdown
index cf67f64..586f171 100644
--- a/README.markdown
+++ b/README.markdown
@@ -159,6 +159,7 @@ under the License.
* Jo Liss ([@joliss](https://github.com/joliss))
* averykhoo ([@averykhoo](https://github.com/averykhoo))
* Curtis Allen ([@curtisallen](https://github.com/curtisallen))
+* Raghavendra Nandagopal ([@RaghavendraNandagopal](https://github.com/RaghavendraNandagopal))
## Acknowledgements
[21/50] storm git commit: Merge remote-tracking branch
'upstream/security' into STORM-406
Posted by bo...@apache.org.
Merge remote-tracking branch 'upstream/security' into STORM-406
Conflicts:
storm-core/pom.xml
storm-core/src/clj/backtype/storm/ui/core.clj
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b62d71d6
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b62d71d6
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b62d71d6
Branch: refs/heads/master
Commit: b62d71d6696163134ca19e71e55cbc96fde0524d
Parents: dd59ad4 7a06ef7
Author: Parth Brahmbhatt <br...@gmail.com>
Authored: Tue Aug 26 15:11:21 2014 -0700
Committer: Parth Brahmbhatt <br...@gmail.com>
Committed: Tue Aug 26 15:11:21 2014 -0700
----------------------------------------------------------------------
CHANGELOG.md | 12 +
README.markdown | 3 +
SECURITY.md | 25 +-
bin/storm | 6 +-
external/storm-hdfs/README.md | 343 +++++++++++++++++++
external/storm-hdfs/pom.xml | 68 ++++
.../storm/hdfs/bolt/AbstractHdfsBolt.java | 139 ++++++++
.../org/apache/storm/hdfs/bolt/HdfsBolt.java | 131 +++++++
.../storm/hdfs/bolt/SequenceFileBolt.java | 149 ++++++++
.../hdfs/bolt/format/DefaultFileNameFormat.java | 86 +++++
.../hdfs/bolt/format/DefaultSequenceFormat.java | 52 +++
.../hdfs/bolt/format/DelimitedRecordFormat.java | 88 +++++
.../storm/hdfs/bolt/format/FileNameFormat.java | 42 +++
.../storm/hdfs/bolt/format/RecordFormat.java | 32 ++
.../storm/hdfs/bolt/format/SequenceFormat.java | 40 +++
.../hdfs/bolt/rotation/FileRotationPolicy.java | 51 +++
.../bolt/rotation/FileSizeRotationPolicy.java | 81 +++++
.../hdfs/bolt/rotation/NoRotationPolicy.java | 35 ++
.../hdfs/bolt/rotation/TimedRotationPolicy.java | 55 +++
.../storm/hdfs/bolt/sync/CountSyncPolicy.java | 46 +++
.../apache/storm/hdfs/bolt/sync/SyncPolicy.java | 46 +++
.../hdfs/common/rotation/MoveFileAction.java | 27 ++
.../hdfs/common/rotation/RotationAction.java | 12 +
.../hdfs/common/security/HdfsSecurityUtil.java | 48 +++
.../apache/storm/hdfs/trident/HdfsState.java | 317 +++++++++++++++++
.../storm/hdfs/trident/HdfsStateFactory.java | 29 ++
.../apache/storm/hdfs/trident/HdfsUpdater.java | 14 +
.../trident/format/DefaultFileNameFormat.java | 83 +++++
.../trident/format/DefaultSequenceFormat.java | 54 +++
.../trident/format/DelimitedRecordFormat.java | 83 +++++
.../hdfs/trident/format/FileNameFormat.java | 40 +++
.../storm/hdfs/trident/format/RecordFormat.java | 32 ++
.../hdfs/trident/format/SequenceFormat.java | 40 +++
.../trident/rotation/FileRotationPolicy.java | 50 +++
.../rotation/FileSizeRotationPolicy.java | 81 +++++
.../hdfs/trident/rotation/NoRotationPolicy.java | 35 ++
.../trident/rotation/TimedRotationPolicy.java | 54 +++
.../hdfs/trident/sync/CountSyncPolicy.java | 46 +++
.../storm/hdfs/trident/sync/SyncPolicy.java | 46 +++
.../storm/hdfs/bolt/HdfsFileTopology.java | 198 +++++++++++
.../storm/hdfs/bolt/SequenceFileTopology.java | 193 +++++++++++
.../storm/hdfs/trident/FixedBatchSpout.java | 80 +++++
.../storm/hdfs/trident/TridentFileTopology.java | 73 ++++
.../hdfs/trident/TridentSequenceTopology.java | 71 ++++
external/storm-kafka/README.md | 197 ++++++++++-
.../jvm/storm/kafka/DynamicBrokersReader.java | 5 +
.../src/jvm/storm/kafka/bolt/KafkaBolt.java | 49 ++-
.../FieldNameBasedTupleToKafkaMapper.java | 48 +++
.../kafka/bolt/mapper/TupleToKafkaMapper.java | 32 ++
.../bolt/selector/DefaultTopicSelector.java | 34 ++
.../kafka/bolt/selector/KafkaTopicSelector.java | 26 ++
.../storm/kafka/trident/TridentKafkaState.java | 99 ++++++
.../kafka/trident/TridentKafkaStateFactory.java | 57 +++
.../kafka/trident/TridentKafkaUpdater.java | 31 ++
.../FieldNameBasedTupleToKafkaMapper.java | 41 +++
.../mapper/TridentTupleToKafkaMapper.java | 28 ++
.../trident/selector/DefaultTopicSelector.java | 34 ++
.../trident/selector/KafkaTopicSelector.java | 26 ++
.../src/test/storm/kafka/TestUtils.java | 62 ++++
.../src/test/storm/kafka/TridentKafkaTest.java | 89 +++++
.../test/storm/kafka/TridentKafkaTopology.java | 100 ++++++
logback/cluster.xml | 12 +-
pom.xml | 13 +-
storm-core/pom.xml | 70 +++-
.../clj/backtype/storm/daemon/supervisor.clj | 9 +-
storm-core/src/clj/backtype/storm/disruptor.clj | 11 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 103 +++---
.../src/clj/backtype/storm/ui/helpers.clj | 15 +-
storm-core/src/clj/backtype/storm/util.clj | 2 +-
storm-core/src/jvm/backtype/storm/Config.java | 33 +-
.../jvm/backtype/storm/multilang/ShellMsg.java | 17 +
.../jvm/backtype/storm/spout/ShellSpout.java | 3 +
.../src/jvm/backtype/storm/task/ShellBolt.java | 6 +
.../backtype/storm/utils/DisruptorQueue.java | 56 ++-
storm-core/src/ui/public/css/style.css | 67 ++++
.../public/templates/index-page-template.html | 2 +-
.../templates/topology-page-template.html | 2 +-
.../test/clj/backtype/storm/supervisor_test.clj | 2 +
storm-core/test/clj/backtype/storm/ui_test.clj | 49 ---
.../storm/utils/DisruptorQueueTest.java | 25 +-
storm-dist/binary/src/main/assembly/binary.xml | 14 +
81 files changed, 4490 insertions(+), 215 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b62d71d6/storm-core/pom.xml
----------------------------------------------------------------------
diff --cc storm-core/pom.xml
index 44bca86,1a676f1..a8641c1
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@@ -60,10 -60,9 +60,15 @@@
<artifactId>ring-jetty-adapter</artifactId>
</dependency>
<dependency>
+ <groupId>ring</groupId>
+ <artifactId>ring-anti-forgery</artifactId>
+ <version>1.0.0</version>
+ </dependency>
++
++ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ </dependency>
<dependency>
<groupId>org.clojure</groupId>
<artifactId>tools.logging</artifactId>
http://git-wip-us.apache.org/repos/asf/storm/blob/b62d71d6/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/ui/core.clj
index 93adb18,0aea7ec..3c90715
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@@ -20,10 -20,9 +20,10 @@@
(:use [hiccup core page-helpers])
(:use [backtype.storm config util log])
(:use [backtype.storm.ui helpers])
- (:use [backtype.storm.daemon [common :only [ACKER-COMPONENT-ID ACKER-INIT-STREAM-ID
- ACKER-ACK-STREAM-ID ACKER-FAIL-STREAM-ID system-id?]]])
+ (:use [backtype.storm.daemon [common :only [ACKER-COMPONENT-ID ACKER-INIT-STREAM-ID ACKER-ACK-STREAM-ID
+ ACKER-FAIL-STREAM-ID system-id? mk-authorization-handler]]])
(:use [ring.adapter.jetty :only [run-jetty]])
+ (:use [ring.middleware.anti-forgery])
(:use [clojure.string :only [blank? lower-case trim]])
(:import [backtype.storm.utils Utils])
(:import [backtype.storm.generated ExecutorSpecificStats
@@@ -687,9 -675,7 +676,8 @@@
"spouts" (spout-comp id spout-comp-summs (.get_errors summ) window include-sys?)
"bolts" (bolt-comp id bolt-comp-summs (.get_errors summ) window include-sys?)
"configuration" topology-conf
- "visualizationTable" (stream-boxes visualizer-data)}))))
+ "visualizationTable" (stream-boxes visualizer-data)
- "uiActionsEnabled" (ui-actions-enabled?)
+ "antiForgeryToken" *anti-forgery-token*}))))
(defn spout-output-stats
[stream-summary window]
http://git-wip-us.apache.org/repos/asf/storm/blob/b62d71d6/storm-core/src/ui/public/css/style.css
----------------------------------------------------------------------
[45/50] storm git commit: Merge branch 'http-auth-plugin-fix' of
https://github.com/revans2/incubator-storm into STORM-482
Posted by bo...@apache.org.
Merge branch 'http-auth-plugin-fix' of https://github.com/revans2/incubator-storm into STORM-482
[STORM-482] (Security) DefaultHttpCredentialsPlugin does not handle no
credential properly
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/fce22a7d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/fce22a7d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/fce22a7d
Branch: refs/heads/master
Commit: fce22a7d48928730873fb9e7eb1724ea72fca356
Parents: d45c187 b07eff3
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Tue Oct 7 16:24:31 2014 -0500
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Tue Oct 7 16:24:31 2014 -0500
----------------------------------------------------------------------
.../security/auth/DefaultHttpCredentialsPlugin.java | 12 ++++++++----
.../security/auth/DefaultHttpCredentialsPlugin_test.clj | 10 +++++-----
2 files changed, 13 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
[50/50] storm git commit: Merge branch 'security'
Posted by bo...@apache.org.
Merge branch 'security'
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/413a6952
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/413a6952
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/413a6952
Branch: refs/heads/master
Commit: 413a6952c2acabdb97a9e4d68eaa774faafc120b
Parents: 64f06c7 e6b1886
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Thu Nov 13 12:47:07 2014 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Thu Nov 13 12:47:07 2014 -0600
----------------------------------------------------------------------
README.markdown | 1 +
SECURITY.md | 356 ++-
STORM-UI-REST-API.md | 17 +-
bin/storm | 15 +-
conf/defaults.yaml | 45 +-
conf/jaas_digest.conf | 8 +-
conf/jaas_kerberos.conf | 15 +
.../src/jvm/storm/starter/util/StormRunner.java | 3 +-
logback/worker.xml | 41 +
pom.xml | 33 +-
storm-core/pom.xml | 299 +-
.../src/clj/backtype/storm/LocalCluster.clj | 4 +
storm-core/src/clj/backtype/storm/LocalDRPC.clj | 4 +-
.../src/clj/backtype/storm/MockAutoCred.clj | 58 +
storm-core/src/clj/backtype/storm/bootstrap.clj | 5 +-
storm-core/src/clj/backtype/storm/cluster.clj | 167 +-
.../storm/command/upload_credentials.clj | 35 +
storm-core/src/clj/backtype/storm/config.clj | 40 +-
.../src/clj/backtype/storm/daemon/common.clj | 18 +-
.../src/clj/backtype/storm/daemon/drpc.clj | 174 +-
.../src/clj/backtype/storm/daemon/executor.clj | 119 +-
.../src/clj/backtype/storm/daemon/logviewer.clj | 271 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 245 +-
.../clj/backtype/storm/daemon/supervisor.clj | 173 +-
.../src/clj/backtype/storm/daemon/task.clj | 7 +-
.../src/clj/backtype/storm/daemon/worker.clj | 97 +-
storm-core/src/clj/backtype/storm/testing.clj | 52 +-
storm-core/src/clj/backtype/storm/testing4j.clj | 2 +-
storm-core/src/clj/backtype/storm/thrift.clj | 28 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 169 +-
.../src/clj/backtype/storm/ui/helpers.clj | 69 +-
storm-core/src/clj/backtype/storm/util.clj | 89 +-
storm-core/src/clj/backtype/storm/zookeeper.clj | 19 +-
storm-core/src/clj/storm/trident/testing.clj | 1 +
.../src/dev/drpc-simple-acl-test-scenario.yaml | 11 +
storm-core/src/jvm/backtype/storm/Config.java | 537 +++-
.../jvm/backtype/storm/ConfigValidation.java | 156 +-
.../src/jvm/backtype/storm/Constants.java | 3 +-
.../backtype/storm/ICredentialsListener.java | 32 +
.../src/jvm/backtype/storm/ILocalCluster.java | 2 +
.../src/jvm/backtype/storm/StormSubmitter.java | 164 +-
.../storm/drpc/DRPCInvocationsClient.java | 91 +-
.../src/jvm/backtype/storm/drpc/DRPCSpout.java | 100 +-
.../jvm/backtype/storm/drpc/ReturnResults.java | 35 +-
.../storm/generated/AuthorizationException.java | 345 ++
.../backtype/storm/generated/Credentials.java | 390 +++
.../storm/generated/DistributedRPC.java | 110 +-
.../generated/DistributedRPCInvocations.java | 352 +-
.../jvm/backtype/storm/generated/Nimbus.java | 3006 +++++++++++++++++-
.../backtype/storm/generated/SubmitOptions.java | 98 +-
.../backtype/storm/generated/TopologyInfo.java | 192 +-
.../storm/generated/TopologySummary.java | 192 +-
.../backtype/storm/messaging/netty/Client.java | 31 +-
.../storm/messaging/netty/ControlMessage.java | 4 +-
.../storm/messaging/netty/MessageDecoder.java | 32 +-
.../storm/messaging/netty/MessageEncoder.java | 4 +
.../storm/messaging/netty/SaslMessageToken.java | 99 +
.../storm/messaging/netty/SaslNettyClient.java | 166 +
.../messaging/netty/SaslNettyClientState.java | 31 +
.../storm/messaging/netty/SaslNettyServer.java | 165 +
.../messaging/netty/SaslNettyServerState.java | 31 +
.../messaging/netty/SaslStormClientHandler.java | 158 +
.../netty/SaslStormServerAuthorizeHandler.java | 83 +
.../messaging/netty/SaslStormServerHandler.java | 155 +
.../storm/messaging/netty/SaslUtils.java | 74 +
.../backtype/storm/messaging/netty/Server.java | 27 +-
.../netty/StormClientPipelineFactory.java | 12 +-
.../netty/StormServerPipelineFactory.java | 20 +-
.../jvm/backtype/storm/scheduler/Cluster.java | 13 +
.../scheduler/multitenant/DefaultPool.java | 219 ++
.../storm/scheduler/multitenant/FreePool.java | 125 +
.../scheduler/multitenant/IsolatedPool.java | 346 ++
.../multitenant/MultitenantScheduler.java | 98 +
.../storm/scheduler/multitenant/Node.java | 343 ++
.../storm/scheduler/multitenant/NodePool.java | 296 ++
.../storm/security/INimbusCredentialPlugin.java | 47 +
.../backtype/storm/security/auth/AuthUtils.java | 228 +-
.../auth/DefaultHttpCredentialsPlugin.java | 87 +
.../security/auth/DefaultPrincipalToLocal.java | 43 +
.../storm/security/auth/IAutoCredentials.java | 55 +
.../security/auth/ICredentialsRenewer.java | 41 +
.../auth/IGroupMappingServiceProvider.java | 42 +
.../security/auth/IHttpCredentialsPlugin.java | 50 +
.../storm/security/auth/IPrincipalToLocal.java | 41 +
.../storm/security/auth/ITransportPlugin.java | 14 +-
.../security/auth/KerberosPrincipalToLocal.java | 45 +
.../storm/security/auth/ReqContext.java | 10 +-
.../security/auth/SaslTransportPlugin.java | 44 +-
.../security/auth/ShellBasedGroupsMapping.java | 94 +
.../security/auth/SimpleTransportPlugin.java | 61 +-
.../security/auth/SingleUserPrincipal.java | 56 +
.../storm/security/auth/TBackoffConnect.java | 77 +
.../storm/security/auth/ThriftClient.java | 85 +-
.../security/auth/ThriftConnectionType.java | 77 +
.../storm/security/auth/ThriftServer.java | 19 +-
.../auth/authorizer/DRPCAuthorizerBase.java | 46 +
.../authorizer/DRPCSimpleACLAuthorizer.java | 157 +
.../auth/authorizer/DenyAuthorizer.java | 4 +-
.../auth/authorizer/NoopAuthorizer.java | 6 +-
.../auth/authorizer/SimpleACLAuthorizer.java | 131 +
.../authorizer/SimpleWhitelistAuthorizer.java | 70 +
.../auth/digest/DigestSaslTransportPlugin.java | 1 +
.../storm/security/auth/hadoop/AutoHDFS.java | 262 ++
.../storm/security/auth/kerberos/AutoTGT.java | 281 ++
.../auth/kerberos/AutoTGTKrb5LoginModule.java | 108 +
.../kerberos/AutoTGTKrb5LoginModuleTest.java | 44 +
.../auth/kerberos/ClientCallbackHandler.java | 104 +
.../kerberos/KerberosSaslTransportPlugin.java | 206 ++
.../auth/kerberos/ServerCallbackHandler.java | 86 +
.../auth/kerberos/jaas_kerberos_cluster.conf | 31 +
.../auth/kerberos/jaas_kerberos_launcher.conf | 12 +
.../testing/SingleUserSimpleTransport.java | 37 +
.../state/TestTransactionalState.java | 47 +
.../transactional/state/TransactionalState.java | 56 +-
.../jvm/backtype/storm/utils/DRPCClient.java | 63 +-
.../jvm/backtype/storm/utils/LocalState.java | 14 +-
.../jvm/backtype/storm/utils/NimbusClient.java | 11 +-
.../jvm/backtype/storm/utils/ShellUtils.java | 498 +++
.../src/jvm/backtype/storm/utils/TestUtils.java | 34 +
.../src/jvm/backtype/storm/utils/Utils.java | 130 +-
.../backtype/storm/utils/ZookeeperAuthInfo.java | 9 +-
.../storm/utils/ZookeeperServerCnxnFactory.java | 84 +
.../trident/drpc/ReturnResultsReducer.java | 13 +-
.../topology/state/TestTransactionalState.java | 47 +
.../topology/state/TransactionalState.java | 58 +-
.../src/native/worker-launcher/.autom4te.cfg | 42 +
.../worker-launcher/.deps/worker-launcher.Po | 1 +
.../src/native/worker-launcher/Makefile.am | 32 +
.../src/native/worker-launcher/configure.ac | 50 +
.../native/worker-launcher/impl/configuration.c | 340 ++
.../native/worker-launcher/impl/configuration.h | 45 +
.../src/native/worker-launcher/impl/main.c | 210 ++
.../worker-launcher/impl/worker-launcher.c | 779 +++++
.../worker-launcher/impl/worker-launcher.h | 129 +
.../worker-launcher/test/test-worker-launcher.c | 340 ++
storm-core/src/py/__init__.py | 16 +
storm-core/src/py/storm/DistributedRPC-remote | 18 +
storm-core/src/py/storm/DistributedRPC.py | 37 +-
.../py/storm/DistributedRPCInvocations-remote | 18 +
.../src/py/storm/DistributedRPCInvocations.py | 96 +-
storm-core/src/py/storm/Nimbus-remote | 25 +
storm-core/src/py/storm/Nimbus.py | 652 +++-
storm-core/src/py/storm/__init__.py | 16 +
storm-core/src/py/storm/constants.py | 16 +
storm-core/src/py/storm/ttypes.py | 1243 +++++++-
storm-core/src/storm.thrift | 58 +-
storm-core/src/ui/public/component.html | 21 +-
storm-core/src/ui/public/css/style.css | 9 +
storm-core/src/ui/public/index.html | 12 +-
storm-core/src/ui/public/js/script.js | 3 +-
.../public/templates/anti-forgery-template.html | 19 +
.../public/templates/index-page-template.html | 12 +
.../templates/topology-page-template.html | 23 +-
.../src/ui/public/templates/user-template.html | 25 +
storm-core/src/ui/public/topology.html | 21 +-
.../test/clj/backtype/storm/cluster_test.clj | 93 +-
.../test/clj/backtype/storm/config_test.clj | 11 +
.../test/clj/backtype/storm/drpc_test.clj | 14 +-
.../test/clj/backtype/storm/logviewer_test.clj | 187 ++
.../storm/messaging/netty_integration_test.clj | 4 +-
.../storm/messaging/netty_unit_test.clj | 4 +
.../test/clj/backtype/storm/nimbus_test.clj | 377 ++-
.../scheduler/multitenant_scheduler_test.clj | 831 +++++
.../storm/security/auth/AuthUtils_test.clj | 16 +-
.../auth/DefaultHttpCredentialsPlugin_test.clj | 40 +
.../storm/security/auth/ThriftClient_test.clj | 28 +-
.../storm/security/auth/ThriftServer_test.clj | 8 +-
.../backtype/storm/security/auth/auth_test.clj | 374 ++-
.../authorizer/DRPCSimpleACLAuthorizer_test.clj | 226 ++
.../security/auth/auto_login_module_test.clj | 91 +
.../storm/security/auth/drpc-auth-alice.jaas | 5 +
.../storm/security/auth/drpc-auth-bob.jaas | 5 +
.../storm/security/auth/drpc-auth-charlie.jaas | 5 +
.../storm/security/auth/drpc-auth-server.jaas | 6 +
.../storm/security/auth/drpc_auth_test.clj | 315 ++
.../storm/security/auth/nimbus_auth_test.clj | 181 ++
.../test/clj/backtype/storm/submitter_test.clj | 75 +
.../test/clj/backtype/storm/supervisor_test.clj | 176 +-
.../clj/backtype/storm/transactional_test.clj | 27 +-
.../utils/ZookeeperServerCnxnFactory_test.clj | 35 +
.../test/clj/backtype/storm/utils_test.clj | 58 +-
.../test/clj/storm/trident/state_test.clj | 25 +-
storm-dist/binary/src/main/assembly/binary.xml | 4 +
183 files changed, 21137 insertions(+), 1285 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/413a6952/storm-core/src/clj/backtype/storm/daemon/executor.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/413a6952/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/413a6952/storm-core/src/clj/backtype/storm/util.clj
----------------------------------------------------------------------
[26/50] storm git commit: Merge remote-tracking branch
'upstream/security' into STORM-406
Posted by bo...@apache.org.
Merge remote-tracking branch 'upstream/security' into STORM-406
Conflicts:
STORM-UI-REST-API.md
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/c7fe9ef9
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/c7fe9ef9
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/c7fe9ef9
Branch: refs/heads/master
Commit: c7fe9ef97962b7cd0eae11754d2bfbdd473139a9
Parents: b62d71d b19fdbb
Author: Parth Brahmbhatt <br...@gmail.com>
Authored: Fri Aug 29 09:52:23 2014 -0700
Committer: Parth Brahmbhatt <br...@gmail.com>
Committed: Fri Aug 29 09:52:23 2014 -0700
----------------------------------------------------------------------
CHANGELOG.md | 15 +++
README.markdown | 7 ++
STORM-UI-REST-API.md | 3 +-
bin/storm.cmd | 10 +-
conf/defaults.yaml | 1 +
.../jvm/storm/starter/PrintSampleStream.java | 4 +-
pom.xml | 13 +-
storm-core/pom.xml | 4 +
.../src/clj/backtype/storm/LocalCluster.clj | 4 +-
.../src/clj/backtype/storm/daemon/common.clj | 3 +
.../src/clj/backtype/storm/daemon/logviewer.clj | 4 +-
.../clj/backtype/storm/daemon/supervisor.clj | 32 +++--
.../src/clj/backtype/storm/daemon/worker.clj | 1 -
storm-core/src/clj/backtype/storm/event.clj | 3 +
.../backtype/storm/scheduler/EvenScheduler.clj | 2 +-
storm-core/src/clj/backtype/storm/testing.clj | 9 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 40 ++++---
storm-core/src/jvm/backtype/storm/Config.java | 42 +++++--
.../storm/multilang/JsonSerializer.java | 14 ++-
.../DefaultSerializationDelegate.java | 61 ++++++++++
.../GzipBridgeSerializationDelegate.java | 64 ++++++++++
.../GzipSerializationDelegate.java | 68 +++++++++++
.../serialization/SerializationDelegate.java | 35 ++++++
.../src/jvm/backtype/storm/task/ShellBolt.java | 9 +-
.../src/jvm/backtype/storm/utils/Utils.java | 60 ++++++----
.../public/templates/index-page-template.html | 2 +-
.../templates/topology-page-template.html | 2 -
.../test/clj/backtype/storm/scheduler_test.clj | 21 ++++
.../test/clj/backtype/storm/supervisor_test.clj | 118 ++++++++-----------
.../GzipBridgeSerializationDelegateTest.java | 82 +++++++++++++
30 files changed, 567 insertions(+), 166 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/c7fe9ef9/STORM-UI-REST-API.md
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/c7fe9ef9/storm-core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/c7fe9ef9/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/ui/core.clj
index 3c90715,d0d71c3..2181b48
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@@ -947,18 -953,12 +955,18 @@@
(try
(handler request)
(catch Exception ex
- (json-response (exception->json ex) 500)))))
+ (json-response (exception->json ex) ((:query-params request) "callback") :status 500)))))
+
+(def csrf-error-response
+ (json-response {"error" "Forbidden action."
+ "errorMessage" "missing CSRF token."} 403))
+
(def app
(handler/site (-> main-routes
- (wrap-reload '[backtype.storm.ui.core])
- catch-errors)))
+ (wrap-reload '[backtype.storm.ui.core])
+ (wrap-anti-forgery {:error-response csrf-error-response})
+ catch-errors)))
(defn start-server!
[]
[49/50] storm git commit: Merge branch 'master' into security-upmerge
Posted by bo...@apache.org.
Merge branch 'master' into security-upmerge
Conflicts:
storm-core/src/clj/backtype/storm/testing.clj
storm-core/src/jvm/backtype/storm/utils/LocalState.java
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e6b18863
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e6b18863
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e6b18863
Branch: refs/heads/master
Commit: e6b18863e1cc5a5af8c7d1da703d31d9e17b70bf
Parents: 04a7937 e15dbe2
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Wed Nov 12 10:43:41 2014 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Wed Nov 12 10:43:41 2014 -0600
----------------------------------------------------------------------
CHANGELOG.md | 7 +++
DEVELOPER.md | 4 ++
LICENSE | 14 ++++-
README.markdown | 1 +
bin/storm.cmd | 11 +++-
external/storm-kafka/README.md | 7 +--
.../src/jvm/storm/kafka/KafkaUtils.java | 63 ++++++++++----------
.../src/jvm/storm/kafka/PartitionManager.java | 11 +++-
.../jvm/storm/kafka/UpdateOffsetException.java | 5 ++
.../src/test/storm/kafka/KafkaUtilsTest.java | 6 +-
logback/cluster.xml | 4 +-
storm-core/pom.xml | 4 ++
.../clj/backtype/storm/daemon/supervisor.clj | 4 ++
storm-core/src/clj/backtype/storm/testing.clj | 4 +-
storm-core/src/clj/backtype/storm/testing4j.clj | 3 +
storm-core/src/dev/resources/storm.py | 8 ++-
.../backtype/storm/messaging/netty/Client.java | 2 +
.../backtype/storm/messaging/netty/Server.java | 14 ++---
.../jvm/backtype/storm/utils/LocalState.java | 29 ++++++---
storm-core/src/multilang/py/storm.py | 8 ++-
.../clj/backtype/storm/local_state_test.clj | 14 ++++-
.../storm/messaging/netty_unit_test.clj | 28 ++++-----
.../test/clj/backtype/storm/supervisor_test.clj | 4 ++
storm-dist/binary/LICENSE | 15 ++++-
24 files changed, 188 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e6b18863/README.markdown
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e6b18863/storm-core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e6b18863/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index c580bc8,b370eb7..6f7995a
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@@ -605,8 -501,9 +605,10 @@@
(defmethod launch-worker
:distributed [supervisor storm-id port worker-id]
(let [conf (:conf supervisor)
+ run-worker-as-user (conf SUPERVISOR-RUN-WORKER-AS-USER)
storm-home (System/getProperty "storm.home")
+ storm-options (System/getProperty "storm.options")
+ storm-conf-file (System/getProperty "storm.conf.file")
storm-log-dir (or (System/getProperty "storm.log.dir") (str storm-home file-path-separator "logs"))
stormroot (supervisor-stormdist-root conf storm-id)
jlp (jlp stormroot conf)
@@@ -637,8 -530,10 +639,10 @@@
[(str "-Djava.library.path=" jlp)
(str "-Dlogfile.name=" logfilename)
(str "-Dstorm.home=" storm-home)
+ (str "-Dstorm.conf.file=" storm-conf-file)
+ (str "-Dstorm.options=" storm-options)
(str "-Dstorm.log.dir=" storm-log-dir)
- (str "-Dlogback.configurationFile=" storm-home file-path-separator "logback" file-path-separator "cluster.xml")
+ (str "-Dlogback.configurationFile=" storm-home file-path-separator "logback" file-path-separator "worker.xml")
(str "-Dstorm.id=" storm-id)
(str "-Dworker.id=" worker-id)
(str "-Dworker.port=" port)
http://git-wip-us.apache.org/repos/asf/storm/blob/e6b18863/storm-core/src/clj/backtype/storm/testing.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e6b18863/storm-core/src/clj/backtype/storm/testing4j.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e6b18863/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e6b18863/storm-core/src/jvm/backtype/storm/messaging/netty/Server.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e6b18863/storm-core/src/jvm/backtype/storm/utils/LocalState.java
----------------------------------------------------------------------
diff --cc storm-core/src/jvm/backtype/storm/utils/LocalState.java
index f412ff3,14a45da..dc64e0f
--- a/storm-core/src/jvm/backtype/storm/utils/LocalState.java
+++ b/storm-core/src/jvm/backtype/storm/utils/LocalState.java
@@@ -35,10 -37,9 +38,10 @@@ public class LocalState
private VersionedStore _vs;
public LocalState(String backingDir) throws IOException {
+ LOG.debug("New Local State for {}", backingDir);
_vs = new VersionedStore(backingDir);
}
-
+
public synchronized Map<Object, Object> snapshot() throws IOException {
int attempts = 0;
while(true) {
http://git-wip-us.apache.org/repos/asf/storm/blob/e6b18863/storm-core/test/clj/backtype/storm/messaging/netty_unit_test.clj
----------------------------------------------------------------------
diff --cc storm-core/test/clj/backtype/storm/messaging/netty_unit_test.clj
index 8aaa7e5,ed5797d..2061ddf
--- a/storm-core/test/clj/backtype/storm/messaging/netty_unit_test.clj
+++ b/storm-core/test/clj/backtype/storm/messaging/netty_unit_test.clj
@@@ -26,10 -26,9 +26,10 @@@
(deftest test-basic
(let [req_msg (String. "0123456789abcdefghijklmnopqrstuvwxyz")
storm-conf {STORM-MESSAGING-TRANSPORT "backtype.storm.messaging.netty.Context"
+ STORM-MESSAGING-NETTY-AUTHENTICATION false
STORM-MESSAGING-NETTY-BUFFER-SIZE 1024
STORM-MESSAGING-NETTY-MAX-RETRIES 10
- STORM-MESSAGING-NETTY-MIN-SLEEP-MS 1000
+ STORM-MESSAGING-NETTY-MIN-SLEEP-MS 1000
STORM-MESSAGING-NETTY-MAX-SLEEP-MS 5000
STORM-MESSAGING-NETTY-SERVER-WORKER-THREADS 1
STORM-MESSAGING-NETTY-CLIENT-WORKER-THREADS 1
@@@ -44,15 -43,14 +44,15 @@@
(is (= req_msg (String. (.message resp))))
(.close client)
(.close server)
- (.term context)))
+ (.term context)))
(deftest test-large-msg
- (let [req_msg (apply str (repeat 2048000 'c'))
+ (let [req_msg (apply str (repeat 2048000 'c'))
storm-conf {STORM-MESSAGING-TRANSPORT "backtype.storm.messaging.netty.Context"
+ STORM-MESSAGING-NETTY-AUTHENTICATION false
STORM-MESSAGING-NETTY-BUFFER-SIZE 102400
STORM-MESSAGING-NETTY-MAX-RETRIES 10
- STORM-MESSAGING-NETTY-MIN-SLEEP-MS 1000
+ STORM-MESSAGING-NETTY-MIN-SLEEP-MS 1000
STORM-MESSAGING-NETTY-MAX-SLEEP-MS 5000
STORM-MESSAGING-NETTY-SERVER-WORKER-THREADS 1
STORM-MESSAGING-NETTY-CLIENT-WORKER-THREADS 1
@@@ -72,10 -70,9 +72,10 @@@
(deftest test-server-delayed
(let [req_msg (String. "0123456789abcdefghijklmnopqrstuvwxyz")
storm-conf {STORM-MESSAGING-TRANSPORT "backtype.storm.messaging.netty.Context"
+ STORM-MESSAGING-NETTY-AUTHENTICATION false
STORM-MESSAGING-NETTY-BUFFER-SIZE 1024
STORM-MESSAGING-NETTY-MAX-RETRIES 10
- STORM-MESSAGING-NETTY-MIN-SLEEP-MS 1000
+ STORM-MESSAGING-NETTY-MIN-SLEEP-MS 1000
STORM-MESSAGING-NETTY-MAX-SLEEP-MS 5000
STORM-MESSAGING-NETTY-SERVER-WORKER-THREADS 1
STORM-MESSAGING-NETTY-CLIENT-WORKER-THREADS 1
@@@ -102,10 -99,9 +102,10 @@@
(deftest test-batch
(let [storm-conf {STORM-MESSAGING-TRANSPORT "backtype.storm.messaging.netty.Context"
+ STORM-MESSAGING-NETTY-AUTHENTICATION false
STORM-MESSAGING-NETTY-BUFFER-SIZE 1024000
STORM-MESSAGING-NETTY-MAX-RETRIES 10
- STORM-MESSAGING-NETTY-MIN-SLEEP-MS 1000
+ STORM-MESSAGING-NETTY-MIN-SLEEP-MS 1000
STORM-MESSAGING-NETTY-MAX-SLEEP-MS 5000
STORM-MESSAGING-NETTY-SERVER-WORKER-THREADS 1
STORM-MESSAGING-NETTY-CLIENT-WORKER-THREADS 1
http://git-wip-us.apache.org/repos/asf/storm/blob/e6b18863/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --cc storm-core/test/clj/backtype/storm/supervisor_test.clj
index ebbf060,0bb47f3..a3594a3
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@@ -259,10 -253,12 +259,12 @@@
opts
topo-opts
["-Djava.library.path="
- (str "-Dlogfile.name=worker-" mock-port ".log")
+ (str "-Dlogfile.name=" mock-storm-id "-worker-" mock-port ".log")
"-Dstorm.home="
+ "-Dstorm.conf.file="
+ "-Dstorm.options="
(str "-Dstorm.log.dir=" file-path-separator "logs")
- (str "-Dlogback.configurationFile=" file-path-separator "logback" file-path-separator "cluster.xml")
+ (str "-Dlogback.configurationFile=" file-path-separator "logback" file-path-separator "worker.xml")
(str "-Dstorm.id=" mock-storm-id)
(str "-Dworker.id=" mock-worker-id)
(str "-Dworker.port=" mock-port)
@@@ -351,101 -338,7 +353,103 @@@
mock-worker-id)
(verify-first-call-args-for-indices launch-process
[2]
- (merge topo-env {"LD_LIBRARY_PATH" nil}))))))))
+ full-env)))))))
+
+(defn rm-r [f]
+ (if (.isDirectory f)
+ (for [sub (.listFiles f)] (rm-r sub))
+ (.delete f)
+ ))
+
+(deftest test-worker-launch-command-run-as-user
+ (testing "*.worker.childopts configuration"
+ (let [mock-port "42"
+ mock-storm-id "fake-storm-id"
+ mock-worker-id "fake-worker-id"
+ mock-cp "mock-classpath'quote-on-purpose"
+ storm-local (str "/tmp/" (UUID/randomUUID))
+ worker-script (str storm-local "/workers/" mock-worker-id "/storm-worker-script.sh")
+ exp-launch ["/bin/worker-launcher"
+ "me"
+ "worker"
+ (str storm-local "/workers/" mock-worker-id)
+ worker-script]
+ exp-script-fn (fn [opts topo-opts]
+ (str "#!/bin/bash\n'export' 'LD_LIBRARY_PATH=';\n\nexec 'java' '-server'"
+ " " (shell-cmd opts)
+ " " (shell-cmd topo-opts)
+ " '-Djava.library.path='"
+ " '-Dlogfile.name=" mock-storm-id "-worker-" mock-port ".log'"
+ " '-Dstorm.home='"
++ " '-Dstorm.conf.file='"
++ " '-Dstorm.options='"
+ " '-Dstorm.log.dir=/logs'"
+ " '-Dlogback.configurationFile=/logback/worker.xml'"
+ " '-Dstorm.id=" mock-storm-id "'"
+ " '-Dworker.id=" mock-worker-id "'"
+ " '-Dworker.port=" mock-port "'"
+ " '-cp' 'mock-classpath'\"'\"'quote-on-purpose'"
+ " 'backtype.storm.daemon.worker'"
+ " '" mock-storm-id "'"
+ " '" mock-port "'"
+ " '" mock-worker-id "';"))]
+ (.mkdirs (io/file storm-local "workers" mock-worker-id))
+ (try
+ (testing "testing *.worker.childopts as strings with extra spaces"
+ (let [string-opts "-Dfoo=bar -Xmx1024m"
+ topo-string-opts "-Dkau=aux -Xmx2048m"
+ exp-script (exp-script-fn ["-Dfoo=bar" "-Xmx1024m"]
+ ["-Dkau=aux" "-Xmx2048m"])
+ mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed
+ STORM-LOCAL-DIR storm-local
+ SUPERVISOR-RUN-WORKER-AS-USER true
+ WORKER-CHILDOPTS string-opts}}]
+ (stubbing [read-supervisor-storm-conf {TOPOLOGY-WORKER-CHILDOPTS
+ topo-string-opts
+ TOPOLOGY-SUBMITTER-USER "me"}
+ add-to-classpath mock-cp
+ supervisor-stormdist-root nil
+ launch-process nil
+ set-worker-user! nil
+ supervisor/java-cmd "java"
+ supervisor/jlp nil
+ supervisor/write-log-metadata! nil]
+ (supervisor/launch-worker mock-supervisor
+ mock-storm-id
+ mock-port
+ mock-worker-id)
+ (verify-first-call-args-for-indices launch-process
+ [0]
+ exp-launch))
+ (is (= (slurp worker-script) exp-script))))
+ (testing "testing *.worker.childopts as list of strings, with spaces in values"
+ (let [list-opts '("-Dopt1='this has a space in it'" "-Xmx1024m")
+ topo-list-opts '("-Dopt2='val with spaces'" "-Xmx2048m")
+ exp-script (exp-script-fn list-opts topo-list-opts)
+ mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed
+ STORM-LOCAL-DIR storm-local
+ SUPERVISOR-RUN-WORKER-AS-USER true
+ WORKER-CHILDOPTS list-opts}}]
+ (stubbing [read-supervisor-storm-conf {TOPOLOGY-WORKER-CHILDOPTS
+ topo-list-opts
+ TOPOLOGY-SUBMITTER-USER "me"}
+ add-to-classpath mock-cp
+ supervisor-stormdist-root nil
+ launch-process nil
+ set-worker-user! nil
+ supervisor/java-cmd "java"
+ supervisor/jlp nil
+ supervisor/write-log-metadata! nil]
+ (supervisor/launch-worker mock-supervisor
+ mock-storm-id
+ mock-port
+ mock-worker-id)
+ (verify-first-call-args-for-indices launch-process
+ [0]
+ exp-launch))
+ (is (= (slurp worker-script) exp-script))))
+(finally (rm-r (io/file storm-local)))
+))))
(deftest test-workers-go-bananas
;; test that multiple workers are started for a port, and test that
[13/50] storm git commit: STORM-349. excluded dependencies from
hadoop-auth. removed Config.UI_ACTIONS_ENABLED.
Posted by bo...@apache.org.
STORM-349. excluded dependencies from hadoop-auth. removed Config.UI_ACTIONS_ENABLED.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b73b051e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b73b051e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b73b051e
Branch: refs/heads/master
Commit: b73b051e683f1f27d299b15d79e99ce74bbd6054
Parents: 3208a8e
Author: Sriharsha Chintalapani <ma...@harsha.io>
Authored: Thu Aug 14 13:42:40 2014 -0700
Committer: Sriharsha Chintalapani <ma...@harsha.io>
Committed: Thu Aug 14 13:42:40 2014 -0700
----------------------------------------------------------------------
storm-core/pom.xml | 64 +++++++++++++++++++---
storm-core/src/jvm/backtype/storm/Config.java | 33 ++++-------
2 files changed, 68 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b73b051e/storm-core/pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/pom.xml b/storm-core/pom.xml
index 3ac8f23..f2d4ee3 100644
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@ -205,14 +205,62 @@
<artifactId>hadoop-auth</artifactId>
<version>2.4.0</version>
<exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
+ <exclusion>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minikdc</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.directory.server</groupId>
+ <artifactId>apacheds-kerberos-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/storm/blob/b73b051e/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index 94d1cb3..5b3da15 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -59,39 +59,39 @@ public class Config extends HashMap<String, Object> {
/**
* Netty based messaging: The buffer size for send/recv buffer
*/
- public static final String STORM_MESSAGING_NETTY_BUFFER_SIZE = "storm.messaging.netty.buffer_size";
+ public static final String STORM_MESSAGING_NETTY_BUFFER_SIZE = "storm.messaging.netty.buffer_size";
public static final Object STORM_MESSAGING_NETTY_BUFFER_SIZE_SCHEMA = ConfigValidation.IntegerValidator;
/**
* Netty based messaging: The max # of retries that a peer will perform when a remote is not accessible
*/
- public static final String STORM_MESSAGING_NETTY_MAX_RETRIES = "storm.messaging.netty.max_retries";
+ public static final String STORM_MESSAGING_NETTY_MAX_RETRIES = "storm.messaging.netty.max_retries";
public static final Object STORM_MESSAGING_NETTY_MAX_RETRIES_SCHEMA = ConfigValidation.IntegerValidator;
/**
* Netty based messaging: The min # of milliseconds that a peer will wait.
*/
- public static final String STORM_MESSAGING_NETTY_MIN_SLEEP_MS = "storm.messaging.netty.min_wait_ms";
+ public static final String STORM_MESSAGING_NETTY_MIN_SLEEP_MS = "storm.messaging.netty.min_wait_ms";
public static final Object STORM_MESSAGING_NETTY_MIN_SLEEP_MS_SCHEMA = ConfigValidation.IntegerValidator;
/**
* Netty based messaging: The max # of milliseconds that a peer will wait.
*/
- public static final String STORM_MESSAGING_NETTY_MAX_SLEEP_MS = "storm.messaging.netty.max_wait_ms";
+ public static final String STORM_MESSAGING_NETTY_MAX_SLEEP_MS = "storm.messaging.netty.max_wait_ms";
public static final Object STORM_MESSAGING_NETTY_MAX_SLEEP_MS_SCHEMA = ConfigValidation.IntegerValidator;
/**
* Netty based messaging: The # of worker threads for the server.
*/
- public static final String STORM_MESSAGING_NETTY_SERVER_WORKER_THREADS = "storm.messaging.netty.server_worker_threads";
+ public static final String STORM_MESSAGING_NETTY_SERVER_WORKER_THREADS = "storm.messaging.netty.server_worker_threads";
public static final Object STORM_MESSAGING_NETTY_SERVER_WORKER_THREADS_SCHEMA = ConfigValidation.IntegerValidator;
/**
* Netty based messaging: The # of worker threads for the client.
*/
- public static final String STORM_MESSAGING_NETTY_CLIENT_WORKER_THREADS = "storm.messaging.netty.client_worker_threads";
+ public static final String STORM_MESSAGING_NETTY_CLIENT_WORKER_THREADS = "storm.messaging.netty.client_worker_threads";
public static final Object STORM_MESSAGING_NETTY_CLIENT_WORKER_THREADS_SCHEMA = ConfigValidation.IntegerValidator;
-
+
/**
* If the Netty messaging layer is busy, the Netty client will try to batch message as more as possible up to the size of STORM_NETTY_MESSAGE_BATCH_SIZE bytes
*/
@@ -107,9 +107,9 @@ public class Config extends HashMap<String, Object> {
/**
* Netty based messaging: Is authentication required for Netty messaging from client worker process to server worker process.
*/
- public static final String STORM_MESSAGING_NETTY_AUTHENTICATION = "storm.messaging.netty.authentication";
+ public static final String STORM_MESSAGING_NETTY_AUTHENTICATION = "storm.messaging.netty.authentication";
public static final Object STORM_MESSAGING_NETTY_AUTHENTICATION_SCHEMA = Boolean.class;
-
+
/**
* A list of hosts of ZooKeeper servers used to manage the cluster.
*/
@@ -514,15 +514,6 @@ public class Config extends HashMap<String, Object> {
public static final Object UI_USERS_SCHEMA = ConfigValidation.StringsValidator;
/**
- * Whether or not actions should be enabled. When disabled, requests to
- * modify the state of topologies via HTTP will not be honored.
- *
- * Defaults to true.
- */
- public static final String UI_ACTIONS_ENABLED = "ui.actions.enabled";
- public static final Object UI_ACTIONS_ENABLED_SCHEMA = Boolean.class;
-
- /**
* List of DRPC servers so that the DRPCSpout knows who to talk to.
*/
public static final String DRPC_SERVERS = "drpc.servers";
@@ -782,7 +773,7 @@ public class Config extends HashMap<String, Object> {
*/
public static final String WORKER_RECEIVER_THREAD_COUNT = "topology.worker.receiver.thread.count";
public static final Object WORKER_RECEIVER_THREAD_COUNT_SCHEMA = ConfigValidation.IntegerValidator;
-
+
/**
* How often this worker should heartbeat to the supervisor.
*/
@@ -947,7 +938,7 @@ public class Config extends HashMap<String, Object> {
* Note that this config parameter has no effect for unreliable spouts that don't tag
* their tuples with a message id.
*/
- public static final String TOPOLOGY_MAX_SPOUT_PENDING="topology.max.spout.pending";
+ public static final String TOPOLOGY_MAX_SPOUT_PENDING="topology.max.spout.pending";
public static final Object TOPOLOGY_MAX_SPOUT_PENDING_SCHEMA = ConfigValidation.IntegerValidator;
/**
@@ -1010,7 +1001,7 @@ public class Config extends HashMap<String, Object> {
public static final Object TOPOLOGY_CLASSPATH_SCHEMA = ConfigValidation.StringOrStringListValidator;
/**
- * Topology-specific environment variables for the worker child process.
+ * Topology-specific environment variables for the worker child process.
* This is added to the existing environment (that of the supervisor)
*/
public static final String TOPOLOGY_ENVIRONMENT="topology.environment";
[18/50] storm git commit: Merge branch 'master' into security-upemerge
Posted by bo...@apache.org.
Merge branch 'master' into security-upemerge
Conflicts:
README.markdown
storm-core/src/ui/public/templates/index-page-template.html
storm-core/src/ui/public/templates/topology-page-template.html
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d20c101f
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d20c101f
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d20c101f
Branch: refs/heads/master
Commit: d20c101fa73c413b8d9fe1065a0291125e18bc96
Parents: a55d792 7a58c2d
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Mon Aug 18 14:25:07 2014 -0500
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Mon Aug 18 14:25:07 2014 -0500
----------------------------------------------------------------------
CHANGELOG.md | 12 +
README.markdown | 3 +
bin/storm | 6 +-
external/storm-hdfs/README.md | 343 +++++++++++++++++++
external/storm-hdfs/pom.xml | 68 ++++
.../storm/hdfs/bolt/AbstractHdfsBolt.java | 139 ++++++++
.../org/apache/storm/hdfs/bolt/HdfsBolt.java | 131 +++++++
.../storm/hdfs/bolt/SequenceFileBolt.java | 149 ++++++++
.../hdfs/bolt/format/DefaultFileNameFormat.java | 86 +++++
.../hdfs/bolt/format/DefaultSequenceFormat.java | 52 +++
.../hdfs/bolt/format/DelimitedRecordFormat.java | 88 +++++
.../storm/hdfs/bolt/format/FileNameFormat.java | 42 +++
.../storm/hdfs/bolt/format/RecordFormat.java | 32 ++
.../storm/hdfs/bolt/format/SequenceFormat.java | 40 +++
.../hdfs/bolt/rotation/FileRotationPolicy.java | 51 +++
.../bolt/rotation/FileSizeRotationPolicy.java | 81 +++++
.../hdfs/bolt/rotation/NoRotationPolicy.java | 35 ++
.../hdfs/bolt/rotation/TimedRotationPolicy.java | 55 +++
.../storm/hdfs/bolt/sync/CountSyncPolicy.java | 46 +++
.../apache/storm/hdfs/bolt/sync/SyncPolicy.java | 46 +++
.../hdfs/common/rotation/MoveFileAction.java | 27 ++
.../hdfs/common/rotation/RotationAction.java | 12 +
.../hdfs/common/security/HdfsSecurityUtil.java | 48 +++
.../apache/storm/hdfs/trident/HdfsState.java | 317 +++++++++++++++++
.../storm/hdfs/trident/HdfsStateFactory.java | 29 ++
.../apache/storm/hdfs/trident/HdfsUpdater.java | 14 +
.../trident/format/DefaultFileNameFormat.java | 83 +++++
.../trident/format/DefaultSequenceFormat.java | 54 +++
.../trident/format/DelimitedRecordFormat.java | 83 +++++
.../hdfs/trident/format/FileNameFormat.java | 40 +++
.../storm/hdfs/trident/format/RecordFormat.java | 32 ++
.../hdfs/trident/format/SequenceFormat.java | 40 +++
.../trident/rotation/FileRotationPolicy.java | 50 +++
.../rotation/FileSizeRotationPolicy.java | 81 +++++
.../hdfs/trident/rotation/NoRotationPolicy.java | 35 ++
.../trident/rotation/TimedRotationPolicy.java | 54 +++
.../hdfs/trident/sync/CountSyncPolicy.java | 46 +++
.../storm/hdfs/trident/sync/SyncPolicy.java | 46 +++
.../storm/hdfs/bolt/HdfsFileTopology.java | 198 +++++++++++
.../storm/hdfs/bolt/SequenceFileTopology.java | 193 +++++++++++
.../storm/hdfs/trident/FixedBatchSpout.java | 80 +++++
.../storm/hdfs/trident/TridentFileTopology.java | 73 ++++
.../hdfs/trident/TridentSequenceTopology.java | 71 ++++
external/storm-kafka/README.md | 197 ++++++++++-
.../jvm/storm/kafka/DynamicBrokersReader.java | 5 +
.../src/jvm/storm/kafka/bolt/KafkaBolt.java | 49 ++-
.../FieldNameBasedTupleToKafkaMapper.java | 48 +++
.../kafka/bolt/mapper/TupleToKafkaMapper.java | 32 ++
.../bolt/selector/DefaultTopicSelector.java | 34 ++
.../kafka/bolt/selector/KafkaTopicSelector.java | 26 ++
.../storm/kafka/trident/TridentKafkaState.java | 99 ++++++
.../kafka/trident/TridentKafkaStateFactory.java | 57 +++
.../kafka/trident/TridentKafkaUpdater.java | 31 ++
.../FieldNameBasedTupleToKafkaMapper.java | 41 +++
.../mapper/TridentTupleToKafkaMapper.java | 28 ++
.../trident/selector/DefaultTopicSelector.java | 34 ++
.../trident/selector/KafkaTopicSelector.java | 26 ++
.../src/test/storm/kafka/TestUtils.java | 62 ++++
.../src/test/storm/kafka/TridentKafkaTest.java | 89 +++++
.../test/storm/kafka/TridentKafkaTopology.java | 100 ++++++
logback/cluster.xml | 12 +-
pom.xml | 5 +-
storm-core/pom.xml | 2 +-
.../clj/backtype/storm/daemon/supervisor.clj | 9 +-
storm-core/src/clj/backtype/storm/disruptor.clj | 11 +-
storm-core/src/clj/backtype/storm/util.clj | 2 +-
.../jvm/backtype/storm/multilang/ShellMsg.java | 17 +
.../jvm/backtype/storm/spout/ShellSpout.java | 3 +
.../src/jvm/backtype/storm/task/ShellBolt.java | 6 +
.../backtype/storm/utils/DisruptorQueue.java | 56 ++-
storm-core/src/ui/public/css/style.css | 67 ++++
.../public/templates/index-page-template.html | 2 +-
.../templates/topology-page-template.html | 2 +-
.../test/clj/backtype/storm/supervisor_test.clj | 1 +
.../storm/utils/DisruptorQueueTest.java | 25 +-
storm-dist/binary/src/main/assembly/binary.xml | 14 +
76 files changed, 4327 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/README.markdown
----------------------------------------------------------------------
diff --cc README.markdown
index 586f171,d5c3b23..27f897b
--- a/README.markdown
+++ b/README.markdown
@@@ -159,8 -159,11 +159,11 @@@ under the License
* Jo Liss ([@joliss](https://github.com/joliss))
* averykhoo ([@averykhoo](https://github.com/averykhoo))
* Curtis Allen ([@curtisallen](https://github.com/curtisallen))
+* Raghavendra Nandagopal ([@RaghavendraNandagopal](https://github.com/RaghavendraNandagopal))
+ * Boris Aksenov ([@miofthena](https://github.com/miofthena))
+ * Itai Frenkel ([@itaifrenkel](https://github.com/itaifrenkel))
+ * Naresh Kosgi ([@NareshKosgi](https://github.com/NareshKosgi))
-
## Acknowledgements
YourKit is kindly supporting open source projects with its full-featured Java Profiler. YourKit, LLC is the creator of innovative and intelligent tools for profiling Java and .NET applications. Take a look at YourKit's leading software products: [YourKit Java Profiler](http://www.yourkit.com/java/profiler/index.jsp) and [YourKit .NET Profiler](http://www.yourkit.com/.net/profiler/index.jsp).
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/bin/storm
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index d8ff6b5,6473ed1..4d83497
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@@ -14,9 -14,9 +14,10 @@@
;; See the License for the specific language governing permissions and
;; limitations under the License.
(ns backtype.storm.daemon.supervisor
+ (:import [java.io OutputStreamWriter BufferedWriter IOException])
(:import [backtype.storm.scheduler ISupervisor]
- [java.net JarURLConnection])
+ [java.net JarURLConnection]
+ [java.net URI])
(:use [backtype.storm bootstrap])
(:use [backtype.storm.daemon common])
(:require [backtype.storm.daemon [worker :as worker]])
@@@ -606,8 -491,8 +607,9 @@@
(defmethod launch-worker
:distributed [supervisor storm-id port worker-id]
(let [conf (:conf supervisor)
+ run-worker-as-user (conf SUPERVISOR-RUN-WORKER-AS-USER)
storm-home (System/getProperty "storm.home")
+ storm-log-dir (or (System/getProperty "storm.log.dir") (str storm-home "/logs"))
stormroot (supervisor-stormdist-root conf storm-id)
jlp (jlp stormroot conf)
stormjar (supervisor-stormjar-path stormroot)
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/src/clj/backtype/storm/util.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/src/ui/public/css/style.css
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/src/ui/public/templates/index-page-template.html
----------------------------------------------------------------------
diff --cc storm-core/src/ui/public/templates/index-page-template.html
index bd78c5c,45906e5..8f1c29b
--- a/storm-core/src/ui/public/templates/index-page-template.html
+++ b/storm-core/src/ui/public/templates/index-page-template.html
@@@ -48,13 -38,11 +48,13 @@@
<tr>
<td><a href="/topology.html?id={{id}}">{{name}}</a></td>
<td>{{id}}</td>
+ <td>{{owner}}</td>
<td>{{status}}</td>
<td>{{uptime}}</td>
- <td>{{tasksTotal}}</td>
<td>{{workersTotal}}</td>
<td>{{executorsTotal}}</td>
+ <td>{{schedulerInfo}}</td>
+ <td>{{tasksTotal}}</td>
</tr>
{{/topologies}}
</tbody>
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/src/ui/public/templates/topology-page-template.html
----------------------------------------------------------------------
diff --cc storm-core/src/ui/public/templates/topology-page-template.html
index c249053,2b93ed6..ddf3c17
--- a/storm-core/src/ui/public/templates/topology-page-template.html
+++ b/storm-core/src/ui/public/templates/topology-page-template.html
@@@ -31,13 -21,11 +31,13 @@@
<tr>
<td>{{name}}</td>
<td>{{id}}</td>
+ <td>{{owner}}</td>
<td>{{status}}</td>
<td>{{uptime}}</td>
- <td>{{tasksTotal}}</td>
<td>{{workersTotal}}</td>
<td>{{executorsTotal}}</td>
+ <td>{{schedulerInfo}}</td>
+ <td>{{tasksTotal}}</td>
</tr>
</tbody>
</table>
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --cc storm-core/test/clj/backtype/storm/supervisor_test.clj
index 6b40060,f6fc285..3eb5de6
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@@ -259,8 -253,9 +259,9 @@@
opts
topo-opts
["-Djava.library.path="
- (str "-Dlogfile.name=worker-" mock-port ".log")
+ (str "-Dlogfile.name=" mock-storm-id "-worker-" mock-port ".log")
"-Dstorm.home="
+ "-Dstorm.log.dir=/logs"
"-Dlogback.configurationFile=/logback/cluster.xml"
(str "-Dstorm.id=" mock-storm-id)
(str "-Dworker.id=" mock-worker-id)
[48/50] storm git commit: Merge branch 'STORM-509' of
https://github.com/harshach/incubator-storm into STORM-509
Posted by bo...@apache.org.
Merge branch 'STORM-509' of https://github.com/harshach/incubator-storm into STORM-509
STORM-509. Make groups checking specific for SimpleACLAuthorizer.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/04a7937d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/04a7937d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/04a7937d
Branch: refs/heads/master
Commit: 04a7937dd5d031f15ae35b37d5ab031fd39130a5
Parents: ea0b8a2 0f6719d
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Mon Oct 27 14:52:32 2014 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Mon Oct 27 14:52:32 2014 -0500
----------------------------------------------------------------------
storm-core/src/jvm/backtype/storm/Config.java | 13 ++++++++++---
.../auth/authorizer/SimpleACLAuthorizer.java | 19 ++++++++++++-------
.../backtype/storm/security/auth/auth_test.clj | 4 ++--
3 files changed, 24 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
[17/50] storm git commit: Merge branch 'STORM-349-V2' of
https://github.com/harshach/incubator-storm into STORM-349
Posted by bo...@apache.org.
Merge branch 'STORM-349-V2' of https://github.com/harshach/incubator-storm into STORM-349
STORM-349: ui actions should have nimbus like authroization.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a55d7922
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a55d7922
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a55d7922
Branch: refs/heads/master
Commit: a55d7922072a0407aa8ae48129cfb31dca7c7338
Parents: 1a1df08 b73b051
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Mon Aug 18 14:04:48 2014 -0500
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Mon Aug 18 14:04:48 2014 -0500
----------------------------------------------------------------------
SECURITY.md | 25 ++++-
pom.xml | 8 +-
storm-core/pom.xml | 69 +++++++++++-
storm-core/src/clj/backtype/storm/ui/core.clj | 105 +++++++++----------
.../src/clj/backtype/storm/ui/helpers.clj | 15 +--
storm-core/src/jvm/backtype/storm/Config.java | 33 +++---
storm-core/test/clj/backtype/storm/ui_test.clj | 49 ---------
7 files changed, 163 insertions(+), 141 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/a55d7922/SECURITY.md
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/a55d7922/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
[04/50] storm git commit: STORM-427: AutoTGT and HBase can expose JVM
kerberos bug.
Posted by bo...@apache.org.
STORM-427: AutoTGT and HBase can expose JVM kerberos bug.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ee5bb179
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ee5bb179
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ee5bb179
Branch: refs/heads/master
Commit: ee5bb1792afc28155bd8cdf87e97ebf719c031d1
Parents: 559c883
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Jul 29 16:47:19 2014 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Tue Jul 29 16:47:19 2014 -0500
----------------------------------------------------------------------
.../storm/security/auth/kerberos/AutoTGT.java | 22 ++++++++++++++++----
1 file changed, 18 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ee5bb179/storm-core/src/jvm/backtype/storm/security/auth/kerberos/AutoTGT.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/security/auth/kerberos/AutoTGT.java b/storm-core/src/jvm/backtype/storm/security/auth/kerberos/AutoTGT.java
index 52bf540..1e07daa 100644
--- a/storm-core/src/jvm/backtype/storm/security/auth/kerberos/AutoTGT.java
+++ b/storm-core/src/jvm/backtype/storm/security/auth/kerberos/AutoTGT.java
@@ -33,12 +33,14 @@ import java.lang.reflect.Method;
import java.lang.reflect.Constructor;
import java.security.Principal;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.Iterator;
import javax.security.auth.kerberos.KerberosTicket;
import javax.security.auth.kerberos.KerberosPrincipal;
import javax.security.auth.login.Configuration;
import javax.security.auth.login.LoginException;
import javax.security.auth.login.LoginContext;
+import javax.security.auth.DestroyFailedException;
import javax.security.auth.RefreshFailedException;
import javax.security.auth.Subject;
import javax.xml.bind.DatatypeConverter;
@@ -152,10 +154,22 @@ public class AutoTGT implements IAutoCredentials, ICredentialsRenewer {
private void populateSubjectWithTGT(Subject subject, Map<String, String> credentials) {
KerberosTicket tgt = getTGT(credentials);
if (tgt != null) {
- KerberosTicket oldTGT = getTGT(subject);
- subject.getPrivateCredentials().add(tgt);
- if (oldTGT != null && !oldTGT.equals(tgt)) {
- subject.getPrivateCredentials().remove(oldTGT);
+ Set<Object> creds = subject.getPrivateCredentials();
+ synchronized(creds) {
+ Iterator<Object> iterator = creds.iterator();
+ while (iterator.hasNext()) {
+ Object o = iterator.next();
+ if (o instanceof KerberosTicket) {
+ KerberosTicket t = (KerberosTicket)o;
+ iterator.remove();
+ try {
+ t.destroy();
+ } catch (DestroyFailedException e) {
+ LOG.warn("Failed to destory ticket ", e);
+ }
+ }
+ }
+ creds.add(tgt);
}
subject.getPrincipals().add(tgt.getClient());
kerbTicket.set(tgt);
[11/50] storm git commit: STORM-349. (Security) ui actions should
have nimbus like authroization. Upgraded ring to 1.3.0. Added IAuthorizer
instance to UI daemon. Added hadoop-auth as a dependency which allow users to
configure AuthenticationFilter.
Posted by bo...@apache.org.
STORM-349. (Security) ui actions should have nimbus like authroization.
Upgraded ring to 1.3.0. Added IAuthorizer instance to UI daemon. Added
hadoop-auth as a dependency which allow users to configure
AuthenticationFilter.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3208a8e3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3208a8e3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3208a8e3
Branch: refs/heads/master
Commit: 3208a8e3f39140227fe6be9c686b795a19777e64
Parents: fa40fbc
Author: Sriharsha Chintalapani <ma...@harsha.io>
Authored: Mon Aug 4 19:18:26 2014 -0700
Committer: Sriharsha Chintalapani <ma...@harsha.io>
Committed: Mon Aug 4 19:18:26 2014 -0700
----------------------------------------------------------------------
SECURITY.md | 25 ++++-
pom.xml | 8 +-
storm-core/pom.xml | 21 +++-
storm-core/src/clj/backtype/storm/ui/core.clj | 105 +++++++++----------
.../src/clj/backtype/storm/ui/helpers.clj | 15 +--
storm-core/test/clj/backtype/storm/ui_test.clj | 49 ---------
6 files changed, 103 insertions(+), 120 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/3208a8e3/SECURITY.md
----------------------------------------------------------------------
diff --git a/SECURITY.md b/SECURITY.md
index 882954a..f919de7 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -54,7 +54,26 @@ must have it set to the actual port that they are going to bind to.
The servlet filters are prefered because it allows indavidual topologies to
specificy who is and who is not allowed to access the pages associated with
-them.
+them.
+
+Storm UI can be configured to use AuthenticationFilter from hadoop-auth.
+```yaml
+ui.filter: "org.apache.hadoop.security.authentication.server.AuthenticationFilter"
+ui.filter.params:
+ "type": "kerberos"
+ "kerberos.principal": "HTTP/nimbus.witzend.com"
+ "kerberos.keytab": "/vagrant/keytabs/http.keytab"
+ "kerberos.name.rules": "RULE:[2:$1@$0]([jt]t@.*EXAMPLE.COM)s/.*/$MAPRED_USER/ RULE:[2:$1@$0]([nd]n@.*EXAMPLE.COM)s/.*/$HDFS_USER/DEFAULT"
+```
+make sure to create a prinicpal 'HTTP/{hostname}' (here hostname should be the one where UI daemon runs
+
+Once configured users needs to do kinit before accessing UI.
+Ex:
+curl -i --negotiate -u:anyUser -b ~/cookiejar.txt -c ~/cookiejar.txt http://storm-ui-hostname:8080/api/v1/cluster/summary
+
+1) Firefox: Goto about:config and search for network.negotiate-auth.trusted-uris double-click to add value "http://storm-ui-hostname:8080"
+2) Google-chrome: start from command line with: google-chrome --auth-server-whitelist="*storm-ui-hostname" --auth-negotiate-delegate-whitelist="*storm-ui-hostname"
+3) IE: Configure trusted websites to include "storm-ui-hostname" and allow negotiation for that website
## Authentication (Kerberos)
@@ -335,7 +354,3 @@ The Logviewer deamon now is also responsible for cleaning up old log files for d
### DRPC
Hopefully more on this soon
-
-
-
-
http://git-wip-us.apache.org/repos/asf/storm/blob/3208a8e3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 87c52ea..04b6ef1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -187,7 +187,8 @@
<clj-time.version>0.4.1</clj-time.version>
<curator.version>2.4.0</curator.version>
<json-simple.version>1.1</json-simple.version>
- <ring.version>0.3.11</ring.version>
+ <ring.version>1.3.0</ring.version>
+ <jetty.version>7.6.13.v20130916</jetty.version>
<clojure.tools.logging.version>0.2.3</clojure.tools.logging.version>
<clojure.math.numeric-tower.version>0.0.1</clojure.math.numeric-tower.version>
<carbonite.version>1.4.0</carbonite.version>
@@ -390,6 +391,11 @@
<version>${ring.version}</version>
</dependency>
<dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.clojure</groupId>
<artifactId>tools.logging</artifactId>
<version>${clojure.tools.logging.version}</version>
http://git-wip-us.apache.org/repos/asf/storm/blob/3208a8e3/storm-core/pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/pom.xml b/storm-core/pom.xml
index c677b7c..3ac8f23 100644
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@ -60,6 +60,10 @@
<artifactId>ring-jetty-adapter</artifactId>
</dependency>
<dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.clojure</groupId>
<artifactId>tools.logging</artifactId>
</dependency>
@@ -96,7 +100,7 @@
<artifactId>data.codec</artifactId>
<scope>test</scope>
</dependency>
-
+
<!--java-->
<dependency>
<groupId>commons-io</groupId>
@@ -197,6 +201,21 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <version>2.4.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.1</version>
http://git-wip-us.apache.org/repos/asf/storm/blob/3208a8e3/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/ui/core.clj b/storm-core/src/clj/backtype/storm/ui/core.clj
index 8dd301e..0aea7ec 100644
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@ -20,8 +20,8 @@
(:use [hiccup core page-helpers])
(:use [backtype.storm config util log])
(:use [backtype.storm.ui helpers])
- (:use [backtype.storm.daemon [common :only [ACKER-COMPONENT-ID ACKER-INIT-STREAM-ID
- ACKER-ACK-STREAM-ID ACKER-FAIL-STREAM-ID system-id?]]])
+ (:use [backtype.storm.daemon [common :only [ACKER-COMPONENT-ID ACKER-INIT-STREAM-ID ACKER-ACK-STREAM-ID
+ ACKER-FAIL-STREAM-ID system-id? mk-authorization-handler]]])
(:use [ring.adapter.jetty :only [run-jetty]])
(:use [clojure.string :only [blank? lower-case trim]])
(:import [backtype.storm.utils Utils])
@@ -30,6 +30,8 @@
ErrorInfo ClusterSummary SupervisorSummary TopologySummary
Nimbus$Client StormTopology GlobalStreamId RebalanceOptions
KillOptions])
+ (:import [backtype.storm.security.auth AuthUtils ReqContext])
+ (:import [backtype.storm.generated AuthorizationException])
(:import [backtype.storm.security.auth AuthUtils])
(:import [java.io File])
(:require [compojure.route :as route]
@@ -40,6 +42,9 @@
(:gen-class))
(def ^:dynamic *STORM-CONF* (read-storm-config))
+(def ^:dynamic *UI-ACL-HANDLER* (mk-authorization-handler (*STORM-CONF* NIMBUS-AUTHORIZER) *STORM-CONF*))
+
+(def http-creds-handler (AuthUtils/GetUiHttpCredentialsPlugin *STORM-CONF*))
(defmacro with-nimbus
[nimbus-sym & body]
@@ -47,33 +52,19 @@
[~nimbus-sym (*STORM-CONF* NIMBUS-HOST) (*STORM-CONF* NIMBUS-THRIFT-PORT)]
~@body))
-(defn authorized-ui-user?
- [user conf topology-conf]
- (let [ui-users (concat (conf UI-USERS)
- (conf NIMBUS-ADMINS)
- (topology-conf UI-USERS)
- (topology-conf TOPOLOGY-USERS))]
- (or (blank? (conf UI-FILTER))
- (and (not (blank? user))
- (some #(= % user) ui-users)))))
-
-(defn assert-authorized-ui-user
- [user conf topology-conf]
- (if (not (authorized-ui-user? user conf topology-conf))
- ;;TODO need a better exception here so the UI can appear better
- (throw (RuntimeException. (str "User " user " is not authorized.")))))
-
-(defn- ui-actions-enabled?
- []
- (= "true" (lower-case (*STORM-CONF* UI-ACTIONS-ENABLED))))
-
-(defn assert-authorized-topology-user
- [user]
- ;;TODO eventually we will want to use the Authorizatin handler from nimbus, but for now
- ;; Disable the calls conditionally
- (if (not (ui-actions-enabled?))
- ;;TODO need a better exception here so the UI can appear better
- (throw (RuntimeException. (str "Topology actions for the UI have been disabled")))))
+(defn assert-authorized-user
+ ([servlet-request op]
+ (assert-authorized-user servlet-request op nil))
+ ([servlet-request op topology-conf]
+ (if http-creds-handler (.populateContext http-creds-handler (ReqContext/context) servlet-request))
+ (if *UI-ACL-HANDLER*
+ (let [context (ReqContext/context)]
+ (if-not (.permit *UI-ACL-HANDLER* context op topology-conf)
+ (let [principal (.principal context)
+ user (if principal (.getName principal) "unknown")]
+ (throw (AuthorizationException.
+ (str "UI request '" op "' for '"
+ user "' user is not authorized")))))))))
(defn get-filled-stats
[summs]
@@ -474,9 +465,9 @@
:checked (is-ack-stream (get m :stream))}))))))]
(map (fn [row]
{:row row}) (partition 4 4 nil streams))))
-
+
(defn mk-visualization-data
- [id window include-sys? user]
+ [id window include-sys?]
(with-nimbus
nimbus
(let [window (if window window ":all-time")
@@ -491,10 +482,9 @@
bolt-comp-summs (group-by-comp bolt-summs)
bolt-comp-summs (filter-key (mk-include-sys-fn include-sys?)
bolt-comp-summs)
- topology-conf (from-json
+ topology-conf (from-json
(.getTopologyConf ^Nimbus$Client nimbus id))]
- (assert-authorized-ui-user user *STORM-CONF* topology-conf)
- (visualization-data
+ (visualization-data
(merge (hashmap-to-persistent spouts)
(hashmap-to-persistent bolts))
spout-comp-summs bolt-comp-summs window id))))
@@ -675,7 +665,6 @@
bolt-comp-summs
window
id)]
- (assert-authorized-ui-user user *STORM-CONF* topology-conf)
(merge
(topology-summary summ)
{"user" user
@@ -686,8 +675,7 @@
"spouts" (spout-comp id spout-comp-summs (.get_errors summ) window include-sys?)
"bolts" (bolt-comp id bolt-comp-summs (.get_errors summ) window include-sys?)
"configuration" topology-conf
- "visualizationTable" (stream-boxes visualizer-data)
- "uiActionsEnabled" (ui-actions-enabled?)}))))
+ "visualizationTable" (stream-boxes visualizer-data)}))))
(defn spout-output-stats
[stream-summary window]
@@ -842,7 +830,6 @@
spec (cond (= type :spout) (spout-stats window summ component summs include-sys?)
(= type :bolt) (bolt-stats window summ component summs include-sys?))
errors (component-errors (get (.get_errors summ) component) topology-id)]
- (assert-authorized-ui-user user *STORM-CONF* topology-conf)
(merge
{"user" user
"id" component
@@ -855,6 +842,10 @@
"windowHint" (window-hint window)}
spec errors))))
+(defn topology-config [topology-id]
+ (with-nimbus nimbus
+ (from-json (.getTopologyConf ^Nimbus$Client nimbus topology-id))))
+
(defn check-include-sys?
[sys?]
(if (or (nil? sys?) (= "false" sys?)) false true))
@@ -871,42 +862,44 @@
(cluster-configuration))
(GET "/api/v1/cluster/summary" [:as {:keys [cookies servlet-request]}]
(let [user (.getUserName http-creds-handler servlet-request)]
+ (assert-authorized-user servlet-request "getClusterInfo")
(json-response (cluster-summary user))))
- (GET "/api/v1/supervisor/summary" []
+ (GET "/api/v1/supervisor/summary" [:as {:keys [cookies servlet-request]}]
+ (assert-authorized-user servlet-request "getClusterInfo")
(json-response (supervisor-summary)))
- (GET "/api/v1/topology/summary" []
+ (GET "/api/v1/topology/summary" [:as {:keys [cookies servlet-request]}]
+ (assert-authorized-user servlet-request "getClusterInfo")
(json-response (all-topologies-summary)))
(GET "/api/v1/topology/:id" [:as {:keys [cookies servlet-request]} id & m]
(let [id (url-decode id)
user (.getUserName http-creds-handler servlet-request)]
+ (assert-authorized-user servlet-request "getTopology" (topology-config id))
(json-response (topology-page id (:window m) (check-include-sys? (:sys m)) user))))
(GET "/api/v1/topology/:id/visualization" [:as {:keys [cookies servlet-request]} id & m]
- (let [id (url-decode id)
- user (.getUserName http-creds-handler servlet-request)]
- (json-response (mk-visualization-data id (:window m) (check-include-sys? (:sys m)) user))))
+ (let [id (url-decode id)]
+ (assert-authorized-user servlet-request "getTopology" (topology-config id))
+ (json-response (mk-visualization-data id (:window m) (check-include-sys? (:sys m))))))
(GET "/api/v1/topology/:id/component/:component" [:as {:keys [cookies servlet-request]} id component & m]
(let [id (url-decode id)
component (url-decode component)
user (.getUserName http-creds-handler servlet-request)]
+ (assert-authorized-user servlet-request "getTopology" (topology-config id))
(json-response (component-page id component (:window m) (check-include-sys? (:sys m)) user))))
(POST "/api/v1/topology/:id/activate" [:as {:keys [cookies servlet-request]} id]
(with-nimbus nimbus
(let [id (url-decode id)
tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
- name (.get_name tplg)
- user (.getUserName http-creds-handler servlet-request)]
- (assert-authorized-topology-user user)
+ name (.get_name tplg)]
+ (assert-authorized-user servlet-request "activate" (topology-config id))
(.activate nimbus name)
(log-message "Activating topology '" name "'")))
(resp/redirect (str "/api/v1/topology/" id)))
-
(POST "/api/v1/topology/:id/deactivate" [:as {:keys [cookies servlet-request]} id]
(with-nimbus nimbus
(let [id (url-decode id)
tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
- name (.get_name tplg)
- user (.getUserName http-creds-handler servlet-request)]
- (assert-authorized-topology-user user)
+ name (.get_name tplg)]
+ (assert-authorized-user servlet-request "deactivate" (topology-config id))
(.deactivate nimbus name)
(log-message "Deactivating topology '" name "'")))
(resp/redirect (str "/api/v1/topology/" id)))
@@ -915,9 +908,8 @@
(let [id (url-decode id)
tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
name (.get_name tplg)
- options (RebalanceOptions.)
- user (.getUserName http-creds-handler servlet-request)]
- (assert-authorized-topology-user user)
+ options (RebalanceOptions.)]
+ (assert-authorized-user servlet-request "rebalance" (topology-config id))
(.set_wait_secs options (Integer/parseInt wait-time))
(.rebalance nimbus name options)
(log-message "Rebalancing topology '" name "' with wait time: " wait-time " secs")))
@@ -927,9 +919,8 @@
(let [id (url-decode id)
tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
name (.get_name tplg)
- options (KillOptions.)
- user (.getUserName http-creds-handler servlet-request)]
- (assert-authorized-topology-user user)
+ options (KillOptions.)]
+ (assert-authorized-user servlet-request "killTopology" (topology-config id))
(.set_wait_secs options (Integer/parseInt wait-time))
(.killTopologyWithOpts nimbus name options)
(log-message "Killing topology '" name "' with wait time: " wait-time " secs")))
@@ -972,7 +963,7 @@
:join? false
:configurator (fn [server]
(doseq [connector (.getConnectors server)]
- (.setHeaderBufferSize connector header-buffer-size))
+ (.setRequestHeaderSize connector header-buffer-size))
(config-filter server app filters-confs))}))
(catch Exception ex
(log-error ex))))
http://git-wip-us.apache.org/repos/asf/storm/blob/3208a8e3/storm-core/src/clj/backtype/storm/ui/helpers.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/ui/helpers.clj b/storm-core/src/clj/backtype/storm/ui/helpers.clj
index 60e7bd2..8033c50 100644
--- a/storm-core/src/clj/backtype/storm/ui/helpers.clj
+++ b/storm-core/src/clj/backtype/storm/ui/helpers.clj
@@ -23,7 +23,8 @@
(:use [backtype.storm.util :only [clojurify-structure uuid defnk url-encode]])
(:use [clj-time coerce format])
(:import [backtype.storm.generated ExecutorInfo ExecutorSummary])
- (:import [org.mortbay.jetty.security SslSocketConnector])
+ (:import [org.eclipse.jetty.server.ssl SslSocketConnector]
+ [org.eclipse.jetty.servlet ServletHolder FilterMapping])
(:require [ring.util servlet])
(:require [compojure.route :as route]
[compojure.handler :as handler]))
@@ -129,7 +130,7 @@ $(\"table#%s\").each(function(i) { $(this).tablesorter({ sortList: %s, headers:
)))
(defn url-format [fmt & args]
- (String/format fmt
+ (String/format fmt
(to-array (map #(url-encode (str %)) args))))
(defn to-tasks [^ExecutorInfo e]
@@ -166,18 +167,18 @@ $(\"table#%s\").each(function(i) { $(this).tablesorter({ sortList: %s, headers:
(defn config-filter [server handler filters-confs]
(if filters-confs
- (let [servlet-holder (org.mortbay.jetty.servlet.ServletHolder.
+ (let [servlet-holder (ServletHolder.
(ring.util.servlet/servlet handler))
- context (doto (org.mortbay.jetty.servlet.Context. server "/")
+ context (doto (org.eclipse.jetty.servlet.ServletContextHandler. server "/")
(.addServlet servlet-holder "/"))]
(doseq [{:keys [filter-name filter-class filter-params]} filters-confs]
(if filter-class
- (let [filter-holder (doto (org.mortbay.jetty.servlet.FilterHolder.)
+ (let [filter-holder (doto (org.eclipse.jetty.servlet.FilterHolder.)
(.setClassName filter-class)
(.setName (or filter-name filter-class))
(.setInitParameters (or filter-params {})))]
- (.addFilter context filter-holder "/*" org.mortbay.jetty.Handler/ALL))))
- (.addHandler server context))))
+ (.addFilter context filter-holder "/*" FilterMapping/ALL))))
+ (.setHandler server context))))
(defn ring-response-from-exception [ex]
{:headers {}
http://git-wip-us.apache.org/repos/asf/storm/blob/3208a8e3/storm-core/test/clj/backtype/storm/ui_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/ui_test.clj b/storm-core/test/clj/backtype/storm/ui_test.clj
deleted file mode 100644
index 21ddb33..0000000
--- a/storm-core/test/clj/backtype/storm/ui_test.clj
+++ /dev/null
@@ -1,49 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns backtype.storm.ui-test
- (:use [clojure test])
- (:use [backtype.storm config])
- (:use [backtype.storm testing])
- (:require [backtype.storm.ui [core :as core]])
- )
-
-(deftest test-authorized-ui-user
- (testing "allow cluster admin"
- (let [conf {UI-FILTER "something" NIMBUS-ADMINS ["alice"]}]
- (is (core/authorized-ui-user? "alice" conf {}))))
-
- (testing "ignore any cluster-set topology.users"
- (let [conf {UI-FILTER "something" TOPOLOGY-USERS ["alice"]}]
- (is (not (core/authorized-ui-user? "alice" conf {})))))
-
- (testing "allow cluster ui user"
- (let [conf {UI-FILTER "something" UI-USERS ["alice"]}]
- (is (core/authorized-ui-user? "alice" conf {}))))
-
- (testing "allow submitted topology user"
- (let [topo-conf {TOPOLOGY-USERS ["alice"]}]
- (is (core/authorized-ui-user? "alice" {UI-FILTER "something"} topo-conf))))
-
- (testing "allow submitted ui user"
- (let [topo-conf {UI-USERS ["alice"]}]
- (is (core/authorized-ui-user? "alice" {UI-FILTER "something"} topo-conf))))
-
- (testing "disallow user not in nimbus admin, topo user, or ui user"
- (is (not (core/authorized-ui-user? "alice" {UI-FILTER "something"} {}))))
-
- (testing "user cannot override nimbus admin"
- (let [topo-conf {NIMBUS-ADMINS ["alice"]}]
- (is (not (core/authorized-ui-user? "alice" {UI-FILTER "something"} topo-conf))))))
[37/50] storm git commit: Merge branch 'STORM-480' of
http://github.com/d2r/incubator-storm into STORM-480
Posted by bo...@apache.org.
Merge branch 'STORM-480' of http://github.com/d2r/incubator-storm into STORM-480
STORM-480: change maven-dependency-plugin version 2.6 -> 2.8
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/429ead71
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/429ead71
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/429ead71
Branch: refs/heads/master
Commit: 429ead716da22466299e3f3f12933ec62de3f9c9
Parents: 5a277f8 f5db183
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Sep 23 13:52:04 2014 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Tue Sep 23 13:52:04 2014 -0500
----------------------------------------------------------------------
storm-core/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/429ead71/storm-core/pom.xml
----------------------------------------------------------------------
[41/50] storm git commit: Merge branch 'STORM-489' of
http://github.com/harshach/incubator-storm into STORM-489
Posted by bo...@apache.org.
Merge branch 'STORM-489' of http://github.com/harshach/incubator-storm into STORM-489
STORM-489: remove uiActionsEnabled from js and templates.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d45c1875
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d45c1875
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d45c1875
Branch: refs/heads/master
Commit: d45c1875c966ef77197910ed4ae41702ea22ec1b
Parents: 154ec7b ce0b18a
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Wed Sep 24 14:42:06 2014 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Wed Sep 24 14:42:06 2014 -0500
----------------------------------------------------------------------
storm-core/src/ui/public/js/script.js | 3 +--
storm-core/src/ui/public/topology.html | 2 +-
2 files changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d45c1875/storm-core/src/ui/public/js/script.js
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d45c1875/storm-core/src/ui/public/topology.html
----------------------------------------------------------------------
[42/50] storm git commit: Merge remote-tracking branch
'upstream/security' into security
Posted by bo...@apache.org.
Merge remote-tracking branch 'upstream/security' into security
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/84002179
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/84002179
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/84002179
Branch: refs/heads/master
Commit: 84002179cf7ae5879c93e900f64a7e576ce4cce1
Parents: 02f6db5 d45c187
Author: Sriharsha Chintalapani <ma...@harsha.io>
Authored: Mon Sep 29 10:17:06 2014 -0700
Committer: Sriharsha Chintalapani <ma...@harsha.io>
Committed: Mon Sep 29 10:17:06 2014 -0700
----------------------------------------------------------------------
CHANGELOG.md | 14 +
DEVELOPER.md | 2 +-
LICENSE | 25 +
README.markdown | 5 +-
STORM-UI-REST-API.md | 4 +-
dev-tools/jira-github-join.py | 2 +-
.../multilang/resources/asyncSplitsentence.js | 32 +
.../multilang/resources/randomsentence.js | 75 +++
.../multilang/resources/splitsentence.js | 26 +
.../storm-starter/multilang/resources/storm.js | 349 +++++++++++
.../storm/starter/WordCountTopologyNode.java | 121 ++++
.../src/jvm/storm/kafka/KafkaUtils.java | 5 +-
.../src/jvm/storm/kafka/PartitionManager.java | 5 +-
logback/worker.xml | 41 ++
pom.xml | 2 +-
storm-core/pom.xml | 55 +-
.../src/clj/backtype/storm/daemon/drpc.clj | 25 +-
.../src/clj/backtype/storm/daemon/logviewer.clj | 9 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 4 +-
.../clj/backtype/storm/daemon/supervisor.clj | 2 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 12 +-
.../src/clj/backtype/storm/ui/helpers.clj | 27 +-
storm-core/src/clj/backtype/storm/util.clj | 2 +-
storm-core/src/dev/resources/storm.js | 349 +++++++++++
storm-core/src/dev/resources/tester_bolt.js | 27 +
storm-core/src/dev/resources/tester_spout.js | 58 ++
.../src/jvm/backtype/storm/StormSubmitter.java | 17 +
.../backtype/storm/messaging/netty/Client.java | 27 +-
.../storm/security/auth/TBackoffConnect.java | 10 +-
.../storm/security/auth/kerberos/AutoTGT.java | 22 +-
.../StormBoundedExponentialBackoffRetry.java | 74 +++
.../src/jvm/backtype/storm/utils/Utils.java | 33 +-
storm-core/src/ui/public/component.html | 6 +
storm-core/src/ui/public/images/spinner.gif | Bin 0 -> 23470 bytes
storm-core/src/ui/public/index.html | 6 +
.../src/ui/public/js/jquery.blockUI.min.js | 6 +
storm-core/src/ui/public/js/script.js | 14 +-
storm-core/src/ui/public/js/visualization.js | 16 +-
.../templates/component-page-template.html | 593 ++++++++++++++-----
.../public/templates/index-page-template.html | 243 ++++++--
.../templates/topology-page-template.html | 208 ++++++-
storm-core/src/ui/public/topology.html | 8 +-
.../test/clj/backtype/storm/multilang_test.clj | 55 +-
.../test/clj/backtype/storm/supervisor_test.clj | 4 +-
.../test/clj/backtype/storm/utils_test.clj | 3 +-
...StormBoundedExponentialBackoffRetryTest.java | 101 ++++
storm-dist/binary/src/main/assembly/binary.xml | 4 +
47 files changed, 2374 insertions(+), 354 deletions(-)
----------------------------------------------------------------------
[25/50] storm git commit: change maven-dependency-plugin version 2.6
-> 2.8
Posted by bo...@apache.org.
change maven-dependency-plugin version 2.6 -> 2.8
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f5db1836
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f5db1836
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f5db1836
Branch: refs/heads/master
Commit: f5db183648ea2cac5fead49efbc6dfc12df823a4
Parents: b19fdbb
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Thu Aug 28 16:24:40 2014 -0500
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Thu Aug 28 16:24:40 2014 -0500
----------------------------------------------------------------------
storm-core/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f5db1836/storm-core/pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/pom.xml b/storm-core/pom.xml
index 3785858..e5812bc 100644
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@ -404,7 +404,7 @@
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
- <version>2.6</version>
+ <version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
[02/50] storm git commit: STORM-348: Netty SASL Authentication
Posted by bo...@apache.org.
STORM-348: Netty SASL Authentication
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/19ad1351
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/19ad1351
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/19ad1351
Branch: refs/heads/master
Commit: 19ad13510cb42963f868fd8bd785b4937dd348ab
Parents: 3bce04c
Author: Raghavendra Nandagopal <sp...@gmail.com>
Authored: Mon Jul 28 17:42:05 2014 -0700
Committer: Raghavendra Nandagopal <sp...@gmail.com>
Committed: Mon Jul 28 17:42:05 2014 -0700
----------------------------------------------------------------------
.../src/jvm/backtype/storm/messaging/netty/SaslUtils.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/19ad1351/storm-core/src/jvm/backtype/storm/messaging/netty/SaslUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/messaging/netty/SaslUtils.java b/storm-core/src/jvm/backtype/storm/messaging/netty/SaslUtils.java
index a4cc0ba..a2d0b26 100644
--- a/storm-core/src/jvm/backtype/storm/messaging/netty/SaslUtils.java
+++ b/storm-core/src/jvm/backtype/storm/messaging/netty/SaslUtils.java
@@ -24,6 +24,7 @@ import java.util.Map;
import javax.security.sasl.Sasl;
import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.Charsets;
import backtype.storm.Config;
@@ -45,8 +46,8 @@ class SaslUtils {
* @return password as a char array.
*/
static char[] encodePassword(byte[] password) {
- return new String(Base64.encodeBase64(password),
- Charset.defaultCharset()).toCharArray();
+ return new String(Base64.encodeBase64(password), Charsets.UTF_8)
+ .toCharArray();
}
/**
@@ -57,8 +58,7 @@ class SaslUtils {
* @return identifier as a char array.
*/
static String encodeIdentifier(byte[] identifier) {
- return new String(Base64.encodeBase64(identifier),
- Charset.defaultCharset());
+ return new String(Base64.encodeBase64(identifier), Charsets.UTF_8);
}
static String getSecretKey(Map conf) {
[43/50] storm git commit: STORM-509. Make groups checking specific
for SimpleACLAuthorizer.
Posted by bo...@apache.org.
STORM-509. Make groups checking specific for SimpleACLAuthorizer.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/11e6fc98
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/11e6fc98
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/11e6fc98
Branch: refs/heads/master
Commit: 11e6fc9821d7d7e676d573836c08272ce18385f2
Parents: 8400217
Author: Sriharsha Chintalapani <ma...@harsha.io>
Authored: Mon Sep 29 13:57:07 2014 -0700
Committer: Sriharsha Chintalapani <ma...@harsha.io>
Committed: Mon Sep 29 13:57:07 2014 -0700
----------------------------------------------------------------------
storm-core/src/jvm/backtype/storm/Config.java | 13 +++++++++---
.../auth/authorizer/SimpleACLAuthorizer.java | 21 +++++++++++++-------
.../backtype/storm/security/auth/auth_test.clj | 4 ++--
3 files changed, 26 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/11e6fc98/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index a38b51e..48e1822 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -115,7 +115,7 @@ public class Config extends HashMap<String, Object> {
*/
public static final String STORM_META_SERIALIZATION_DELEGATE = "storm.meta.serialization.delegate";
public static final Object STORM_META_SERIALIZATION_DELEGATE_SCHEMA = String.class;
-
+
/**
* A list of hosts of ZooKeeper servers used to manage the cluster.
*/
@@ -769,7 +769,7 @@ public class Config extends HashMap<String, Object> {
* The jvm opts provided to workers launched by this supervisor. All "%ID%", "%WORKER-ID%", "%TOPOLOGY-ID%"
* and "%WORKER-PORT%" substrings are replaced with:
* %ID% -> port (for backward compatibility),
- * %WORKER-ID% -> worker-id,
+ * %WORKER-ID% -> worker-id,
* %TOPOLOGY-ID% -> topology-id,
* %WORKER-PORT% -> port.
*/
@@ -829,6 +829,13 @@ public class Config extends HashMap<String, Object> {
public static final Object TOPOLOGY_USERS_SCHEMA = ConfigValidation.StringsValidator;
/**
+ * A list of groups that are allowed to interact with the topology. To use this set
+ * nimbus.authorizer to backtype.storm.security.auth.authorizer.SimpleACLAuthorizer
+ */
+ public static final String TOPOLOGY_GROUPS = "topology.groups";
+ public static final Object TOPOLOGY_GROUPS_SCHEMA = ConfigValidation.StringsValidator;
+
+ /**
* True if Storm should timeout messages or not. Defaults to true. This is meant to be used
* in unit tests to prevent tuples from being accidentally timed out during the test.
*/
@@ -1304,7 +1311,7 @@ public class Config extends HashMap<String, Object> {
public void registerSerialization(Class klass, Class<? extends Serializer> serializerClass) {
registerSerialization(this, klass, serializerClass);
}
-
+
public static void registerMetricsConsumer(Map conf, Class klass, Object argument, long parallelismHint) {
HashMap m = new HashMap();
m.put("class", klass.getCanonicalName());
http://git-wip-us.apache.org/repos/asf/storm/blob/11e6fc98/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleACLAuthorizer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleACLAuthorizer.java b/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleACLAuthorizer.java
index 0b867e8..daef956 100644
--- a/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleACLAuthorizer.java
+++ b/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleACLAuthorizer.java
@@ -49,7 +49,7 @@ public class SimpleACLAuthorizer implements IAuthorizer {
protected Set<String> _admins;
protected Set<String> _supervisors;
protected IPrincipalToLocal _ptol;
- protected IGroupMappingServiceProvider _groups;
+ protected IGroupMappingServiceProvider _groupMappingProvider;
/**
* Invoked once immediately after construction
* @param conf Storm configuration
@@ -65,8 +65,9 @@ public class SimpleACLAuthorizer implements IAuthorizer {
if (conf.containsKey(Config.NIMBUS_SUPERVISOR_USERS)) {
_supervisors.addAll((Collection<String>)conf.get(Config.NIMBUS_SUPERVISOR_USERS));
}
+
_ptol = AuthUtils.GetPrincipalToLocalPlugin(conf);
- _groups = AuthUtils.GetGroupMappingServiceProviderPlugin(conf);
+ _groupMappingProvider = AuthUtils.GetGroupMappingServiceProviderPlugin(conf);
}
/**
@@ -107,12 +108,18 @@ public class SimpleACLAuthorizer implements IAuthorizer {
if (topoUsers.contains(principal) || topoUsers.contains(user)) {
return true;
}
- if(_groups != null) {
+
+ Set<String> topoGroups = new HashSet<String>();
+ if (topology_conf.containsKey(Config.TOPOLOGY_GROUPS)) {
+ topoGroups.addAll((Collection<String>)topology_conf.get(Config.TOPOLOGY_GROUPS));
+ }
+
+ if(_groupMappingProvider != null && topoGroups.size() > 0) {
+ System.out.println("hello in if");
try {
- String topologySubmitterUser = (String) topology_conf.get(Config.TOPOLOGY_SUBMITTER_USER);
- Set<String> userGroups = _groups.getGroups(user);
- Set<String> topoUserGroups = _groups.getGroups(topologySubmitterUser);
- for (String tgroup : topoUserGroups) {
+ Set<String> userGroups = _groupMappingProvider.getGroups(user);
+ for (String tgroup : topoGroups) {
+ System.out.println("tgroup "+tgroup);
if(userGroups.contains(tgroup))
return true;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/11e6fc98/storm-core/test/clj/backtype/storm/security/auth/auth_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/security/auth/auth_test.clj b/storm-core/test/clj/backtype/storm/security/auth/auth_test.clj
index 12411e7..a01cac3 100644
--- a/storm-core/test/clj/backtype/storm/security/auth/auth_test.clj
+++ b/storm-core/test/clj/backtype/storm/security/auth/auth_test.clj
@@ -284,8 +284,8 @@
user-name (System/getProperty "user.name")]
(.prepare groups cluster-conf)
(>= 0 (.size (.getGroups groups user-name)))
- (>= 0 (.size (.getGroups groups "userDoesNotExist")))
- (>= 0 (.size (.getGroups groups nil)))))
+ (= 0 (.size (.getGroups groups "userDoesNotExist")))
+ (= 0 (.size (.getGroups groups nil)))))
(deftest simple-acl-same-user-auth-test
(let [cluster-conf (merge (read-storm-config)
[14/50] storm git commit: STORM-406: Fix for storm CSRF vulnerability
using ring-anti-forgery.
Posted by bo...@apache.org.
STORM-406: Fix for storm CSRF vulnerability using ring-anti-forgery.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/57672629
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/57672629
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/57672629
Branch: refs/heads/master
Commit: 576726292a88c67aa9e9ba7563de114540487f54
Parents: 1a1df08
Author: Parth Brahmbhatt <br...@gmail.com>
Authored: Sun Aug 17 14:56:27 2014 -0700
Committer: Parth Brahmbhatt <br...@gmail.com>
Committed: Sun Aug 17 14:56:27 2014 -0700
----------------------------------------------------------------------
storm-core/pom.xml | 7 ++++++-
storm-core/src/clj/backtype/storm/ui/core.clj | 14 +++++++++++---
storm-core/src/ui/public/css/style.css | 4 ++++
storm-core/src/ui/public/js/script.js | 3 ++-
.../public/templates/anti-forgery-template.html | 19 +++++++++++++++++++
storm-core/src/ui/public/topology.html | 7 +++++++
6 files changed, 49 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/57672629/storm-core/pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/pom.xml b/storm-core/pom.xml
index c677b7c..44bca86 100644
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@ -60,6 +60,11 @@
<artifactId>ring-jetty-adapter</artifactId>
</dependency>
<dependency>
+ <groupId>ring</groupId>
+ <artifactId>ring-anti-forgery</artifactId>
+ <version>1.0.0</version>
+ </dependency>
+ <dependency>
<groupId>org.clojure</groupId>
<artifactId>tools.logging</artifactId>
</dependency>
@@ -96,7 +101,7 @@
<artifactId>data.codec</artifactId>
<scope>test</scope>
</dependency>
-
+
<!--java-->
<dependency>
<groupId>commons-io</groupId>
http://git-wip-us.apache.org/repos/asf/storm/blob/57672629/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/ui/core.clj b/storm-core/src/clj/backtype/storm/ui/core.clj
index 8dd301e..f771ee6 100644
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@ -23,6 +23,7 @@
(:use [backtype.storm.daemon [common :only [ACKER-COMPONENT-ID ACKER-INIT-STREAM-ID
ACKER-ACK-STREAM-ID ACKER-FAIL-STREAM-ID system-id?]]])
(:use [ring.adapter.jetty :only [run-jetty]])
+ (:use [ring.middleware.anti-forgery])
(:use [clojure.string :only [blank? lower-case trim]])
(:import [backtype.storm.utils Utils])
(:import [backtype.storm.generated ExecutorSpecificStats
@@ -687,7 +688,8 @@
"bolts" (bolt-comp id bolt-comp-summs (.get_errors summ) window include-sys?)
"configuration" topology-conf
"visualizationTable" (stream-boxes visualizer-data)
- "uiActionsEnabled" (ui-actions-enabled?)}))))
+ "uiActionsEnabled" (ui-actions-enabled?)
+ "anti-forgery-token" *anti-forgery-token*}))))
(defn spout-output-stats
[stream-summary window]
@@ -956,10 +958,16 @@
(catch Exception ex
(json-response (exception->json ex) 500)))))
+
+(def csrf-error-response
+ (json-response {"error" "Forbidden action."
+ "errorMessage" "missing CSRF token."} 403))
+
(def app
(handler/site (-> main-routes
- (wrap-reload '[backtype.storm.ui.core])
- catch-errors)))
+ (wrap-reload '[backtype.storm.ui.core])
+ (wrap-anti-forgery {:error-response csrf-error-response})
+ catch-errors)))
(defn start-server!
[]
http://git-wip-us.apache.org/repos/asf/storm/blob/57672629/storm-core/src/ui/public/css/style.css
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/css/style.css b/storm-core/src/ui/public/css/style.css
index 29a45eb..85a8961 100644
--- a/storm-core/src/ui/public/css/style.css
+++ b/storm-core/src/ui/public/css/style.css
@@ -24,6 +24,10 @@
padding: 0.5em;
}
+.anti-forgery-token {
+visibility:hidden;
+}
+
body {
color: #808080;
padding: 0.2em;
http://git-wip-us.apache.org/repos/asf/storm/blob/57672629/storm-core/src/ui/public/js/script.js
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/js/script.js b/storm-core/src/ui/public/js/script.js
index fef3b59..09ecd57 100644
--- a/storm-core/src/ui/public/js/script.js
+++ b/storm-core/src/ui/public/js/script.js
@@ -71,7 +71,8 @@ function ensureInt(n) {
function confirmAction(id, name, action, wait, defaultWait) {
var opts = {
type:'POST',
- url:'/api/v1/topology/' + id + '/' + action
+ url:'/api/v1/topology/' + id + '/' + action,
+ headers: { 'x-csrf-token': $.trim($('#anti-forgery-token').text()) }
};
if (wait) {
var waitSecs = prompt('Do you really want to ' + action + ' topology "' + name + '"? ' +
http://git-wip-us.apache.org/repos/asf/storm/blob/57672629/storm-core/src/ui/public/templates/anti-forgery-template.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/templates/anti-forgery-template.html b/storm-core/src/ui/public/templates/anti-forgery-template.html
new file mode 100644
index 0000000..1a63ace
--- /dev/null
+++ b/storm-core/src/ui/public/templates/anti-forgery-template.html
@@ -0,0 +1,19 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<script id="anti-forgery-template" type="text/html">
+{{#anti-forgery-token}}{{anti-forgery-token}}{{/anti-forgery-token}}
+</script>
http://git-wip-us.apache.org/repos/asf/storm/blob/57672629/storm-core/src/ui/public/topology.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/topology.html b/storm-core/src/ui/public/topology.html
index eb74522..805b6b4 100644
--- a/storm-core/src/ui/public/topology.html
+++ b/storm-core/src/ui/public/topology.html
@@ -52,6 +52,8 @@
<p id="toggle-switch" style="display: block;" class="js-only"></p>
<div id="json-response-error">
</div>
+<div id="anti-forgery-token" class="anti-forgery-token">
+</div>
</body>
<script>
$(document).ready(function() {
@@ -76,6 +78,11 @@ $(document).ready(function() {
uiUser.append(Mustache.render($(template).filter("#user-template").html(),response));
});
+ var antiForgeryToken = $("#anti-forgery-token");
+ $.get("/templates/anti-forgery-template.html", function(template) {
+ antiForgeryToken.append(Mustache.render($(template).filter("#anti-forgery-template").html(),response));
+ });
+
var topologySummary = $("#topology-summary");
var topologyStats = $("#topology-stats");
var spoutStats = $("#spout-stats");
[07/50] storm git commit: STORM-348: Netty SASL Authentication
Posted by bo...@apache.org.
STORM-348: Netty SASL Authentication
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/00f7d632
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/00f7d632
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/00f7d632
Branch: refs/heads/master
Commit: 00f7d632eb48bf967b5a7b72f5460352d0a8dbd9
Parents: 90cf0e6
Author: Raghavendra Nandagopal <sp...@gmail.com>
Authored: Tue Jul 29 17:50:24 2014 -0700
Committer: Raghavendra Nandagopal <sp...@gmail.com>
Committed: Tue Jul 29 17:50:24 2014 -0700
----------------------------------------------------------------------
.../clj/backtype/storm/messaging/netty_integration_test.clj | 2 +-
.../test/clj/backtype/storm/messaging/netty_unit_test.clj | 8 ++++----
2 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/00f7d632/storm-core/test/clj/backtype/storm/messaging/netty_integration_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/messaging/netty_integration_test.clj b/storm-core/test/clj/backtype/storm/messaging/netty_integration_test.clj
index dea4abe..98144cc 100644
--- a/storm-core/test/clj/backtype/storm/messaging/netty_integration_test.clj
+++ b/storm-core/test/clj/backtype/storm/messaging/netty_integration_test.clj
@@ -25,7 +25,7 @@
(with-simulated-time-local-cluster [cluster :supervisors 4 :supervisor-slot-port-min 6710
:daemon-conf {STORM-LOCAL-MODE-ZMQ true
STORM-MESSAGING-TRANSPORT "backtype.storm.messaging.netty.Context"
- STORM_MESSAGING_NETTY_AUTHENTICATION false
+ STORM-MESSAGING-NETTY-AUTHENTICATION false
STORM-MESSAGING-NETTY-BUFFER-SIZE 1024000
STORM-MESSAGING-NETTY-MAX-RETRIES 10
STORM-MESSAGING-NETTY-MIN-SLEEP-MS 1000
http://git-wip-us.apache.org/repos/asf/storm/blob/00f7d632/storm-core/test/clj/backtype/storm/messaging/netty_unit_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/messaging/netty_unit_test.clj b/storm-core/test/clj/backtype/storm/messaging/netty_unit_test.clj
index 04d25ec..8aaa7e5 100644
--- a/storm-core/test/clj/backtype/storm/messaging/netty_unit_test.clj
+++ b/storm-core/test/clj/backtype/storm/messaging/netty_unit_test.clj
@@ -26,7 +26,7 @@
(deftest test-basic
(let [req_msg (String. "0123456789abcdefghijklmnopqrstuvwxyz")
storm-conf {STORM-MESSAGING-TRANSPORT "backtype.storm.messaging.netty.Context"
- STORM_MESSAGING_NETTY_AUTHENTICATION false
+ STORM-MESSAGING-NETTY-AUTHENTICATION false
STORM-MESSAGING-NETTY-BUFFER-SIZE 1024
STORM-MESSAGING-NETTY-MAX-RETRIES 10
STORM-MESSAGING-NETTY-MIN-SLEEP-MS 1000
@@ -49,7 +49,7 @@
(deftest test-large-msg
(let [req_msg (apply str (repeat 2048000 'c'))
storm-conf {STORM-MESSAGING-TRANSPORT "backtype.storm.messaging.netty.Context"
- STORM_MESSAGING_NETTY_AUTHENTICATION false
+ STORM-MESSAGING-NETTY-AUTHENTICATION false
STORM-MESSAGING-NETTY-BUFFER-SIZE 102400
STORM-MESSAGING-NETTY-MAX-RETRIES 10
STORM-MESSAGING-NETTY-MIN-SLEEP-MS 1000
@@ -72,7 +72,7 @@
(deftest test-server-delayed
(let [req_msg (String. "0123456789abcdefghijklmnopqrstuvwxyz")
storm-conf {STORM-MESSAGING-TRANSPORT "backtype.storm.messaging.netty.Context"
- STORM_MESSAGING_NETTY_AUTHENTICATION false
+ STORM-MESSAGING-NETTY-AUTHENTICATION false
STORM-MESSAGING-NETTY-BUFFER-SIZE 1024
STORM-MESSAGING-NETTY-MAX-RETRIES 10
STORM-MESSAGING-NETTY-MIN-SLEEP-MS 1000
@@ -102,7 +102,7 @@
(deftest test-batch
(let [storm-conf {STORM-MESSAGING-TRANSPORT "backtype.storm.messaging.netty.Context"
- STORM_MESSAGING_NETTY_AUTHENTICATION false
+ STORM-MESSAGING-NETTY-AUTHENTICATION false
STORM-MESSAGING-NETTY-BUFFER-SIZE 1024000
STORM-MESSAGING-NETTY-MAX-RETRIES 10
STORM-MESSAGING-NETTY-MIN-SLEEP-MS 1000
[47/50] storm git commit: Merge branch 'master' into security-upmerge
Posted by bo...@apache.org.
Merge branch 'master' into security-upmerge
Conflicts:
storm-core/src/clj/backtype/storm/daemon/supervisor.clj
storm-core/src/clj/backtype/storm/ui/core.clj
storm-core/src/ui/public/templates/topology-page-template.html
storm-core/test/clj/backtype/storm/supervisor_test.clj
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ea0b8a2a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ea0b8a2a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ea0b8a2a
Branch: refs/heads/master
Commit: ea0b8a2a51332020a569ed7aeb3e0647477af763
Parents: 4788c7e cfe7e63
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Wed Oct 15 11:05:10 2014 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Wed Oct 15 11:05:10 2014 -0500
----------------------------------------------------------------------
CHANGELOG.md | 16 +-
DEVELOPER.md | 26 +-
LICENSE | 20 +-
README.markdown | 3 +
bin/storm | 3 +-
examples/storm-starter/README.markdown | 9 +-
examples/storm-starter/pom.xml | 2 +-
external/storm-hbase/LICENSE | 202 +++++++++++
external/storm-hbase/README.md | 215 ++++++++++++
external/storm-hbase/pom.xml | 78 +++++
.../storm/hbase/bolt/AbstractHBaseBolt.java | 69 ++++
.../org/apache/storm/hbase/bolt/HBaseBolt.java | 77 ++++
.../storm/hbase/bolt/HBaseLookupBolt.java | 83 +++++
.../storm/hbase/bolt/mapper/HBaseMapper.java | 48 +++
.../bolt/mapper/HBaseProjectionCriteria.java | 85 +++++
.../hbase/bolt/mapper/HBaseValueMapper.java | 43 +++
.../hbase/bolt/mapper/SimpleHBaseMapper.java | 90 +++++
.../apache/storm/hbase/common/ColumnList.java | 202 +++++++++++
.../apache/storm/hbase/common/HBaseClient.java | 135 +++++++
.../org/apache/storm/hbase/common/IColumn.java | 30 ++
.../org/apache/storm/hbase/common/ICounter.java | 29 ++
.../org/apache/storm/hbase/common/Utils.java | 69 ++++
.../storm/hbase/security/HBaseSecurityUtil.java | 52 +++
.../mapper/SimpleTridentHBaseMapper.java | 89 +++++
.../trident/mapper/TridentHBaseMapper.java | 31 ++
.../hbase/trident/state/HBaseMapState.java | 233 +++++++++++++
.../storm/hbase/trident/state/HBaseQuery.java | 40 +++
.../storm/hbase/trident/state/HBaseState.java | 164 +++++++++
.../hbase/trident/state/HBaseStateFactory.java | 40 +++
.../storm/hbase/trident/state/HBaseUpdater.java | 32 ++
.../storm/hbase/topology/LookupWordCount.java | 79 +++++
.../hbase/topology/PersistentWordCount.java | 81 +++++
.../storm/hbase/topology/TotalWordCounter.java | 70 ++++
.../storm/hbase/topology/WordCountClient.java | 57 +++
.../hbase/topology/WordCountValueMapper.java | 70 ++++
.../storm/hbase/topology/WordCounter.java | 59 ++++
.../apache/storm/hbase/topology/WordSpout.java | 88 +++++
.../storm/hbase/trident/PrintFunction.java | 40 +++
.../storm/hbase/trident/WordCountTrident.java | 104 ++++++
external/storm-hdfs/pom.xml | 2 +-
external/storm-kafka/pom.xml | 2 +-
.../jvm/storm/kafka/DynamicBrokersReader.java | 2 +-
.../storm/kafka/DynamicBrokersReaderTest.java | 1 +
.../src/test/storm/kafka/TridentKafkaTest.java | 9 +-
pom.xml | 27 +-
.../maven-shade-clojure-transformer/pom.xml | 2 +-
storm-core/pom.xml | 6 +-
.../clj/backtype/storm/daemon/supervisor.clj | 4 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 51 +--
storm-core/src/dev/resources/storm.py | 248 ++++++++++++-
storm-core/src/dev/resources/storm.rb | 228 +++++++++++-
.../src/jvm/backtype/storm/tuple/ITuple.java | 136 ++++++++
.../src/jvm/backtype/storm/tuple/Tuple.java | 113 +-----
.../storm/trident/testing/MockTridentTuple.java | 166 ---------
.../jvm/storm/trident/tuple/TridentTuple.java | 46 +--
.../storm/trident/tuple/TridentTupleView.java | 35 +-
.../jvm/storm/trident/tuple/ValuePointer.java | 1 -
storm-core/src/multilang/js/storm.js | 349 +++++++++++++++++++
storm-core/src/ui/public/component.html | 8 +-
storm-core/src/ui/public/js/purl.js | 267 --------------
storm-core/src/ui/public/js/script.js | 3 +-
storm-core/src/ui/public/js/url.min.js | 1 +
storm-core/src/ui/public/js/visualization.js | 2 +-
.../templates/component-page-template.html | 6 +-
.../public/templates/index-page-template.html | 2 +-
.../templates/topology-page-template.html | 6 +-
storm-core/src/ui/public/topology.html | 12 +-
.../test/clj/backtype/storm/supervisor_test.clj | 12 +-
.../test/clj/storm/trident/tuple_test.clj | 28 +-
storm-dist/binary/LICENSE | 10 +-
storm-dist/binary/pom.xml | 2 +-
storm-dist/binary/src/main/assembly/binary.xml | 14 +
storm-dist/source/pom.xml | 2 +-
73 files changed, 3949 insertions(+), 717 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/README.markdown
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/bin/storm
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index 1fe5b2e,cce89bc..c580bc8
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@@ -605,9 -501,8 +605,9 @@@
(defmethod launch-worker
:distributed [supervisor storm-id port worker-id]
(let [conf (:conf supervisor)
+ run-worker-as-user (conf SUPERVISOR-RUN-WORKER-AS-USER)
storm-home (System/getProperty "storm.home")
- storm-log-dir (or (System/getProperty "storm.log.dir") (str storm-home "/logs"))
+ storm-log-dir (or (System/getProperty "storm.log.dir") (str storm-home file-path-separator "logs"))
stormroot (supervisor-stormdist-root conf storm-id)
jlp (jlp stormroot conf)
stormjar (supervisor-stormjar-path stormroot)
@@@ -638,7 -529,7 +638,7 @@@
(str "-Dlogfile.name=" logfilename)
(str "-Dstorm.home=" storm-home)
(str "-Dstorm.log.dir=" storm-log-dir)
- (str "-Dlogback.configurationFile=" storm-home "/logback/worker.xml")
- (str "-Dlogback.configurationFile=" storm-home file-path-separator "logback" file-path-separator "cluster.xml")
++ (str "-Dlogback.configurationFile=" storm-home file-path-separator "logback" file-path-separator "worker.xml")
(str "-Dstorm.id=" storm-id)
(str "-Dworker.id=" worker-id)
(str "-Dworker.port=" port)
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/ui/core.clj
index 600c96a,969b514..e980a55
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@@ -541,8 -519,9 +542,10 @@@
([summs]
{"topologies"
(for [^TopologySummary t summs]
- {"id" (.get_id t)
+ {
+ "id" (.get_id t)
+ "encodedId" (url-encode (.get_id t))
+ "owner" (.get_owner t)
"name" (.get_name t)
"status" (.get_status t)
"uptime" (pretty-uptime-sec (.get_uptime_secs t))
@@@ -573,8 -551,9 +576,9 @@@
stats-seq include-sys?))
last-error (most-recent-error (get errors id))
error-host (get-error-host last-error)
- error-port (get-error-port last-error error-host top-id) ]]
+ error-port (get-error-port last-error error-host top-id)]]
{"spoutId" id
+ "encodedSpoutId" (url-encode id)
"executors" (count summs)
"tasks" (sum-tasks summs)
"emitted" (get-in stats [:emitted window])
@@@ -596,8 -575,9 +600,9 @@@
stats-seq include-sys?))
last-error (most-recent-error (get errors id))
error-host (get-error-host last-error)
- error-port (get-error-port last-error error-host top-id) ]]
+ error-port (get-error-port last-error error-host top-id)]]
{"boltId" id
+ "encodedBoltId" (url-encode id)
"executors" (count summs)
"tasks" (sum-tasks summs)
"emitted" (get-in stats [:emitted window])
@@@ -619,7 -599,7 +624,8 @@@
workers (set (for [^ExecutorSummary e executors]
[(.get_host e) (.get_port e)]))]
{"id" (.get_id summ)
+ "encodedId" (url-encode (.get_id summ))
+ "owner" (.get_owner summ)
"name" (.get_name summ)
"status" (.get_status summ)
"uptime" (pretty-uptime-sec (.get_uptime_secs summ))
@@@ -832,8 -811,8 +841,9 @@@
(= type :bolt) (bolt-stats window summ component summs include-sys?))
errors (component-errors (get (.get_errors summ) component) topology-id)]
(merge
- { "id" component
+ {"user" user
+ "id" component
+ "encodedId" (url-encode component)
"name" (.get_name summ)
"executors" (count summs)
"tasks" (sum-tasks summs)
@@@ -869,67 -843,47 +880,59 @@@
(GET "/api/v1/cluster/configuration" [& m]
(json-response (cluster-configuration)
(:callback m) :serialize-fn identity))
- (GET "/api/v1/cluster/summary" [& m]
- (json-response (cluster-summary) (:callback m)))
- (GET "/api/v1/supervisor/summary" [& m]
+ (GET "/api/v1/cluster/summary" [:as {:keys [cookies servlet-request]} & m]
+ (let [user (.getUserName http-creds-handler servlet-request)]
+ (assert-authorized-user servlet-request "getClusterInfo")
+ (json-response (cluster-summary user) (:callback m))))
+ (GET "/api/v1/supervisor/summary" [:as {:keys [cookies servlet-request]} & m]
+ (assert-authorized-user servlet-request "getClusterInfo")
(json-response (supervisor-summary) (:callback m)))
- (GET "/api/v1/topology/summary" [& m]
+ (GET "/api/v1/topology/summary" [:as {:keys [cookies servlet-request]} & m]
+ (assert-authorized-user servlet-request "getClusterInfo")
(json-response (all-topologies-summary) (:callback m)))
- (GET "/api/v1/topology/:id" [id & m]
- (json-response (topology-page id (:window m) (check-include-sys? (:sys m))) (:callback m)))
+ (GET "/api/v1/topology/:id" [:as {:keys [cookies servlet-request]} id & m]
- (let [id (url-decode id)
- user (.getUserName http-creds-handler servlet-request)]
++ (let [user (.getUserName http-creds-handler servlet-request)]
+ (assert-authorized-user servlet-request "getTopology" (topology-config id))
+ (json-response (topology-page id (:window m) (check-include-sys? (:sys m)) user) (:callback m))))
(GET "/api/v1/topology/:id/visualization" [:as {:keys [cookies servlet-request]} id & m]
- (let [id (url-decode id)]
- (assert-authorized-user servlet-request "getTopology" (topology-config id))
- (json-response (mk-visualization-data id (:window m) (check-include-sys? (:sys m))) (:callback m))))
- (json-response (mk-visualization-data id (:window m) (check-include-sys? (:sys m))) (:callback m)))
- (GET "/api/v1/topology/:id/component/:component" [id component & m]
- (json-response (component-page id component (:window m) (check-include-sys? (:sys m))) (:callback m)))
- (POST "/api/v1/topology/:id/activate" [id]
++ (assert-authorized-user servlet-request "getTopology" (topology-config id))
++ (json-response (mk-visualization-data id (:window m) (check-include-sys? (:sys m))) (:callback m)))
+ (GET "/api/v1/topology/:id/component/:component" [:as {:keys [cookies servlet-request]} id component & m]
- (let [id (url-decode id)
- component (url-decode component)
- user (.getUserName http-creds-handler servlet-request)]
++ (let [user (.getUserName http-creds-handler servlet-request)]
+ (assert-authorized-user servlet-request "getTopology" (topology-config id))
+ (json-response (component-page id component (:window m) (check-include-sys? (:sys m)) user) (:callback m))))
+ (POST "/api/v1/topology/:id/activate" [:as {:keys [cookies servlet-request]} id]
(with-nimbus nimbus
- (let [id (url-decode id)
- tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+ (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
name (.get_name tplg)]
+ (assert-authorized-user servlet-request "activate" (topology-config id))
(.activate nimbus name)
(log-message "Activating topology '" name "'")))
- (resp/redirect (str "/api/v1/topology/" (url-encode id))))
-
- (POST "/api/v1/topology/:id/deactivate" [id]
+ (resp/redirect (str "/api/v1/topology/" id)))
+ (POST "/api/v1/topology/:id/deactivate" [:as {:keys [cookies servlet-request]} id]
(with-nimbus nimbus
- (let [id (url-decode id)
- tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+ (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
name (.get_name tplg)]
+ (assert-authorized-user servlet-request "deactivate" (topology-config id))
(.deactivate nimbus name)
(log-message "Deactivating topology '" name "'")))
- (resp/redirect (str "/api/v1/topology/" id)))
+ (resp/redirect (str "/api/v1/topology/" (url-encode id))))
- (POST "/api/v1/topology/:id/rebalance/:wait-time" [id wait-time]
+ (POST "/api/v1/topology/:id/rebalance/:wait-time" [:as {:keys [cookies servlet-request]} id wait-time]
(with-nimbus nimbus
- (let [id (url-decode id)
- tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+ (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
name (.get_name tplg)
options (RebalanceOptions.)]
+ (assert-authorized-user servlet-request "rebalance" (topology-config id))
(.set_wait_secs options (Integer/parseInt wait-time))
(.rebalance nimbus name options)
(log-message "Rebalancing topology '" name "' with wait time: " wait-time " secs")))
- (resp/redirect (str "/api/v1/topology/" id)))
+ (resp/redirect (str "/api/v1/topology/" (url-encode id))))
- (POST "/api/v1/topology/:id/kill/:wait-time" [id wait-time]
+ (POST "/api/v1/topology/:id/kill/:wait-time" [:as {:keys [cookies servlet-request]} id wait-time]
(with-nimbus nimbus
- (let [id (url-decode id)
- tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+ (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
name (.get_name tplg)
options (KillOptions.)]
+ (assert-authorized-user servlet-request "killTopology" (topology-config id))
(.set_wait_secs options (Integer/parseInt wait-time))
(.killTopologyWithOpts nimbus name options)
(log-message "Killing topology '" name "' with wait time: " wait-time " secs")))
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/ui/public/component.html
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/ui/public/js/script.js
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/ui/public/templates/index-page-template.html
----------------------------------------------------------------------
diff --cc storm-core/src/ui/public/templates/index-page-template.html
index 800b579,e3aab5e..9c48e83
--- a/storm-core/src/ui/public/templates/index-page-template.html
+++ b/storm-core/src/ui/public/templates/index-page-template.html
@@@ -128,9 -118,8 +128,9 @@@
<tbody>
{{#topologies}}
<tr>
- <td><a href="/topology.html?id={{id}}">{{name}}</a></td>
+ <td><a href="/topology.html?id={{encodedId}}">{{name}}</a></td>
<td>{{id}}</td>
+ <td>{{owner}}</td>
<td>{{status}}</td>
<td>{{uptime}}</td>
<td>{{workersTotal}}</td>
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/ui/public/templates/topology-page-template.html
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/ui/public/topology.html
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --cc storm-core/test/clj/backtype/storm/supervisor_test.clj
index 4261b4d,ba74d88..ebbf060
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@@ -259,10 -253,10 +259,10 @@@
opts
topo-opts
["-Djava.library.path="
- (str "-Dlogfile.name=worker-" mock-port ".log")
+ (str "-Dlogfile.name=" mock-storm-id "-worker-" mock-port ".log")
"-Dstorm.home="
- "-Dstorm.log.dir=/logs"
- "-Dlogback.configurationFile=/logback/worker.xml"
+ (str "-Dstorm.log.dir=" file-path-separator "logs")
- (str "-Dlogback.configurationFile=" file-path-separator "logback" file-path-separator "cluster.xml")
++ (str "-Dlogback.configurationFile=" file-path-separator "logback" file-path-separator "worker.xml")
(str "-Dstorm.id=" mock-storm-id)
(str "-Dworker.id=" mock-worker-id)
(str "-Dworker.port=" mock-port)
@@@ -322,10 -312,8 +322,10 @@@
(stubbing [read-supervisor-storm-conf {TOPOLOGY-CLASSPATH topo-cp}
supervisor-stormdist-root nil
supervisor/jlp nil
+ set-worker-user! nil
+ supervisor/write-log-metadata! nil
launch-process nil
- current-classpath "/base"]
+ current-classpath (str file-path-separator "base")]
(supervisor/launch-worker mock-supervisor
mock-storm-id
mock-port
@@@ -342,9 -329,7 +342,9 @@@
supervisor-stormdist-root nil
supervisor/jlp nil
launch-process nil
+ set-worker-user! nil
+ supervisor/write-log-metadata! nil
- current-classpath "/base"]
+ current-classpath (str file-path-separator "base")]
(supervisor/launch-worker mock-supervisor
mock-storm-id
mock-port
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-dist/binary/src/main/assembly/binary.xml
----------------------------------------------------------------------
[20/50] storm git commit: changing variable name to antiForgeryToken,
updating UI Rest API Document.
Posted by bo...@apache.org.
changing variable name to antiForgeryToken, updating UI Rest API Document.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/dd59ad41
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/dd59ad41
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/dd59ad41
Branch: refs/heads/master
Commit: dd59ad41caf9463a110d60e4e5b09dba50574428
Parents: 029b1e1
Author: Parth Brahmbhatt <br...@gmail.com>
Authored: Tue Aug 26 14:59:38 2014 -0700
Committer: Parth Brahmbhatt <br...@gmail.com>
Committed: Tue Aug 26 14:59:38 2014 -0700
----------------------------------------------------------------------
STORM-UI-REST-API.md | 19 +++++++++++++++++--
storm-core/src/clj/backtype/storm/ui/core.clj | 2 +-
.../public/templates/anti-forgery-template.html | 2 +-
3 files changed, 19 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/dd59ad41/STORM-UI-REST-API.md
----------------------------------------------------------------------
diff --git a/STORM-UI-REST-API.md b/STORM-UI-REST-API.md
index 6dda435..848880b 100644
--- a/STORM-UI-REST-API.md
+++ b/STORM-UI-REST-API.md
@@ -1,7 +1,7 @@
# Storm UI REST API
Storm UI server provides a REST Api to access cluster, topology, component overview and metrics.
This api returns json response.
-Please ignore undocumented elements in the json repsonse.
+Please ignore undocumented elements in the json response.
## Using the UI REST Api
@@ -179,7 +179,7 @@ Response Fields:
|bolts.errorLapsedSecs| Integer |Number of seconds elapsed since that last error happened in a bolt|
|bolts.errorWorkerLogLink| String | Link to the worker log that reported the exception |
|bolts.emitted| Long |Number of tuples emitted|
-
+|antiForgeryToken| String | CSRF token|
Examples:
@@ -324,6 +324,7 @@ Sample Response:
"supervisor.enable": true,
"storm.messaging.netty.server_worker_threads": 1
},
+ "antiForgeryToken": "lAFTN\/5iSedRLwJeUNqkJ8hgYubRl2OxjXGoDf9A4Bt1nZY3rvJW0\/P4zqu9yAk\/LvDhlmn7gigw\/z8C"
}
```
@@ -518,6 +519,20 @@ Sample Response:
]
}
```
+## Cross site request forgery(CSRF) prevention in post requests
+In order to prevent CSRF vulnerability, storm rest API uses a CSRF token. This is primarily done for the ui however we
+do not have alternative apis/paths for ui and non ui clients.
+
+The token is generated during the /api/v1/topology/:id (GET) request.The json response for this GET request contains
+a field called "antiForgeryToken". All the post requests below must include a header "x-csrf-token" with the value of
+"antiForgeryToken" from the GET response. In absence of this header with the right token value you will get following
+error response:
+````
+{{
+ "error" : "Forbidden action."
+ "errorMessage" : "missing CSRF token."
+}}
+````
### /api/v1/topology/:id/activate (POST)
activates a topology
http://git-wip-us.apache.org/repos/asf/storm/blob/dd59ad41/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/ui/core.clj b/storm-core/src/clj/backtype/storm/ui/core.clj
index f771ee6..93adb18 100644
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@ -689,7 +689,7 @@
"configuration" topology-conf
"visualizationTable" (stream-boxes visualizer-data)
"uiActionsEnabled" (ui-actions-enabled?)
- "anti-forgery-token" *anti-forgery-token*}))))
+ "antiForgeryToken" *anti-forgery-token*}))))
(defn spout-output-stats
[stream-summary window]
http://git-wip-us.apache.org/repos/asf/storm/blob/dd59ad41/storm-core/src/ui/public/templates/anti-forgery-template.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/templates/anti-forgery-template.html b/storm-core/src/ui/public/templates/anti-forgery-template.html
index 1a63ace..3adcb64 100644
--- a/storm-core/src/ui/public/templates/anti-forgery-template.html
+++ b/storm-core/src/ui/public/templates/anti-forgery-template.html
@@ -15,5 +15,5 @@
limitations under the License.
-->
<script id="anti-forgery-template" type="text/html">
-{{#anti-forgery-token}}{{anti-forgery-token}}{{/anti-forgery-token}}
+{{#antiForgeryToken}}{{antiForgeryToken}}{{/antiForgeryToken}}
</script>
[03/50] storm git commit: Merge branch 'master' into security-upmerge
Posted by bo...@apache.org.
Merge branch 'master' into security-upmerge
Conflicts:
bin/storm
conf/defaults.yaml
storm-core/src/clj/backtype/storm/daemon/drpc.clj
storm-core/src/clj/backtype/storm/daemon/logviewer.clj
storm-core/src/clj/backtype/storm/daemon/nimbus.clj
storm-core/src/clj/backtype/storm/daemon/supervisor.clj
storm-core/src/clj/backtype/storm/daemon/worker.clj
storm-core/src/clj/backtype/storm/testing.clj
storm-core/src/clj/backtype/storm/ui/helpers.clj
storm-core/src/clj/backtype/storm/util.clj
storm-core/src/jvm/backtype/storm/Config.java
storm-core/src/jvm/backtype/storm/utils/Utils.java
storm-core/src/ui/public/component.html
storm-core/test/clj/backtype/storm/supervisor_test.clj
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/559c883d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/559c883d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/559c883d
Branch: refs/heads/master
Commit: 559c883d5331362808b7e1ada647cbac76a88ab3
Parents: ff8336b b2a8a77
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Tue Jul 29 15:57:03 2014 -0500
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Tue Jul 29 15:57:03 2014 -0500
----------------------------------------------------------------------
BYLAWS.md | 96 ++++
CHANGELOG.md | 31 +
LICENSE | 30 +-
README.markdown | 14 +
STORM-UI-REST-API.md | 567 +++++++++++++++++++
bin/storm | 77 ++-
conf/defaults.yaml | 5 +-
conf/storm_env.ini | 2 +-
dev-tools/github/__init__.py | 109 ++++
dev-tools/jira-github-join.py | 80 +++
dev-tools/jira/__init__.py | 232 ++++++++
examples/storm-starter/README.markdown | 30 +-
.../storm-starter/multilang/resources/storm.py | 2 +-
.../src/jvm/storm/starter/RollingTopWords.java | 62 +-
.../src/jvm/storm/starter/util/StormRunner.java | 9 +
.../storm-kafka/src/jvm/storm/kafka/Broker.java | 9 +-
.../src/jvm/storm/kafka/KafkaConfig.java | 2 +-
.../src/jvm/storm/kafka/Partition.java | 9 +-
pom.xml | 6 +-
.../src/clj/backtype/storm/LocalCluster.clj | 7 +-
storm-core/src/clj/backtype/storm/cluster.clj | 31 +-
.../src/clj/backtype/storm/command/monitor.clj | 37 ++
.../src/clj/backtype/storm/daemon/common.clj | 2 +-
.../src/clj/backtype/storm/daemon/drpc.clj | 6 +-
.../src/clj/backtype/storm/daemon/executor.clj | 4 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 7 +-
.../clj/backtype/storm/daemon/supervisor.clj | 104 +++-
.../src/clj/backtype/storm/daemon/worker.clj | 72 +--
storm-core/src/clj/backtype/storm/disruptor.clj | 2 +-
storm-core/src/clj/backtype/storm/event.clj | 2 +-
storm-core/src/clj/backtype/storm/testing.clj | 49 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 2 +-
.../src/clj/backtype/storm/ui/helpers.clj | 5 -
storm-core/src/clj/backtype/storm/util.clj | 52 +-
storm-core/src/clj/backtype/storm/zookeeper.clj | 25 +
.../src/dev/resources/tester_bolt_metrics.py | 35 ++
.../src/dev/resources/tester_spout_metrics.py | 51 ++
storm-core/src/jvm/backtype/storm/Config.java | 171 +++---
.../jvm/backtype/storm/ConfigValidation.java | 70 +++
.../backtype/storm/messaging/netty/Client.java | 13 +-
.../metric/api/rpc/AssignableShellMetric.java | 30 +
.../metric/api/rpc/CombinedShellMetric.java | 31 +
.../storm/metric/api/rpc/CountShellMetric.java | 38 ++
.../storm/metric/api/rpc/IShellMetric.java | 31 +
.../metric/api/rpc/ReducedShellMetric.java | 32 ++
.../storm/multilang/JsonSerializer.java | 15 +
.../jvm/backtype/storm/multilang/ShellMsg.java | 46 ++
.../src/jvm/backtype/storm/spout/ISpout.java | 2 +-
.../jvm/backtype/storm/spout/ShellSpout.java | 72 ++-
.../src/jvm/backtype/storm/task/ShellBolt.java | 75 ++-
.../backtype/storm/task/TopologyContext.java | 28 +
.../storm/testing/PythonShellMetricsBolt.java | 32 ++
.../storm/testing/PythonShellMetricsSpout.java | 35 ++
.../src/jvm/backtype/storm/utils/Monitor.java | 249 ++++++++
.../jvm/backtype/storm/utils/ShellProcess.java | 46 +-
.../src/jvm/backtype/storm/utils/Utils.java | 23 +-
storm-core/src/multilang/py/storm.py | 30 +-
storm-core/src/multilang/rb/storm.rb | 24 +-
storm-core/src/ui/public/component.html | 3 +-
.../src/ui/public/js/jquery.tablesorter.min.js | 9 +-
storm-core/src/ui/public/js/moment.min.js | 6 +
storm-core/src/ui/public/js/script.js | 9 +
.../test/clj/backtype/storm/cluster_test.clj | 3 +-
.../test/clj/backtype/storm/config_test.clj | 41 +-
.../test/clj/backtype/storm/metrics_test.clj | 206 ++++---
.../test/clj/backtype/storm/supervisor_test.clj | 135 ++++-
66 files changed, 3003 insertions(+), 367 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/bin/storm
----------------------------------------------------------------------
diff --cc bin/storm
index 550e01d,a4aadb1..1b5be36
--- a/bin/storm
+++ b/bin/storm
@@@ -449,8 -468,7 +480,8 @@@ COMMANDS = {"jar": jar, "kill": kill, "
"drpc": drpc, "supervisor": supervisor, "localconfvalue": print_localconfvalue,
"remoteconfvalue": print_remoteconfvalue, "repl": repl, "classpath": print_classpath,
"activate": activate, "deactivate": deactivate, "rebalance": rebalance, "help": print_usage,
- "list": listtopos, "dev-zookeeper": dev_zookeeper, "version": version,
- "list": listtopos, "dev-zookeeper": dev_zookeeper, "version": version, "monitor": monitor}
++ "list": listtopos, "dev-zookeeper": dev_zookeeper, "version": version, "monitor": monitor,
+ "upload-credentials": upload_credentials}
def parse_config(config_list):
global CONFIG_OPTS
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/conf/defaults.yaml
----------------------------------------------------------------------
diff --cc conf/defaults.yaml
index d4283a4,2864adc..05948e1
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@@ -188,6 -150,7 +189,8 @@@ topology.max.error.report.per.interval
topology.kryo.factory: "backtype.storm.serialization.DefaultKryoFactory"
topology.tuple.serializer: "backtype.storm.serialization.types.ListDelegateSerializer"
topology.trident.batch.emit.interval.millis: 500
+topology.testing.always.try.serialize: false
+ topology.classpath: null
+ topology.environment: null
dev.zookeeper.path: "/tmp/dev-storm-zookeeper"
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/examples/storm-starter/src/jvm/storm/starter/util/StormRunner.java
----------------------------------------------------------------------
diff --cc examples/storm-starter/src/jvm/storm/starter/util/StormRunner.java
index f916ec6,f3017ce..eb25a86
--- a/examples/storm-starter/src/jvm/storm/starter/util/StormRunner.java
+++ b/examples/storm-starter/src/jvm/storm/starter/util/StormRunner.java
@@@ -19,6 -19,9 +19,10 @@@ package storm.starter.util
import backtype.storm.Config;
import backtype.storm.LocalCluster;
+ import backtype.storm.StormSubmitter;
+ import backtype.storm.generated.AlreadyAliveException;
++import backtype.storm.generated.AuthorizationException;
+ import backtype.storm.generated.InvalidTopologyException;
import backtype.storm.generated.StormTopology;
public final class StormRunner {
@@@ -36,4 -39,9 +40,9 @@@
cluster.killTopology(topologyName);
cluster.shutdown();
}
+
+ public static void runTopologyRemotely(StormTopology topology, String topologyName, Config conf)
- throws AlreadyAliveException, InvalidTopologyException {
++ throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {
+ StormSubmitter.submitTopology(topologyName, conf, topology);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/src/clj/backtype/storm/LocalCluster.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/src/clj/backtype/storm/cluster.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/cluster.clj
index 2c6f6f3,8ff5a2c..8ead710
--- a/storm-core/src/clj/backtype/storm/cluster.clj
+++ b/storm-core/src/clj/backtype/storm/cluster.clj
@@@ -25,15 -23,16 +25,17 @@@
(:require [backtype.storm.daemon [common :as common]]))
(defprotocol ClusterState
- (set-ephemeral-node [this path data])
+ (set-ephemeral-node [this path data acls])
(delete-node [this path])
- (create-sequential [this path data])
+ (create-sequential [this path data acls])
;; if node does not exist, create persistent with this data
- (set-data [this path data])
+ (set-data [this path data acls])
(get-data [this path watch?])
+ (get-version [this path watch?])
+ (get-data-with-version [this path watch?])
(get-children [this path watch?])
- (mkdirs [this path])
+ (mkdirs [this path acls])
+ (exists-node? [this path watch?])
(close [this])
(register [this callback])
(unregister [this id]))
@@@ -240,31 -231,31 +252,33 @@@
(into {}))))
;; Watches should be used for optimization. When ZK is reconnecting, they're not guaranteed to be called.
-(defn mk-storm-cluster-state
- [cluster-state-spec]
+(defnk mk-storm-cluster-state
+ [cluster-state-spec :acls nil]
(let [[solo? cluster-state] (if (satisfies? ClusterState cluster-state-spec)
[false cluster-state-spec]
- [true (mk-distributed-cluster-state cluster-state-spec)])
+ [true (mk-distributed-cluster-state cluster-state-spec :auth-conf cluster-state-spec :acls acls)])
assignment-info-callback (atom {})
+ assignment-info-with-version-callback (atom {})
+ assignment-version-callback (atom {})
supervisors-callback (atom nil)
assignments-callback (atom nil)
storm-base-callback (atom {})
+ credentials-callback (atom {})
state-id (register
- cluster-state
- (fn [type path]
- (let [[subtree & args] (tokenize-path path)]
- (condp = subtree
+ cluster-state
+ (fn [type path]
+ (let [[subtree & args] (tokenize-path path)]
+ (condp = subtree
ASSIGNMENTS-ROOT (if (empty? args)
- (issue-callback! assignments-callback)
- (issue-map-callback! assignment-info-callback (first args)))
+ (issue-callback! assignments-callback)
+ (issue-map-callback! assignment-info-callback (first args)))
SUPERVISORS-ROOT (issue-callback! supervisors-callback)
STORMS-ROOT (issue-map-callback! storm-base-callback (first args))
+ CREDENTIALS-ROOT (issue-map-callback! credentials-callback (first args))
;; this should never happen
- (halt-process! 30 "Unknown callback for subtree " subtree args)))))]
+ (exit-process! 30 "Unknown callback for subtree " subtree args)))))]
(doseq [p [ASSIGNMENTS-SUBTREE STORMS-SUBTREE SUPERVISORS-SUBTREE WORKERBEATS-SUBTREE ERRORS-SUBTREE]]
- (mkdirs cluster-state p))
+ (mkdirs cluster-state p acls))
(reify
StormClusterState
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/src/clj/backtype/storm/daemon/common.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/src/clj/backtype/storm/daemon/drpc.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/drpc.clj
index 1340e49,3527b7c..68128c3
--- a/storm-core/src/clj/backtype/storm/daemon/drpc.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/drpc.clj
@@@ -199,48 -133,37 +199,48 @@@
(let [conf (read-storm-config)
worker-threads (int (conf DRPC-WORKER-THREADS))
queue-size (int (conf DRPC-QUEUE-SIZE))
- service-handler (service-handler)
-
- ;; Requests and returns need to be on separate thread pools, since
- ;; calls to "execute" don't unblock until other thrift methods are
- ;; called. So if 64 threads are calling execute, the server won't
- ;; accept the result invocations that will unblock those threads.
-
- handler-server
- (THsHaServer. (-> (TNonblockingServerSocket. (int (conf DRPC-PORT)))
- (THsHaServer$Args.)
- (.workerThreads 64)
- (.executorService
- (ThreadPoolExecutor.
- worker-threads worker-threads 60 TimeUnit/SECONDS
- (ArrayBlockingQueue. queue-size)))
- (.protocolFactory (TBinaryProtocol$Factory.))
- (.processor (DistributedRPC$Processor. service-handler))))
-
- invoke-server
- (THsHaServer. (-> (TNonblockingServerSocket. (int (conf DRPC-INVOCATIONS-PORT)))
- (THsHaServer$Args.)
- (.workerThreads 64)
- (.protocolFactory (TBinaryProtocol$Factory.))
- (.processor
- (DistributedRPCInvocations$Processor. service-handler))))]
+ drpc-http-port (int (conf DRPC-HTTP-PORT))
+ drpc-port (int (conf DRPC-PORT))
+ drpc-service-handler (service-handler conf)
+ ;; requests and returns need to be on separate thread pools, since calls to
+ ;; "execute" don't unblock until other thrift methods are called. So if
+ ;; 64 threads are calling execute, the server won't accept the result
+ ;; invocations that will unblock those threads
+ handler-server (when (> drpc-port 0)
+ (ThriftServer. conf
+ (DistributedRPC$Processor. drpc-service-handler)
+ ThriftConnectionType/DRPC))
+ invoke-server (ThriftServer. conf
+ (DistributedRPCInvocations$Processor. drpc-service-handler)
+ ThriftConnectionType/DRPC_INVOCATIONS)
+ http-creds-handler (AuthUtils/GetDrpcHttpCredentialsPlugin conf)]
- (.addShutdownHook (Runtime/getRuntime) (Thread. (fn []
- (if handler-server (.stop handler-server))
- (.stop invoke-server))))
+ (add-shutdown-hook-with-force-kill-in-1-sec (fn []
- (.stop handler-server)
++ (if handler-server (.stop handler-server))
+ (.stop invoke-server)))
(log-message "Starting Distributed RPC servers...")
(future (.serve invoke-server))
- (.serve handler-server))))
+ (when (> drpc-http-port 0)
+ (let [app (webapp drpc-service-handler http-creds-handler)
+ filter-class (conf DRPC-HTTP-FILTER)
+ filter-params (conf DRPC-HTTP-FILTER-PARAMS)
+ filters-confs [{:filter-class filter-class
+ :filter-params filter-params}]
+ https-port (int (conf DRPC-HTTPS-PORT))
+ https-ks-path (conf DRPC-HTTPS-KEYSTORE-PATH)
+ https-ks-password (conf DRPC-HTTPS-KEYSTORE-PASSWORD)
+ https-ks-type (conf DRPC-HTTPS-KEYSTORE-TYPE)]
+
+ (run-jetty app
+ {:port drpc-http-port :join? false
+ :configurator (fn [server]
+ (config-ssl server
+ https-port
+ https-ks-path
+ https-ks-password
+ https-ks-type)
+ (config-filter server app filters-confs))})))
+ (when handler-server
+ (.serve handler-server)))))
(defn -main []
(launch-server!))
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/src/clj/backtype/storm/daemon/executor.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/nimbus.clj
index e2952d0,6265479..44ccc6c
--- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
@@@ -81,11 -73,9 +81,11 @@@
:validator (new-instance (conf NIMBUS-TOPOLOGY-VALIDATOR))
:timer (mk-timer :kill-fn (fn [t]
(log-error t "Error when processing event")
- (halt-process! 20 "Error when processing an event")
+ (exit-process! 20 "Error when processing an event")
))
:scheduler (mk-scheduler conf inimbus)
+ :id->sched-status (atom {})
+ :cred-renewers (AuthUtils/GetCredentialRenewers conf)
}))
(defn inbox [nimbus]
@@@ -1304,13 -1148,18 +1304,14 @@@
(defn launch-server! [conf nimbus]
(validate-distributed-mode! conf)
(let [service-handler (service-handler conf nimbus)
- ;;TODO need to honor NIMBUS-THRIFT-MAX-BUFFER-SIZE for different transports
- options (-> (TNonblockingServerSocket. (int (conf NIMBUS-THRIFT-PORT)))
- (THsHaServer$Args.)
- (.workerThreads 64)
- (.protocolFactory (TBinaryProtocol$Factory. false true (conf NIMBUS-THRIFT-MAX-BUFFER-SIZE)))
- (.processor (Nimbus$Processor. service-handler))
- )
- server (THsHaServer. (do (set! (. options maxReadBufferBytes)(conf NIMBUS-THRIFT-MAX-BUFFER-SIZE)) options))]
+ server (ThriftServer. conf (Nimbus$Processor. service-handler)
+ ThriftConnectionType/NIMBUS)]
- (.addShutdownHook (Runtime/getRuntime) (Thread. (fn [] (.shutdown service-handler) (.stop server))))
+ (add-shutdown-hook-with-force-kill-in-1-sec (fn []
+ (.shutdown service-handler)
+ (.stop server)))
(log-message "Starting Nimbus server...")
- (.serve server)))
+ (.serve server)
+ service-handler))
;; distributed implementation
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index 2fe7fb8,cfa8f85..d8ff6b5
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@@ -14,8 -14,8 +14,9 @@@
;; See the License for the specific language governing permissions and
;; limitations under the License.
(ns backtype.storm.daemon.supervisor
+ (:import [java.io OutputStreamWriter BufferedWriter IOException])
- (:import [backtype.storm.scheduler ISupervisor])
+ (:import [backtype.storm.scheduler ISupervisor]
+ [java.net JarURLConnection])
(:use [backtype.storm bootstrap])
(:use [backtype.storm.daemon common])
(:require [backtype.storm.daemon [worker :as worker]])
@@@ -235,22 -190,16 +253,26 @@@
(when thread-pid
(psim/kill-process thread-pid))
(doseq [pid pids]
- (kill-process-with-sig-term pid))
+ (if as-user
++ (worker-launcher-and-wait conf user ["signal" pid "9"] :log-prefix (str "kill -15 " pid))
++ (kill-process-with-sig-term pid)))
+ (if-not (empty? pids) (sleep-secs 1)) ;; allow 1 second for execution of cleanup threads on worker.
+ (doseq [pid pids]
- (force-kill-process pid)
- (try
- (rmpath (worker-pid-path conf id pid))
- (catch Exception e))) ;; on windows, the supervisor may still holds the lock on the worker directory
- (try-cleanup-worker conf id))
++ (if as-user
+ (worker-launcher-and-wait conf user ["signal" pid "9"] :log-prefix (str "kill -9 " pid))
- (ensure-process-killed! pid))
++ (force-kill-process pid))
+ (if as-user
- (rmr-as-user conf id user (worker-pid-path conf id pid))
++ (rmr-as-user conf id user (worker-pid-path conf id pid))
+ (try
+ (rmpath (worker-pid-path conf id pid))
- (catch Exception e)) ;; on windows, the supervisor may still holds the lock on the worker directory
- ))
++ (catch Exception e)))) ;; on windows, the supervisor may still holds the lock on the worker directory
+ (try-cleanup-worker conf id user))
(log-message "Shut down " (:supervisor-id supervisor) ":" id))
+(def SUPERVISOR-ZK-ACLS
+ [(first ZooDefs$Ids/CREATOR_ALL_ACL)
+ (ACL. (bit-or ZooDefs$Perms/READ ZooDefs$Perms/CREATE) ZooDefs$Ids/ANYONE_ID_UNSAFE)])
+
(defn supervisor-data [conf shared-context ^ISupervisor isupervisor]
{:conf conf
:shared-context shared-context
@@@ -582,14 -495,20 +612,23 @@@
jlp (jlp stormroot conf)
stormjar (supervisor-stormjar-path stormroot)
storm-conf (read-supervisor-storm-conf conf storm-id)
- classpath (add-to-classpath (current-classpath) [stormjar])
+ topo-classpath (if-let [cp (storm-conf TOPOLOGY-CLASSPATH)]
+ [cp]
+ [])
+ classpath (-> (current-classpath)
+ (add-to-classpath [stormjar])
+ (add-to-classpath topo-classpath))
+ top-gc-opts (storm-conf TOPOLOGY-WORKER-GC-CHILDOPTS)
+ gc-opts (substitute-childopts (if top-gc-opts top-gc-opts (conf WORKER-GC-CHILDOPTS)) worker-id storm-id port)
+ user (storm-conf TOPOLOGY-SUBMITTER-USER)
+ logfilename (logs-filename storm-id port)
-
- worker-childopts (substitute-childopts (conf WORKER-CHILDOPTS) worker-id storm-id port)
- topo-worker-childopts (substitute-childopts (storm-conf TOPOLOGY-WORKER-CHILDOPTS) worker-id storm-id port)
+ worker-childopts (when-let [s (conf WORKER-CHILDOPTS)]
- (substitute-worker-childopts s port))
++ (substitute-childopts s worker-id storm-id port))
+ topo-worker-childopts (when-let [s (storm-conf TOPOLOGY-WORKER-CHILDOPTS)]
- (substitute-worker-childopts s port))
++ (substitute-childopts s worker-id storm-id port))
+ topology-worker-environment (if-let [env (storm-conf TOPOLOGY-ENVIRONMENT)]
+ (merge env {"LD_LIBRARY_PATH" jlp})
+ {"LD_LIBRARY_PATH" jlp})
- logfilename (str "worker-" port ".log")
command (concat
[(java-cmd) "-server"]
worker-childopts
@@@ -608,21 -526,13 +647,20 @@@
(:assignment-id supervisor)
port
worker-id])
- command (->> command (map str) (filter (complement empty?)))
- shell-cmd (->> command
- (map #(str \' (clojure.string/escape % {\' "\\'"}) \'))
- (clojure.string/join " "))]
- (log-message "Launching worker with command: " shell-cmd)
- (launch-process command :environment topology-worker-environment)
- ))
+ command (->> command (map str) (filter (complement empty?)))]
-
+ (log-message "Launching worker with command: " (shell-cmd command))
+ (write-log-metadata! storm-conf user worker-id storm-id port conf)
+ (set-worker-user! conf worker-id user)
+ (let [log-prefix (str "Worker Process " worker-id)
+ callback (fn [exit-code]
+ (log-message log-prefix " exited with code: " exit-code)
+ (add-dead-worker worker-id))]
+ (remove-dead-worker worker-id)
+ (if run-worker-as-user
+ (let [worker-dir (worker-root conf worker-id)]
- (worker-launcher conf user ["worker" worker-dir (write-script worker-dir command :environment {"LD_LIBRARY_PATH" jlp})] :log-prefix log-prefix :exit-code-callback callback))
- (launch-process command :environment {"LD_LIBRARY_PATH" jlp} :log-prefix log-prefix :exit-code-callback callback)
++ (worker-launcher conf user ["worker" worker-dir (write-script worker-dir command :environment topology-worker-environment)] :log-prefix log-prefix :exit-code-callback callback))
++ (launch-process command :environment topology-worker-environment :log-prefix log-prefix :exit-code-callback callback)
+ ))))
;; local implementation
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/src/clj/backtype/storm/daemon/worker.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/worker.clj
index d09ea35,aeabdf6..f1acdec
--- a/storm-core/src/clj/backtype/storm/daemon/worker.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/worker.clj
@@@ -190,17 -175,22 +191,19 @@@
)
:timer-name timer-name))
- (defn recursive-map-worker-data [conf mq-context storm-id assignment-id port
- storm-conf
- worker-id
- cluster-state
- storm-cluster-state
- executors
- transfer-queue
- executor-receive-queue-map
- receive-queue-map
- topology]
- (recursive-map
-(defn worker-data [conf mq-context storm-id assignment-id port worker-id]
++(defn worker-data [conf mq-context storm-id assignment-id port worker-id storm-conf cluster-state storm-cluster-state]
+ (let [assignment-versions (atom {})
- cluster-state (cluster/mk-distributed-cluster-state conf)
- storm-cluster-state (cluster/mk-storm-cluster-state cluster-state)
- storm-conf (read-supervisor-storm-conf conf storm-id)
+ executors (set (read-worker-executors storm-conf storm-cluster-state storm-id assignment-id port assignment-versions))
+ transfer-queue (disruptor/disruptor-queue "worker-transfer-queue" (storm-conf TOPOLOGY-TRANSFER-BUFFER-SIZE)
+ :wait-strategy (storm-conf TOPOLOGY-DISRUPTOR-WAIT-STRATEGY))
+ executor-receive-queue-map (mk-receive-queue-map storm-conf executors)
+
+ receive-queue-map (->> executor-receive-queue-map
+ (mapcat (fn [[e queue]] (for [t (executor-id->tasks e)] [t queue])))
+ (into {}))
+
+ topology (read-supervisor-topology conf storm-id)]
+ (recursive-map
:conf conf
:mq-context (if mq-context
mq-context
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/src/clj/backtype/storm/testing.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/testing.clj
index 39f3759,54f40e0..f603086
--- a/storm-core/src/clj/backtype/storm/testing.clj
+++ b/storm-core/src/clj/backtype/storm/testing.clj
@@@ -208,8 -208,8 +213,8 @@@
supervisors
; because a worker may already be dead
workers)]
- (while-timeout TEST-TIMEOUT-MS (not (every? (memfn waiting?) daemons))
+ (while-timeout timeout-ms (not (every? (memfn waiting?) daemons))
- (Thread/sleep 10)
+ (Thread/sleep (rand-int 20))
;; (doseq [d daemons]
;; (if-not ((memfn waiting?) d)
;; (println d)))
@@@ -481,10 -483,7 +490,10 @@@
(submit-local-topology (:nimbus cluster-map) storm-name storm-conf topology)
(let [storm-id (common/get-storm-id state storm-name)]
+ ;;Give the topology time to come up without using it to wait for the spouts to complete
+ (simulate-wait cluster-map)
+
- (while-timeout TEST-TIMEOUT-MS (not (every? exhausted? (spout-objects spouts)))
+ (while-timeout timeout-ms (not (every? exhausted? (spout-objects spouts)))
(simulate-wait cluster-map))
(.killTopologyWithOpts (:nimbus cluster-map) storm-name (doto (KillOptions.) (.set_wait_secs 0)))
@@@ -583,22 -582,25 +592,24 @@@
(defn tracked-wait
"Waits until topology is idle and 'amt' more tuples have been emitted by spouts."
([tracked-topology]
- (tracked-wait tracked-topology 1))
+ (tracked-wait tracked-topology 1 TEST-TIMEOUT-MS))
([tracked-topology amt]
+ (tracked-wait tracked-topology amt TEST-TIMEOUT-MS))
+ ([tracked-topology amt timeout-ms]
- (let [target (+ amt @(:last-spout-emit tracked-topology))
- track-id (-> tracked-topology :cluster ::track-id)
- waiting? (fn []
- (or (not= target (global-amt track-id "spout-emitted"))
- (not= (global-amt track-id "transferred")
- (global-amt track-id "processed"))
- ))]
- (while-timeout TEST-TIMEOUT-MS (waiting?)
- ;; (println "Spout emitted: " (global-amt track-id "spout-emitted"))
- ;; (println "Processed: " (global-amt track-id "processed"))
- ;; (println "Transferred: " (global-amt track-id "transferred"))
- (Thread/sleep 500))
- (reset! (:last-spout-emit tracked-topology) target))))
-
-(defnk test-tuple
+ (let [target (+ amt @(:last-spout-emit tracked-topology))
+ track-id (-> tracked-topology :cluster ::track-id)
+ waiting? (fn []
+ (or (not= target (global-amt track-id "spout-emitted"))
+ (not= (global-amt track-id "transferred")
+ (global-amt track-id "processed"))))]
- (while-timeout TEST-TIMEOUT-MS (waiting?)
++ (while-timeout timeout-ms (waiting?)
+ ;; (println "Spout emitted: " (global-amt track-id "spout-emitted"))
+ ;; (println "Processed: " (global-amt track-id "processed"))
+ ;; (println "Transferred: " (global-amt track-id "transferred"))
+ (Thread/sleep (rand-int 200)))
+ (reset! (:last-spout-emit tracked-topology) target))))
+
+(defnk test-tuple
[values
:stream Utils/DEFAULT_STREAM_ID
:component "component"
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/ui/core.clj
index 3c1be09,13e4d41..ec9759c
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@@ -728,10 -688,10 +728,10 @@@
reverse)]
{"componentErrors"
(for [^ErrorInfo e errors]
- {"time" (date-str (.get_error_time_secs e))
+ {"time" (* 1000 (long (.get_error_time_secs e)))
"errorHost" (.get_host e)
"errorPort" (.get_port e)
- "errorWorkerLogLink" (worker-log-link (.get_host e) (.get_port e))
+ "errorWorkerLogLink" (worker-log-link (.get_host e) (.get_port e) topology-id)
"error" (.get_error e)})}))
(defn spout-stats
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/src/clj/backtype/storm/ui/helpers.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/src/clj/backtype/storm/util.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/util.clj
index ed2e30b,3df25b7..0173081
--- a/storm-core/src/clj/backtype/storm/util.clj
+++ b/storm-core/src/clj/backtype/storm/util.clj
@@@ -398,25 -411,30 +414,35 @@@
(catch ExecuteException e
(log-message "Error when trying to kill " pid ". Process is probably already dead."))))
+(defn read-and-log-stream
+ [prefix stream]
+ (try
+ (let [reader (BufferedReader. (InputStreamReader. stream))]
+ (loop []
+ (if-let [line (.readLine reader)]
+ (do
+ (log-warn (str prefix ":" line))
+ (recur)))))
+ (catch IOException e
+ (log-warn "Error while trying to log stream" e))))
+
- (defn sleep-secs [secs]
- (when (pos? secs)
- (Time/sleep (* (long secs) 1000))))
+ (defn force-kill-process
+ [pid]
+ (send-signal-to-process pid sig-kill))
- (defn sleep-until-secs [target-secs]
- (Time/sleepUntil (* (long target-secs) 1000)))
+ (defn kill-process-with-sig-term
+ [pid]
+ (send-signal-to-process pid sig-term))
+
+ (defn add-shutdown-hook-with-force-kill-in-1-sec
+ "adds the user supplied function as a shutdown hook for cleanup.
+ Also adds a function that sleeps for a second and then sends kill -9 to process to avoid any zombie process in case
+ cleanup function hangs."
+ [func]
+ (.addShutdownHook (Runtime/getRuntime) (Thread. #(func)))
+ (.addShutdownHook (Runtime/getRuntime) (Thread. #((sleep-secs 1)
+ (.halt (Runtime/getRuntime) 20)))))
-(defnk launch-process [command :environment {}]
- (let [builder (ProcessBuilder. command)
- process-env (.environment builder)]
- (doseq [[k v] environment]
- (.put process-env k v))
- (.start builder)))
-
(defprotocol SmartThread
(start [this])
(join [this])
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/src/clj/backtype/storm/zookeeper.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --cc storm-core/src/jvm/backtype/storm/Config.java
index ea54313,ac8b6b6..d6b45ea
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@@ -17,10 -17,8 +17,9 @@@
*/
package backtype.storm;
- import backtype.storm.ConfigValidation;
import backtype.storm.serialization.IKryoDecorator;
import backtype.storm.serialization.IKryoFactory;
+import backtype.storm.utils.Utils;
import com.esotericsoftware.kryo.Serializer;
import java.util.ArrayList;
import java.util.HashMap;
@@@ -66,20 -57,20 +65,20 @@@ public class Config extends HashMap<Str
/**
* Netty based messaging: The max # of retries that a peer will perform when a remote is not accessible
*/
- public static final String STORM_MESSAGING_NETTY_MAX_RETRIES = "storm.messaging.netty.max_retries";
- public static final Object STORM_MESSAGING_NETTY_MAX_RETRIES_SCHEMA = Number.class;
+ public static final String STORM_MESSAGING_NETTY_MAX_RETRIES = "storm.messaging.netty.max_retries";
+ public static final Object STORM_MESSAGING_NETTY_MAX_RETRIES_SCHEMA = ConfigValidation.IntegerValidator;
/**
- * Netty based messaging: The min # of milliseconds that a peer will wait.
+ * Netty based messaging: The min # of milliseconds that a peer will wait.
*/
- public static final String STORM_MESSAGING_NETTY_MIN_SLEEP_MS = "storm.messaging.netty.min_wait_ms";
- public static final Object STORM_MESSAGING_NETTY_MIN_SLEEP_MS_SCHEMA = Number.class;
+ public static final String STORM_MESSAGING_NETTY_MIN_SLEEP_MS = "storm.messaging.netty.min_wait_ms";
+ public static final Object STORM_MESSAGING_NETTY_MIN_SLEEP_MS_SCHEMA = ConfigValidation.IntegerValidator;
/**
- * Netty based messaging: The max # of milliseconds that a peer will wait.
+ * Netty based messaging: The max # of milliseconds that a peer will wait.
*/
- public static final String STORM_MESSAGING_NETTY_MAX_SLEEP_MS = "storm.messaging.netty.max_wait_ms";
- public static final Object STORM_MESSAGING_NETTY_MAX_SLEEP_MS_SCHEMA = Number.class;
+ public static final String STORM_MESSAGING_NETTY_MAX_SLEEP_MS = "storm.messaging.netty.max_wait_ms";
+ public static final Object STORM_MESSAGING_NETTY_MAX_SLEEP_MS_SCHEMA = ConfigValidation.IntegerValidator;
/**
* Netty based messaging: The # of worker threads for the server.
@@@ -103,9 -94,9 +102,8 @@@
* We check with this interval that whether the Netty channel is writable and try to write pending messages
*/
public static final String STORM_NETTY_FLUSH_CHECK_INTERVAL_MS = "storm.messaging.netty.flush.check.interval.ms";
- public static final Object STORM_NETTY_FLUSH_CHECK_INTERVAL_MS_SCHEMA = Number.class;
-
-
+ public static final Object STORM_NETTY_FLUSH_CHECK_INTERVAL_MS_SCHEMA = ConfigValidation.IntegerValidator;
+
-
/**
* A list of hosts of ZooKeeper servers used to manage the cluster.
*/
@@@ -234,10 -200,10 +232,10 @@@
* The ceiling of the interval between retries of a Zookeeper operation.
*/
public static final String STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING="storm.zookeeper.retry.intervalceiling.millis";
- public static final Object STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING_SCHEMA = Number.class;
+ public static final Object STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING_SCHEMA = ConfigValidation.IntegerValidator;
/**
- * The Zookeeper authentication scheme to use, e.g. "digest". Defaults to no authentication.
+ * The cluster Zookeeper authentication scheme to use, e.g. "digest". Defaults to no authentication.
*/
public static final String STORM_ZOOKEEPER_AUTH_SCHEME="storm.zookeeper.auth.scheme";
public static final Object STORM_ZOOKEEPER_AUTH_SCHEME_SCHEMA = String.class;
@@@ -307,35 -231,15 +305,35 @@@
* connect to this port to upload jars and submit topologies.
*/
public static final String NIMBUS_THRIFT_PORT = "nimbus.thrift.port";
- public static final Object NIMBUS_THRIFT_PORT_SCHEMA = Number.class;
+ public static final Object NIMBUS_THRIFT_PORT_SCHEMA = ConfigValidation.IntegerValidator;
/**
+ * The number of threads that should be used by the nimbus thrift server.
+ */
+ public static final String NIMBUS_THRIFT_THREADS = "nimbus.thrift.threads";
+ public static final Object NIMBUS_THRIFT_THREADS_SCHEMA = Number.class;
+
+ /**
+ * A list of users that are cluster admins and can run any command. To use this set
+ * nimbus.authorizer to backtype.storm.security.auth.authorizer.SimpleACLAuthorizer
+ */
+ public static final String NIMBUS_ADMINS = "nimbus.admins";
+ public static final Object NIMBUS_ADMINS_SCHEMA = ConfigValidation.StringsValidator;
+
+ /**
+ * A list of users that run the supervisors and should be authorized to interact with
+ * nimbus as a supervisor would. To use this set
+ * nimbus.authorizer to backtype.storm.security.auth.authorizer.SimpleACLAuthorizer
+ */
+ public static final String NIMBUS_SUPERVISOR_USERS = "nimbus.supervisor.users";
+ public static final Object NIMBUS_SUPERVISOR_USERS_SCHEMA = ConfigValidation.StringsValidator;
+
+ /**
* The maximum buffer size thrift should use when reading messages.
*/
public static final String NIMBUS_THRIFT_MAX_BUFFER_SIZE = "nimbus.thrift.max_buffer_size";
- public static final Object NIMBUS_THRIFT_MAX_BUFFER_SIZE_SCHEMA = Number.class;
+ public static final Object NIMBUS_THRIFT_MAX_BUFFER_SIZE_SCHEMA = ConfigValidation.IntegerValidator;
-
/**
* This parameter is used by the storm-deploy project to configure the
* jvm options for the nimbus daemon.
@@@ -395,10 -299,10 +393,10 @@@
* to launching new JVM's and configuring them.</p>
*/
public static final String NIMBUS_TASK_LAUNCH_SECS = "nimbus.task.launch.secs";
- public static final Object NIMBUS_TASK_LAUNCH_SECS_SCHEMA = Number.class;
+ public static final Object NIMBUS_TASK_LAUNCH_SECS_SCHEMA = ConfigValidation.IntegerValidator;
/**
- * Whether or not nimbus should reassign tasks if it detects that a task goes down.
+ * Whether or not nimbus should reassign tasks if it detects that a task goes down.
* Defaults to true, and it's not recommended to change this value.
*/
public static final String NIMBUS_REASSIGN = "nimbus.reassign";
@@@ -565,77 -369,27 +563,77 @@@
* This port is used by Storm DRPC for receiving DPRC requests from clients.
*/
public static final String DRPC_PORT = "drpc.port";
- public static final Object DRPC_PORT_SCHEMA = Number.class;
+ public static final Object DRPC_PORT_SCHEMA = ConfigValidation.IntegerValidator;
/**
- * DRPC thrift server worker threads
+ * Class name for authorization plugin for DRPC client
+ */
+ public static final String DRPC_AUTHORIZER = "drpc.authorizer";
+ public static final Object DRPC_AUTHORIZER_SCHEMA = String.class;
+
+ /**
+ * The Access Control List for the DRPC Authorizer.
+ * @see DRPCSimpleAclAuthorizer
+ */
+ public static final String DRPC_AUTHORIZER_ACL = "drpc.authorizer.acl";
+ public static final Object DRPC_AUTHORIZER_ACL_SCHEMA = Map.class;
+
+ /**
+ * File name of the DRPC Authorizer ACL.
+ * @see DRPCSimpleAclAuthorizer
+ */
+ public static final String DRPC_AUTHORIZER_ACL_FILENAME = "drpc.authorizer.acl.filename";
+ public static final Object DRPC_AUTHORIZER_ACL_FILENAME_SCHEMA = String.class;
+
+ /**
+ * Whether the DRPCSimpleAclAuthorizer should deny requests for operations
+ * involving functions that have no explicit ACL entry. When set to false
+ * (the default) DRPC functions that have no entry in the ACL will be
+ * permitted, which is appropriate for a development environment. When set
+ * to true, explicit ACL entries are required for every DRPC function, and
+ * any request for functions will be denied.
+ * @see DRPCSimpleAclAuthorizer
+ */
+ public static final String DRPC_AUTHORIZER_ACL_STRICT = "drpc.authorizer.acl.strict";
+ public static final Object DRPC_AUTHORIZER_ACL_STRICT_SCHEMA = Boolean.class;
+
+ /**
+ * DRPC thrift server worker threads
*/
public static final String DRPC_WORKER_THREADS = "drpc.worker.threads";
- public static final Object DRPC_WORKER_THREADS_SCHEMA = Number.class;
+ public static final Object DRPC_WORKER_THREADS_SCHEMA = ConfigValidation.IntegerValidator;
/**
- * DRPC thrift server queue size
+ * The maximum buffer size thrift should use when reading messages for DRPC.
+ */
+ public static final String DRPC_MAX_BUFFER_SIZE = "drpc.max_buffer_size";
+ public static final Object DRPC_MAX_BUFFER_SIZE_SCHEMA = Number.class;
+
+ /**
+ * DRPC thrift server queue size
*/
public static final String DRPC_QUEUE_SIZE = "drpc.queue.size";
- public static final Object DRPC_QUEUE_SIZE_SCHEMA = Number.class;
+ public static final Object DRPC_QUEUE_SIZE_SCHEMA = ConfigValidation.IntegerValidator;
/**
- * This port on Storm DRPC is used by DRPC topologies to receive function invocations and send results back.
+ * The DRPC invocations transport plug-in for Thrift client/server communication
+ */
+ public static final String DRPC_INVOCATIONS_THRIFT_TRANSPORT_PLUGIN = "drpc.invocations.thrift.transport";
+ public static final Object DRPC_INVOCATIONS_THRIFT_TRANSPORT_PLUGIN_SCHEMA = String.class;
+
+ /**
+ * This port on Storm DRPC is used by DRPC topologies to receive function invocations and send results back.
*/
public static final String DRPC_INVOCATIONS_PORT = "drpc.invocations.port";
- public static final Object DRPC_INVOCATIONS_PORT_SCHEMA = Number.class;
+ public static final Object DRPC_INVOCATIONS_PORT_SCHEMA = ConfigValidation.IntegerValidator;
/**
+ * DRPC invocations thrift server worker threads
+ */
+ public static final String DRPC_INVOCATIONS_THREADS = "drpc.invocations.threads";
+ public static final Object DRPC_INVOCATIONS_THREADS_SCHEMA = Number.class;
+
+ /**
* The timeout on DRPC requests within the DRPC server. Defaults to 10 minutes. Note that requests can also
* timeout based on the socket timeout on the DRPC client, and separately based on the topology message
* timeout for the topology implementing the DRPC function.
@@@ -672,32 -414,8 +670,32 @@@
* how many workers run on each machine.
*/
public static final String SUPERVISOR_SLOTS_PORTS = "supervisor.slots.ports";
- public static final Object SUPERVISOR_SLOTS_PORTS_SCHEMA = ConfigValidation.NumbersValidator;
+ public static final Object SUPERVISOR_SLOTS_PORTS_SCHEMA = ConfigValidation.IntegersValidator;
+ /**
+ * A number representing the maximum number of workers any single topology can acquire.
+ */
+ public static final String NIMBUS_SLOTS_PER_TOPOLOGY = "nimbus.slots.perTopology";
+ public static final Object NIMBUS_SLOTS_PER_TOPOLOGY_SCHEMA = Number.class;
+
+ /**
+ * A class implementing javax.servlet.Filter for DRPC HTTP requests
+ */
+ public static final String DRPC_HTTP_FILTER = "drpc.http.filter";
+ public static final Object DRPC_HTTP_FILTER_SCHEMA = String.class;
+
+ /**
+ * Initialization parameters for the javax.servlet.Filter of the DRPC HTTP
+ * service
+ */
+ public static final String DRPC_HTTP_FILTER_PARAMS = "drpc.http.filter.params";
+ public static final Object DRPC_HTTP_FILTER_PARAMS_SCHEMA = Map.class;
+
+ /**
+ * A number representing the maximum number of executors any single topology can acquire.
+ */
+ public static final String NIMBUS_EXECUTORS_PER_TOPOLOGY = "nimbus.executors.perTopology";
+ public static final Object NIMBUS_EXECUTORS_PER_TOPOLOGY_SCHEMA = Number.class;
/**
* This parameter is used by the storm-deploy project to configure the
@@@ -711,8 -430,9 +709,8 @@@
* restart the worker process.
*/
public static final String SUPERVISOR_WORKER_TIMEOUT_SECS = "supervisor.worker.timeout.secs";
- public static final Object SUPERVISOR_WORKER_TIMEOUT_SECS_SCHEMA = Number.class;
+ public static final Object SUPERVISOR_WORKER_TIMEOUT_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
/**
* How long a worker can go without heartbeating during the initial launch before
* the supervisor tries to restart the worker process. This value override
@@@ -720,8 -440,9 +718,8 @@@
* overhead to starting and configuring the JVM on launch.
*/
public static final String SUPERVISOR_WORKER_START_TIMEOUT_SECS = "supervisor.worker.start.timeout.secs";
- public static final Object SUPERVISOR_WORKER_START_TIMEOUT_SECS_SCHEMA = Number.class;
+ public static final Object SUPERVISOR_WORKER_START_TIMEOUT_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
/**
* Whether or not the supervisor should launch workers assigned to it. Defaults
* to true -- and you should probably never change this value. This configuration
@@@ -742,25 -464,11 +740,25 @@@
* need to be restarted.
*/
public static final String SUPERVISOR_MONITOR_FREQUENCY_SECS = "supervisor.monitor.frequency.secs";
- public static final Object SUPERVISOR_MONITOR_FREQUENCY_SECS_SCHEMA = Number.class;
+ public static final Object SUPERVISOR_MONITOR_FREQUENCY_SECS_SCHEMA = ConfigValidation.IntegerValidator;
/**
+ * Should the supervior try to run the worker as the lauching user or not. Defaults to false.
+ */
+ public static final String SUPERVISOR_RUN_WORKER_AS_USER = "supervisor.run.worker.as.user";
+ public static final Object SUPERVISOR_RUN_WORKER_AS_USER_SCHEMA = Boolean.class;
+
+ /**
+ * Full path to the worker-laucher executable that will be used to lauch workers when
+ * SUPERVISOR_RUN_WORKER_AS_USER is set to true.
+ */
+ public static final String SUPERVISOR_WORKER_LAUNCHER = "supervisor.worker.launcher";
+ public static final Object SUPERVISOR_WORKER_LAUNCHER_SCHEMA = String.class;
+
+ /**
* The jvm opts provided to workers launched by this supervisor. All "%ID%" substrings are replaced
- * with an identifier for this worker.
+ * with an identifier for this worker. Also, "%WORKER-ID%", "%STORM-ID%" and "%WORKER-PORT%" are
+ * replaced with appropriate runtime values for this worker.
*/
public static final String WORKER_CHILDOPTS = "worker.childopts";
public static final Object WORKER_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
@@@ -800,22 -500,9 +798,22 @@@
* come through.
*/
public static final String TASK_REFRESH_POLL_SECS = "task.refresh.poll.secs";
- public static final Object TASK_REFRESH_POLL_SECS_SCHEMA = Number.class;
+ public static final Object TASK_REFRESH_POLL_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+ /**
+ * How often a task should sync credentials, worst case.
+ */
+ public static final String TASK_CREDENTIALS_POLL_SECS = "task.credentials.poll.secs";
+ public static final Object TASK_CREDENTIALS_POLL_SECS_SCHEMA = Number.class;
+
+
+ /**
+ * A list of users that are allowed to interact with the topology. To use this set
+ * nimbus.authorizer to backtype.storm.security.auth.authorizer.SimpleACLAuthorizer
+ */
+ public static final String TOPOLOGY_USERS = "topology.users";
+ public static final Object TOPOLOGY_USERS_SCHEMA = ConfigValidation.StringsValidator;
/**
* True if Storm should timeout messages or not. Defaults to true. This is meant to be used
@@@ -936,15 -623,15 +934,15 @@@
/**
- * The maximum number of tuples that can be pending on a spout task at any given time.
- * This config applies to individual tasks, not to spouts or topologies as a whole.
- *
+ * The maximum number of tuples that can be pending on a spout task at any given time.
+ * This config applies to individual tasks, not to spouts or topologies as a whole.
+ *
* A pending tuple is one that has been emitted from a spout but has not been acked or failed yet.
- * Note that this config parameter has no effect for unreliable spouts that don't tag
+ * Note that this config parameter has no effect for unreliable spouts that don't tag
* their tuples with a message id.
*/
- public static final String TOPOLOGY_MAX_SPOUT_PENDING="topology.max.spout.pending";
- public static final Object TOPOLOGY_MAX_SPOUT_PENDING_SCHEMA = Number.class;
+ public static final String TOPOLOGY_MAX_SPOUT_PENDING="topology.max.spout.pending";
+ public static final Object TOPOLOGY_MAX_SPOUT_PENDING_SCHEMA = ConfigValidation.IntegerValidator;
/**
* A class that implements a strategy for what to do when a spout needs to wait. Waiting is
@@@ -973,13 -660,13 +971,13 @@@
* The percentage of tuples to sample to produce stats for a task.
*/
public static final String TOPOLOGY_STATS_SAMPLE_RATE="topology.stats.sample.rate";
- public static final Object TOPOLOGY_STATS_SAMPLE_RATE_SCHEMA = Number.class;
+ public static final Object TOPOLOGY_STATS_SAMPLE_RATE_SCHEMA = ConfigValidation.DoubleValidator;
/**
- * The time period that builtin metrics data in bucketed into.
+ * The time period that builtin metrics data in bucketed into.
*/
public static final String TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS="topology.builtin.metrics.bucket.size.secs";
- public static final Object TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS_SCHEMA = Number.class;
+ public static final Object TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS_SCHEMA = ConfigValidation.IntegerValidator;
/**
* Whether or not to use Java serialization in a topology.
@@@ -994,12 -681,19 +992,25 @@@
public static final Object TOPOLOGY_WORKER_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
/**
+ * Topology-specific options GC for the worker child process. This overrides WORKER_GC_CHILDOPTS.
+ */
+ public static final String TOPOLOGY_WORKER_GC_CHILDOPTS="topology.worker.gc.childopts";
+ public static final Object TOPOLOGY_WORKER_GC_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
+
+ /**
+ * Topology-specific classpath for the worker child process. This is combined to the usual classpath.
+ */
+ public static final String TOPOLOGY_CLASSPATH="topology.classpath";
+ public static final Object TOPOLOGY_CLASSPATH_SCHEMA = ConfigValidation.StringOrStringListValidator;
+
+ /**
+ * Topology-specific environment variables for the worker child process.
+ * This is added to the existing environment (that of the supervisor)
+ */
+ public static final String TOPOLOGY_ENVIRONMENT="topology.environment";
+ public static final Object TOPOLOGY_ENVIRONMENT_SCHEMA = Map.class;
+
+ /**
* This config is available for TransactionalSpouts, and contains the id ( a String) for
* the transactional topology. This id is used to store the state of the transactional
* topology in Zookeeper.
@@@ -1061,10 -755,10 +1072,10 @@@
* via the TopologyContext.
*/
public static final String TOPOLOGY_WORKER_SHARED_THREAD_POOL_SIZE="topology.worker.shared.thread.pool.size";
- public static final Object TOPOLOGY_WORKER_SHARED_THREAD_POOL_SIZE_SCHEMA = Number.class;
+ public static final Object TOPOLOGY_WORKER_SHARED_THREAD_POOL_SIZE_SCHEMA = ConfigValidation.IntegerValidator;
/**
- * The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example,
+ * The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example,
* an interval of 10 seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be
* reported to Zookeeper per task for every 10 second interval of time.
*/
@@@ -1183,25 -853,27 +1194,41 @@@
* to backtype.storm.scheduler.IsolationScheduler to make use of the isolation scheduler.
*/
public static final String ISOLATION_SCHEDULER_MACHINES = "isolation.scheduler.machines";
- public static final Object ISOLATION_SCHEDULER_MACHINES_SCHEMA = Map.class;
+ public static final Object ISOLATION_SCHEDULER_MACHINES_SCHEMA = ConfigValidation.MapOfStringToNumberValidator;
+
+ /**
+ * A map from the user name to the number of machines that should that user is allowed to use. Set storm.scheduler
+ * to backtype.storm.scheduler.multitenant.MultitenantScheduler
+ */
+ public static final String MULTITENANT_SCHEDULER_USER_POOLS = "multitenant.scheduler.user.pools";
+ public static final Object MULTITENANT_SCHEDULER_USER_POOLS_SCHEMA = ConfigValidation.MapOfStringToNumberValidator;
+
+ /**
+ * The number of machines that should be used by this topology to isolate it from all others. Set storm.scheduler
+ * to backtype.storm.scheduler.multitenant.MultitenantScheduler
+ */
+ public static final String TOPOLOGY_ISOLATED_MACHINES = "topology.isolate.machines";
+ public static final Object TOPOLOGY_ISOLATED_MACHINES_SCHEMA = Number.class;
+ public static void setClasspath(Map conf, String cp) {
+ conf.put(Config.TOPOLOGY_CLASSPATH, cp);
+ }
+
+ public void setClasspath(String cp) {
+ setClasspath(this, cp);
+ }
+
+ public static void setEnvironment(Map conf, Map env) {
+ conf.put(Config.TOPOLOGY_ENVIRONMENT, env);
+ }
+
+ public void setEnvironment(Map env) {
+ setEnvironment(this, env);
+ }
+
public static void setDebug(Map conf, boolean isOn) {
conf.put(Config.TOPOLOGY_DEBUG, isOn);
- }
+ }
public void setDebug(boolean isOn) {
setDebug(this, isOn);
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/src/jvm/backtype/storm/ConfigValidation.java
----------------------------------------------------------------------
diff --cc storm-core/src/jvm/backtype/storm/ConfigValidation.java
index e990921,156ccf8..14394a0
--- a/storm-core/src/jvm/backtype/storm/ConfigValidation.java
+++ b/storm-core/src/jvm/backtype/storm/ConfigValidation.java
@@@ -177,9 -79,79 +177,79 @@@ public class ConfigValidation
/**
* Validates is a list of Maps.
*/
- public static Object MapsValidator = FieldListValidatorFactory(Map.class);
+ public static Object MapsValidator = listFv(Map.class, true);
/**
+ * Validates a Integer.
+ */
+ public static Object IntegerValidator = new FieldValidator() {
+ @Override
+ public void validateField(String name, Object o) throws IllegalArgumentException {
+ if (o == null) {
+ // A null value is acceptable.
+ return;
+ }
+ final long i;
+ if (o instanceof Number &&
+ (i = ((Number)o).longValue()) == ((Number)o).doubleValue()) {
+ if (i <= Integer.MAX_VALUE && i >= Integer.MIN_VALUE) {
+ return;
+ }
+ }
+
+ throw new IllegalArgumentException("Field " + name + " must be an Integer within type range.");
+ }
+ };
+
+ /**
+ * Validates is a list of Integers.
+ */
+ public static Object IntegersValidator = new FieldValidator() {
+ @Override
+ public void validateField(String name, Object field)
+ throws IllegalArgumentException {
+ if (field == null) {
+ // A null value is acceptable.
+ return;
+ }
+ if (field instanceof Iterable) {
+ for (Object o : (Iterable)field) {
+ final long i;
+ if (o instanceof Number &&
+ ((i = ((Number)o).longValue()) == ((Number)o).doubleValue()) &&
+ (i <= Integer.MAX_VALUE && i >= Integer.MIN_VALUE)) {
+ // pass the test
+ } else {
+ throw new IllegalArgumentException(
+ "Each element of the list " + name + " must be an Integer within type range.");
+ }
+ }
+ return;
+ }
+ }
+ };
+
+ /**
+ * Validates a Double.
+ */
+ public static Object DoubleValidator = new FieldValidator() {
+ @Override
+ public void validateField(String name, Object o) throws IllegalArgumentException {
+ if (o == null) {
+ // A null value is acceptable.
+ return;
+ }
+
+ // we can provide a lenient way to convert int/long to double with losing some precision
+ if (o instanceof Number) {
+ return;
+ }
+
+ throw new IllegalArgumentException("Field " + name + " must be an Double.");
+ }
+ };
+
+ /**
* Validates a power of 2.
*/
public static Object PowerOf2Validator = new FieldValidator() {
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/src/jvm/backtype/storm/utils/Utils.java
----------------------------------------------------------------------
diff --cc storm-core/src/jvm/backtype/storm/utils/Utils.java
index 87f1654,364b53f..fff91e6
--- a/storm-core/src/jvm/backtype/storm/utils/Utils.java
+++ b/storm-core/src/jvm/backtype/storm/utils/Utils.java
@@@ -323,18 -314,19 +323,21 @@@ public class Utils
if (null == o) {
return defaultValue;
}
-
- if(o instanceof Long) {
- return ((Long) o ).intValue();
- } else if (o instanceof Integer) {
- return (Integer) o;
- } else if (o instanceof Short) {
- return ((Short) o).intValue();
+
+ if (o instanceof Integer ||
+ o instanceof Short ||
+ o instanceof Byte) {
+ return ((Number) o).intValue();
+ } else if (o instanceof Long) {
+ final long l = (Long) o;
+ if (l <= Integer.MAX_VALUE && l >= Integer.MIN_VALUE) {
+ return (int) l;
+ }
+ } else if (o instanceof String) {
+ return Integer.parseInt((String) o);
- } else {
- throw new IllegalArgumentException("Don't know how to convert " + o + " + to int");
}
+
+ throw new IllegalArgumentException("Don't know how to convert " + o + " to int");
}
public static boolean getBoolean(Object o, boolean defaultValue) {
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/src/ui/public/component.html
----------------------------------------------------------------------
diff --cc storm-core/src/ui/public/component.html
index 092eb40,6353142..803fffb
--- a/storm-core/src/ui/public/component.html
+++ b/storm-core/src/ui/public/component.html
@@@ -26,10 -26,9 +26,11 @@@
<script src="/js/purl.js" type="text/javascript"></script>
<script src="/js/bootstrap-twipsy.js" type="text/javascript"></script>
<script src="/js/script.js" type="text/javascript"></script>
+ <script src="/js/moment.min.js" type="text/javascript"></script>
</head>
<body>
+<div id="ui-user">
+</div>
<h1><a href="/">Storm UI</a></h1>
<div id="component-summary">
</div>
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/src/ui/public/js/script.js
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/test/clj/backtype/storm/cluster_test.clj
----------------------------------------------------------------------
diff --cc storm-core/test/clj/backtype/storm/cluster_test.clj
index f30c6a8,63efd30..7ed1028
--- a/storm-core/test/clj/backtype/storm/cluster_test.clj
+++ b/storm-core/test/clj/backtype/storm/cluster_test.clj
@@@ -220,10 -206,11 +220,11 @@@
(with-inprocess-zookeeper zk-port
(with-simulated-time
(let [state (mk-storm-state zk-port)]
- (.report-error state "a" "1"(local-hostname) 6700 (RuntimeException.))
+ (.report-error state "a" "1" (local-hostname) 6700 (RuntimeException.))
(validate-errors! state "a" "1" ["RuntimeException"])
+ (advance-time-secs! 1)
(.report-error state "a" "1" (local-hostname) 6700 (IllegalArgumentException.))
- (validate-errors! state "a" "1" ["RuntimeException" "IllegalArgumentException"])
+ (validate-errors! state "a" "1" ["IllegalArgumentException" "RuntimeException"])
(doseq [i (range 10)]
(.report-error state "a" "2" (local-hostname) 6700 (RuntimeException.))
(advance-time-secs! 2))
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/test/clj/backtype/storm/config_test.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/559c883d/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --cc storm-core/test/clj/backtype/storm/supervisor_test.clj
index 3dcd275,584f0d9..6b40060
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@@ -311,100 -303,39 +313,137 @@@
mock-worker-id)
(verify-first-call-args-for-indices launch-process
[0]
- exp-args)))))))
+ exp-args))))
+ (testing "testing topology.classpath is added to classpath"
+ (let [topo-cp "/any/path"
+ exp-args (exp-args-fn [] [] (add-to-classpath mock-cp [topo-cp]))
+ mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed}}]
+ (stubbing [read-supervisor-storm-conf {TOPOLOGY-CLASSPATH topo-cp}
+ supervisor-stormdist-root nil
+ supervisor/jlp nil
++ set-worker-user! nil
++ supervisor/write-log-metadata! nil
+ launch-process nil
+ current-classpath "/base"]
+ (supervisor/launch-worker mock-supervisor
+ mock-storm-id
+ mock-port
+ mock-worker-id)
+ (verify-first-call-args-for-indices launch-process
+ [0]
+ exp-args))))
+ (testing "testing topology.environment is added to environment for worker launch"
+ (let [topo-env {"THISVAR" "somevalue" "THATVAR" "someothervalue"}
++ full-env (merge topo-env {"LD_LIBRARY_PATH" nil})
+ exp-args (exp-args-fn [] [] mock-cp)
+ mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed}}]
+ (stubbing [read-supervisor-storm-conf {TOPOLOGY-ENVIRONMENT topo-env}
+ supervisor-stormdist-root nil
+ supervisor/jlp nil
+ launch-process nil
++ set-worker-user! nil
++ supervisor/write-log-metadata! nil
+ current-classpath "/base"]
+ (supervisor/launch-worker mock-supervisor
+ mock-storm-id
+ mock-port
+ mock-worker-id)
+ (verify-first-call-args-for-indices launch-process
+ [2]
- (merge topo-env {"LD_LIBRARY_PATH" nil}))))))))
++ full-env)))))))
+
+(defn rm-r [f]
+ (if (.isDirectory f)
+ (for [sub (.listFiles f)] (rm-r sub))
+ (.delete f)
+ ))
+
+(deftest test-worker-launch-command-run-as-user
+ (testing "*.worker.childopts configuration"
+ (let [mock-port "42"
+ mock-storm-id "fake-storm-id"
+ mock-worker-id "fake-worker-id"
+ mock-cp "mock-classpath'quote-on-purpose"
+ storm-local (str "/tmp/" (UUID/randomUUID))
+ worker-script (str storm-local "/workers/" mock-worker-id "/storm-worker-script.sh")
+ exp-launch ["/bin/worker-launcher"
+ "me"
+ "worker"
+ (str storm-local "/workers/" mock-worker-id)
+ worker-script]
+ exp-script-fn (fn [opts topo-opts]
+ (str "#!/bin/bash\n'export' 'LD_LIBRARY_PATH=';\n\nexec 'java' '-server'"
+ " " (shell-cmd opts)
+ " " (shell-cmd topo-opts)
+ " '-Djava.library.path='"
+ " '-Dlogfile.name=" mock-storm-id "-worker-" mock-port ".log'"
+ " '-Dstorm.home='"
+ " '-Dlogback.configurationFile=/logback/cluster.xml'"
+ " '-Dstorm.id=" mock-storm-id "'"
+ " '-Dworker.id=" mock-worker-id "'"
+ " '-Dworker.port=" mock-port "'"
+ " '-cp' 'mock-classpath'\"'\"'quote-on-purpose'"
+ " 'backtype.storm.daemon.worker'"
+ " '" mock-storm-id "'"
+ " '" mock-port "'"
+ " '" mock-worker-id "';"))]
+ (.mkdirs (io/file storm-local "workers" mock-worker-id))
+ (try
+ (testing "testing *.worker.childopts as strings with extra spaces"
+ (let [string-opts "-Dfoo=bar -Xmx1024m"
+ topo-string-opts "-Dkau=aux -Xmx2048m"
+ exp-script (exp-script-fn ["-Dfoo=bar" "-Xmx1024m"]
+ ["-Dkau=aux" "-Xmx2048m"])
+ mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed
+ STORM-LOCAL-DIR storm-local
+ SUPERVISOR-RUN-WORKER-AS-USER true
+ WORKER-CHILDOPTS string-opts}}]
+ (stubbing [read-supervisor-storm-conf {TOPOLOGY-WORKER-CHILDOPTS
+ topo-string-opts
+ TOPOLOGY-SUBMITTER-USER "me"}
+ add-to-classpath mock-cp
+ supervisor-stormdist-root nil
+ launch-process nil
+ set-worker-user! nil
+ supervisor/java-cmd "java"
+ supervisor/jlp nil
+ supervisor/write-log-metadata! nil]
+ (supervisor/launch-worker mock-supervisor
+ mock-storm-id
+ mock-port
+ mock-worker-id)
+ (verify-first-call-args-for-indices launch-process
+ [0]
+ exp-launch))
+ (is (= (slurp worker-script) exp-script))))
+ (testing "testing *.worker.childopts as list of strings, with spaces in values"
+ (let [list-opts '("-Dopt1='this has a space in it'" "-Xmx1024m")
+ topo-list-opts '("-Dopt2='val with spaces'" "-Xmx2048m")
+ exp-script (exp-script-fn list-opts topo-list-opts)
+ mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed
+ STORM-LOCAL-DIR storm-local
+ SUPERVISOR-RUN-WORKER-AS-USER true
+ WORKER-CHILDOPTS list-opts}}]
+ (stubbing [read-supervisor-storm-conf {TOPOLOGY-WORKER-CHILDOPTS
+ topo-list-opts
+ TOPOLOGY-SUBMITTER-USER "me"}
+ add-to-classpath mock-cp
+ supervisor-stormdist-root nil
+ launch-process nil
+ set-worker-user! nil
+ supervisor/java-cmd "java"
+ supervisor/jlp nil
+ supervisor/write-log-metadata! nil]
+ (supervisor/launch-worker mock-supervisor
+ mock-storm-id
+ mock-port
+ mock-worker-id)
+ (verify-first-call-args-for-indices launch-process
+ [0]
+ exp-launch))
+ (is (= (slurp worker-script) exp-script))))
+(finally (rm-r (io/file storm-local)))
+))))
(deftest test-workers-go-bananas
;; test that multiple workers are started for a port, and test that
@@@ -422,142 -353,59 +461,198 @@@
;; TODO just do reassign, and check that cleans up worker states after killing but doesn't get rid of downloaded code
)
+(deftest test-supervisor-data-acls
+ (testing "supervisor-data uses correct ACLs"
+ (let [scheme "digest"
+ digest "storm:thisisapoorpassword"
+ auth-conf {STORM-ZOOKEEPER-AUTH-SCHEME scheme
+ STORM-ZOOKEEPER-AUTH-PAYLOAD digest}
+ expected-acls supervisor/SUPERVISOR-ZK-ACLS
+ fake-isupervisor (reify ISupervisor
+ (getSupervisorId [this] nil)
+ (getAssignmentId [this] nil))]
+ (stubbing [uptime-computer nil
+ cluster/mk-storm-cluster-state nil
+ supervisor-state nil
+ local-hostname nil
+ mk-timer nil]
+ (supervisor/supervisor-data auth-conf nil fake-isupervisor)
+ (verify-call-times-for cluster/mk-storm-cluster-state 1)
+ (verify-first-call-args-for-indices cluster/mk-storm-cluster-state [2]
+ expected-acls)))))
+
+(deftest test-write-log-metadata
+ (testing "supervisor writes correct data to logs metadata file"
+ (let [exp-owner "alice"
+ exp-worker-id "42"
+ exp-storm-id "0123456789"
+ exp-port 4242
+ exp-logs-users ["bob" "charlie" "daryl"]
+ storm-conf {TOPOLOGY-SUBMITTER-USER "alice"
+ TOPOLOGY-USERS ["charlie" "bob"]
+ LOGS-USERS ["daryl"]}
+ exp-data {TOPOLOGY-SUBMITTER-USER exp-owner
+ "worker-id" exp-worker-id
+ LOGS-USERS exp-logs-users}
+ conf {}]
+ (mocking [supervisor/write-log-metadata-to-yaml-file!]
+ (supervisor/write-log-metadata! storm-conf exp-owner exp-worker-id
+ exp-storm-id exp-port conf)
+ (verify-called-once-with-args supervisor/write-log-metadata-to-yaml-file!
+ exp-storm-id exp-port exp-data conf)))))
+
+(deftest test-worker-launcher-requires-user
+ (testing "worker-launcher throws on blank user"
+ (mocking [launch-process]
+ (is (thrown-cause-with-msg? java.lang.IllegalArgumentException
+ #"(?i).*user cannot be blank.*"
+ (supervisor/worker-launcher {} nil ""))))))
+
+(defn found? [sub-str input-str]
+ (if (string? input-str)
+ (contrib-str/substring? sub-str (str input-str))
+ (some? #(contrib-str/substring? sub-str %) input-str)))
+
+(defn not-found? [sub-str input-str]
+ (complement (found? sub-str input-str)))
+
+(deftest test-substitute-childopts-happy-path
+ (testing "worker-launcher replaces ids in childopts"
+ (let [ worker-id "w-01"
+ storm-id "s-01"
+ port 9999
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log"
+ ]
+ (def childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port))
+ (is (not-found? "%WORKER-ID%" childopts-with-ids))
+ (is (found? "w-01" childopts-with-ids))
+ (is (not-found? "%STORM-ID%" childopts-with-ids))
+ (is (found? "s-01" childopts-with-ids))
+ (is (not-found? "%WORKER-PORT%" childopts-with-ids))
+ (is (found? "-9999." childopts-with-ids))
+ (is (not-found? "%ID%" childopts-with-ids))
+ (is (found? "worker-9999" childopts-with-ids) (str childopts-with-ids))
+ )))
+
+(deftest test-substitute-childopts-storm-id-alone
+ (testing "worker-launcher replaces ids in childopts"
+ (let [ worker-id "w-01"
+ storm-id "s-01"
+ port 9999
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%STORM-ID%.log"]
+ (def childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port))
+ (is (not-found? "%WORKER-ID%" childopts-with-ids))
+ (is (not-found? "w-01" childopts-with-ids))
+ (is (not-found? "%STORM-ID%" childopts-with-ids))
+ (is (found? "s-01" childopts-with-ids))
+ (is (not-found? "%WORKER-PORT%" childopts-with-ids))
+ (is (not-found? "-9999." childopts-with-ids))
+ (is (not-found? "%ID%" childopts-with-ids))
+ (is (not-found? "worker-9999" childopts-with-ids) (str childopts-with-ids)) )))
+
+(deftest test-substitute-childopts-no-keys
+ (testing "worker-launcher has no ids to replace in childopts"
+ (let [ worker-id "w-01"
+ storm-id "s-01"
+ port 9999
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log"]
+ (def childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port))
+ (is (not-found? "%WORKER-ID%" childopts-with-ids))
+ (is (not-found? "w-01" childopts-with-ids))
+ (is (not-found? "%STORM-ID%" childopts-with-ids))
+ (is (not-found? "s-01" childopts-with-ids))
+ (is (not-found? "%WORKER-PORT%" childopts-with-ids))
+ (is (not-found? "-9999." childopts-with-ids))
+ (is (not-found? "%ID%" childopts-with-ids))
+ (is (not-found? "worker-9999" childopts-with-ids) (str childopts-with-ids)) )))
+
+(deftest test-substitute-childopts-nil-childopts
+ (testing "worker-launcher has nil childopts"
+ (let [ worker-id "w-01"
+ storm-id "s-01"
+ port 9999
+ childopts nil]
+ (def childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port))
+ (is (not-found? "%WORKER-ID%" childopts-with-ids))
+ (is (not-found? "w-01" childopts-with-ids))
+ (is (not-found? "%STORM-ID%" childopts-with-ids))
+ (is (not-found? "s-01" childopts-with-ids))
+ (is (not-found? "%WORKER-PORT%" childopts-with-ids))
+ (is (not-found? "-9999." childopts-with-ids))
+ (is (not-found? "%ID%" childopts-with-ids))
+ (is (not-found? "worker-9999" childopts-with-ids) (str childopts-with-ids))
+ )))
+
+(deftest test-substitute-childopts-nil-ids
+ (testing "worker-launcher has nil ids"
+ (let [ worker-id nil
+ storm-id "s-01"
+ port 9999
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log"]
+ (def childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port))
+ (is (not-found? "%WORKER-ID%" childopts-with-ids))
+ (is (not-found? "w-01" childopts-with-ids))
+ (is (not-found? "%STORM-ID%" childopts-with-ids))
+ (is (found? "s-01" childopts-with-ids))
+ (is (not-found? "%WORKER-PORT%" childopts-with-ids))
+ (is (found? "-9999." childopts-with-ids))
+ (is (not-found? "%ID%" childopts-with-ids))
+ (is (found? "worker-9999" childopts-with-ids) (str childopts-with-ids))
+ )))
+
+ (deftest test-retry-read-assignments
+ (with-simulated-time-local-cluster [cluster
+ :supervisors 0
+ :ports-per-supervisor 2
+ :daemon-conf {NIMBUS-REASSIGN false
+ NIMBUS-MONITOR-FREQ-SECS 10
+ TOPOLOGY-MESSAGE-TIMEOUT-SECS 30
+ TOPOLOGY-ACKER-EXECUTORS 0}]
+ (letlocals
+ (bind sup1 (add-supervisor cluster :id "sup1" :ports [1 2 3 4]))
+ (bind topology1 (thrift/mk-topology
+ {"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 2)}
+ {}))
+ (bind topology2 (thrift/mk-topology
+ {"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 2)}
+ {}))
+ (bind state (:storm-cluster-state cluster))
+ (bind changed (capture-changed-workers
+ (submit-mocked-assignment
+ (:nimbus cluster)
+ "topology1"
+ {TOPOLOGY-WORKERS 2}
+ topology1
+ {1 "1"
+ 2 "1"}
+ {[1] ["sup1" 1]
+ [2] ["sup1" 2]
+ })
+ (submit-mocked-assignment
+ (:nimbus cluster)
+ "topology2"
+ {TOPOLOGY-WORKERS 2}
+ topology2
+ {1 "1"
+ 2 "1"}
+ {[1] ["sup1" 1]
+ [2] ["sup1" 2]
+ })
+ (advance-cluster-time cluster 10)
+ ))
+ (is (empty? (:launched changed)))
+ (bind options (RebalanceOptions.))
+ (.set_wait_secs options 0)
+ (bind changed (capture-changed-workers
+ (.rebalance (:nimbus cluster) "topology2" options)
+ (advance-cluster-time cluster 10)
+ (heartbeat-workers cluster "sup1" [1 2 3 4])
+ (advance-cluster-time cluster 10)
+ ))
+ (validate-launched-once (:launched changed)
+ {"sup1" [1 2]}
+ (get-storm-id (:storm-cluster-state cluster) "topology1"))
+ (validate-launched-once (:launched changed)
+ {"sup1" [3 4]}
+ (get-storm-id (:storm-cluster-state cluster) "topology2"))
+ )))
[22/50] storm git commit: Scheduler handles already used slot not
reported by supervisor
Posted by bo...@apache.org.
Scheduler handles already used slot not reported by supervisor
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6e3c77a6
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6e3c77a6
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6e3c77a6
Branch: refs/heads/master
Commit: 6e3c77a69c48c11ce9564b5098953ff7a722e95a
Parents: 7a06ef7
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Thu Aug 28 09:39:36 2014 -0500
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Thu Aug 28 09:39:36 2014 -0500
----------------------------------------------------------------------
.../storm/scheduler/multitenant/Node.java | 20 ++-
.../scheduler/multitenant_scheduler_test.clj | 163 +++++++++++++------
2 files changed, 126 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/6e3c77a6/storm-core/src/jvm/backtype/storm/scheduler/multitenant/Node.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/scheduler/multitenant/Node.java b/storm-core/src/jvm/backtype/storm/scheduler/multitenant/Node.java
index 1c601ca..883c65f 100644
--- a/storm-core/src/jvm/backtype/storm/scheduler/multitenant/Node.java
+++ b/storm-core/src/jvm/backtype/storm/scheduler/multitenant/Node.java
@@ -127,10 +127,19 @@ public class Node {
boolean assignInternal(WorkerSlot ws, String topId, boolean dontThrow) {
validateSlot(ws);
if (!_freeSlots.remove(ws)) {
- if (dontThrow) {
- return true;
+ for (Entry<String, Set<WorkerSlot>> topologySetEntry : _topIdToUsedSlots.entrySet()) {
+ if (topologySetEntry.getValue().contains(ws)) {
+ if (dontThrow) {
+ LOG.warn("Worker slot [" + ws + "] can't be assigned to " + topId +
+ ". Its already assigned to " + topologySetEntry.getKey() + ".");
+ return true;
+ }
+ throw new IllegalStateException("Worker slot [" + ws + "] can't be assigned to "
+ + topId + ". Its already assigned to " + topologySetEntry.getKey() + ".");
+ }
}
- throw new IllegalStateException("Assigning a slot that was not free " + ws);
+ LOG.warn("Adding Worker slot [" + ws + "] that was not reported in the supervisor heartbeats," +
+ " but the worker is already running for topology " + topId + ".");
}
Set<WorkerSlot> usedSlots = _topIdToUsedSlots.get(topId);
if (usedSlots == null) {
@@ -291,7 +300,7 @@ public class Node {
//Node ID and supervisor ID are the same.
String id = sup.getId();
boolean isAlive = !cluster.isBlackListed(id);
- LOG.debug("Found a {} Node {} {}",
+ LOG.debug("Found a {} Node {} {}",
new Object[] {isAlive? "living":"dead", id, sup.getAllPorts()});
nodeIdToNode.put(id, new Node(id, sup.getAllPorts(), isAlive));
}
@@ -311,7 +320,8 @@ public class Node {
node.addOrphanedSlot(ws);
}
if (node.assignInternal(ws, topId, true)) {
- LOG.warn("Bad scheduling state, "+ws+" assigned multiple workers, unassigning everything...");
+ LOG.warn("Bad scheduling state for topology [" + topId+ "], the slot " +
+ ws + " assigned to multiple workers, un-assigning everything...");
node.free(ws, cluster, true);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6e3c77a6/storm-core/test/clj/backtype/storm/scheduler/multitenant_scheduler_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/scheduler/multitenant_scheduler_test.clj b/storm-core/test/clj/backtype/storm/scheduler/multitenant_scheduler_test.clj
index b3cdb13..049a0d6 100644
--- a/storm-core/test/clj/backtype/storm/scheduler/multitenant_scheduler_test.clj
+++ b/storm-core/test/clj/backtype/storm/scheduler/multitenant_scheduler_test.clj
@@ -716,57 +716,116 @@
))
(deftest test-multitenant-scheduler-bad-starting-state
- (let [supers (gen-supervisors 10)
- topology1 (TopologyDetails. "topology1"
- {TOPOLOGY-NAME "topology-name-1"
- TOPOLOGY-SUBMITTER-USER "userC"}
- (StormTopology.)
- 4
- (mk-ed-map [["spout1" 0 5]
- ["bolt1" 5 10]
- ["bolt2" 10 15]
- ["bolt3" 15 20]]))
- topology2 (TopologyDetails. "topology2"
- {TOPOLOGY-NAME "topology-name-2"
- TOPOLOGY-ISOLATED-MACHINES 2
- TOPOLOGY-SUBMITTER-USER "userA"}
- (StormTopology.)
- 4
- (mk-ed-map [["spout11" 0 5]
- ["bolt12" 5 6]
- ["bolt13" 6 7]
- ["bolt14" 7 10]]))
- topology3 (TopologyDetails. "topology3"
- {TOPOLOGY-NAME "topology-name-3"
- TOPOLOGY-ISOLATED-MACHINES 5
- TOPOLOGY-SUBMITTER-USER "userB"}
- (StormTopology.)
- 10
- (mk-ed-map [["spout21" 0 10]
- ["bolt22" 10 20]
- ["bolt23" 20 30]
- ["bolt24" 30 40]]))
- existing-assignments {
- "topology2" (SchedulerAssignmentImpl. "topology2" {(ExecutorDetails. 0 5) (WorkerSlot. "super1" 1)})
- "topology3" (SchedulerAssignmentImpl. "topology3" {(ExecutorDetails. 0 10) (WorkerSlot. "super1" 1)})
- }
- cluster (Cluster. (nimbus/standalone-nimbus) supers existing-assignments)
- topologies (Topologies. (to-top-map [topology1 topology2 topology3]))
- conf {MULTITENANT-SCHEDULER-USER-POOLS {"userA" 5 "userB" 5}}
- scheduler (MultitenantScheduler.)]
- (.prepare scheduler conf)
- (.schedule scheduler topologies cluster)
- (let [assignment (.getAssignmentById cluster "topology1")
- assigned-slots (.getSlots assignment)
- executors (.getExecutors assignment)]
- ;; 4 slots on 1 machine, all executors assigned
- (is (= 4 (.size assigned-slots)))
- (is (= 1 (.size (into #{} (for [slot assigned-slots] (.getNodeId slot))))))
- (is (= 20 (.size executors)))
- )
- (is (= "Fully Scheduled" (.get (.getStatusMap cluster) "topology1")))
- (is (= "Scheduled Isolated on 2 Nodes" (.get (.getStatusMap cluster) "topology2")))
- (is (= "Scheduled Isolated on 5 Nodes" (.get (.getStatusMap cluster) "topology3")))
-))
+ (testing "Assiging same worker slot to different topologies is bad state"
+ (let [supers (gen-supervisors 5)
+ topology1 (TopologyDetails. "topology1"
+ {TOPOLOGY-NAME "topology-name-1"
+ TOPOLOGY-SUBMITTER-USER "userC"}
+ (StormTopology.)
+ 1
+ (mk-ed-map [["spout1" 0 1]]))
+ topology2 (TopologyDetails. "topology2"
+ {TOPOLOGY-NAME "topology-name-2"
+ TOPOLOGY-ISOLATED-MACHINES 2
+ TOPOLOGY-SUBMITTER-USER "userA"}
+ (StormTopology.)
+ 1
+ (mk-ed-map [["spout11" 1 2]]))
+ topology3 (TopologyDetails. "topology3"
+ {TOPOLOGY-NAME "topology-name-3"
+ TOPOLOGY-ISOLATED-MACHINES 1
+ TOPOLOGY-SUBMITTER-USER "userB"}
+ (StormTopology.)
+ 1
+ (mk-ed-map [["spout21" 2 3]]))
+ worker-slot-with-multiple-assignments (WorkerSlot. "super1" 1)
+ existing-assignments {"topology2" (SchedulerAssignmentImpl. "topology2" {(ExecutorDetails. 1 1) worker-slot-with-multiple-assignments})
+ "topology3" (SchedulerAssignmentImpl. "topology3" {(ExecutorDetails. 2 2) worker-slot-with-multiple-assignments})}
+ cluster (Cluster. (nimbus/standalone-nimbus) supers existing-assignments)
+ topologies (Topologies. (to-top-map [topology1 topology2 topology3]))
+ conf {MULTITENANT-SCHEDULER-USER-POOLS {"userA" 2 "userB" 1}}
+ scheduler (MultitenantScheduler.)]
+ (.prepare scheduler conf)
+ (.schedule scheduler topologies cluster)
+ (let [assignment (.getAssignmentById cluster "topology1")
+ assigned-slots (.getSlots assignment)
+ executors (.getExecutors assignment)]
+ (is (= 1 (.size assigned-slots))))
+ (is (= "Fully Scheduled" (.get (.getStatusMap cluster) "topology1")))
+ (is (= "Scheduled Isolated on 2 Nodes" (.get (.getStatusMap cluster) "topology2")))
+ (is (= "Scheduled Isolated on 1 Nodes" (.get (.getStatusMap cluster) "topology3"))))))
+(deftest test-existing-assignment-slot-not-found-in-supervisor
+ (testing "Scheduler should handle discrepancy when a live supervisor heartbeat does not report slot,
+ but worker heartbeat says its running on that slot"
+ (let [supers (gen-supervisors 1)
+ port-not-reported-by-supervisor 6
+ topology1 (TopologyDetails. "topology1"
+ {TOPOLOGY-NAME "topology-name-1"
+ TOPOLOGY-SUBMITTER-USER "userA"}
+ (StormTopology.)
+ 1
+ (mk-ed-map [["spout11" 0 1]]))
+ existing-assignments {"topology1"
+ (SchedulerAssignmentImpl. "topology1"
+ {(ExecutorDetails. 0 0) (WorkerSlot. "super0" port-not-reported-by-supervisor)})}
+ cluster (Cluster. (nimbus/standalone-nimbus) supers existing-assignments)
+ topologies (Topologies. (to-top-map [topology1]))
+ conf {}
+ scheduler (MultitenantScheduler.)]
+ (.prepare scheduler conf)
+ (.schedule scheduler topologies cluster)
+ (let [assignment (.getAssignmentById cluster "topology1")
+ assigned-slots (.getSlots assignment)
+ executors (.getExecutors assignment)]
+ (is (= 1 (.size assigned-slots))))
+ (is (= "Fully Scheduled" (.get (.getStatusMap cluster) "topology1"))))))
+(deftest test-existing-assignment-slot-on-dead-supervisor
+ (testing "Dead supervisor could have slot with duplicate assignments or slot never reported by supervisor"
+ (let [supers (gen-supervisors 1)
+ dead-supervisor "super1"
+ port-not-reported-by-supervisor 6
+ topology1 (TopologyDetails. "topology1"
+ {TOPOLOGY-NAME "topology-name-1"
+ TOPOLOGY-SUBMITTER-USER "userA"}
+ (StormTopology.)
+ 2
+ (mk-ed-map [["spout11" 0 1]
+ ["bolt12" 1 2]]))
+ topology2 (TopologyDetails. "topology2"
+ {TOPOLOGY-NAME "topology-name-2"
+ TOPOLOGY-SUBMITTER-USER "userA"}
+ (StormTopology.)
+ 2
+ (mk-ed-map [["spout21" 4 5]
+ ["bolt22" 5 6]]))
+ worker-slot-with-multiple-assignments (WorkerSlot. dead-supervisor 1)
+ existing-assignments {"topology1"
+ (SchedulerAssignmentImpl. "topology1"
+ {(ExecutorDetails. 0 0) worker-slot-with-multiple-assignments
+ (ExecutorDetails. 1 1) (WorkerSlot. dead-supervisor 3)})
+ "topology2"
+ (SchedulerAssignmentImpl. "topology2"
+ {(ExecutorDetails. 4 4) worker-slot-with-multiple-assignments
+ (ExecutorDetails. 5 5) (WorkerSlot. dead-supervisor port-not-reported-by-supervisor)})}
+ cluster (Cluster. (nimbus/standalone-nimbus) supers existing-assignments)
+ topologies (Topologies. (to-top-map [topology1 topology2]))
+ conf {}
+ scheduler (MultitenantScheduler.)]
+ (.prepare scheduler conf)
+ (.schedule scheduler topologies cluster)
+ (let [assignment (.getAssignmentById cluster "topology1")
+ assigned-slots (.getSlots assignment)
+ executors (.getExecutors assignment)]
+ (is (= 2 (.size assigned-slots)))
+ (is (= 2 (.size (into #{} (for [slot assigned-slots] (.getNodeId slot))))))
+ (is (= 2 (.size executors))))
+ (is (= "Fully Scheduled" (.get (.getStatusMap cluster) "topology1")))
+ (let [assignment (.getAssignmentById cluster "topology2")
+ assigned-slots (.getSlots assignment)
+ executors (.getExecutors assignment)]
+ (is (= 2 (.size assigned-slots)))
+ (is (= 2 (.size (into #{} (for [slot assigned-slots] (.getNodeId slot))))))
+ (is (= 2 (.size executors))))
+ (is (= "Fully Scheduled" (.get (.getStatusMap cluster) "topology2"))))))
[16/50] storm git commit: Merge branch 'STORM-406' of
https://github.com/Parth-Brahmbhatt/incubator-storm into STORM-406
Posted by bo...@apache.org.
Merge branch 'STORM-406' of https://github.com/Parth-Brahmbhatt/incubator-storm into STORM-406
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/029b1e17
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/029b1e17
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/029b1e17
Branch: refs/heads/master
Commit: 029b1e171e726e31af12aa0c3ff9c0336bf7f8ae
Parents: 4ee22b1 5767262
Author: Parth Brahmbhatt <br...@gmail.com>
Authored: Mon Aug 18 08:54:09 2014 -0700
Committer: Parth Brahmbhatt <br...@gmail.com>
Committed: Mon Aug 18 08:54:09 2014 -0700
----------------------------------------------------------------------
----------------------------------------------------------------------