You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2015/01/14 17:05:22 UTC
[01/44] storm git commit: exponential backoff for failed messages
Repository: storm
Updated Branches:
refs/heads/master fca7c7673 -> 94b96087b
exponential backoff for failed messages
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/cac70e77
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/cac70e77
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/cac70e77
Branch: refs/heads/master
Commit: cac70e77d3417e13082495aaac126f25a6d07eff
Parents: 4da836f
Author: Rick Kilgore <ri...@hbo.com>
Authored: Fri Aug 22 21:49:08 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Fri Aug 22 21:49:08 2014 -0700
----------------------------------------------------------------------
.../src/jvm/storm/kafka/PartitionManager.java | 66 ++++++++++++++++++--
.../src/jvm/storm/kafka/SpoutConfig.java | 3 +
2 files changed, 65 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/cac70e77/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
index 9b48678..e80b19a 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
@@ -44,7 +44,8 @@ public class PartitionManager {
private final CountMetric _fetchAPIMessageCount;
Long _emittedToOffset;
SortedSet<Long> _pending = new TreeSet<Long>();
- SortedSet<Long> failed = new TreeSet<Long>();
+ private SortedSet<Long> failed = new TreeSet<Long>();
+ private Map<Long,MessageRetryRecord> retryRecords = new HashMap<Long,MessageRetryRecord>();
Long _committedTo;
LinkedList<MessageAndRealOffset> _waitingToEmit = new LinkedList<MessageAndRealOffset>();
Partition _partition;
@@ -144,14 +145,30 @@ public class PartitionManager {
}
}
+ /**
+ * Fetch the failed messages ready for retry. If there are no failed messages, or none are ready for retry, then it
+ * returns an empty List (i.e., not null).
+ */
+ private SortedSet<Long> failedMsgsReadyForRetry() {
+ SortedSet<Long> ready = new TreeSet<Long>();
+ for (Long offset : this.failed) {
+ if (this.retryRecords.get(offset).isReadyForRetry()) {
+ ready.add(offset);
+ }
+ }
+ return ready;
+ }
+
+
private void fill() {
long start = System.nanoTime();
long offset;
- final boolean had_failed = !failed.isEmpty();
+ final SortedSet<Long> failedReady = failedMsgsReadyForRetry();
// Are there failed tuples? If so, fetch those first.
+ final boolean had_failed = !failedReady.isEmpty();
if (had_failed) {
- offset = failed.first();
+ offset = failedReady.first();
} else {
offset = _emittedToOffset;
}
@@ -171,7 +188,7 @@ public class PartitionManager {
// Skip any old offsets.
continue;
}
- if (!had_failed || failed.contains(cur_offset)) {
+ if (!had_failed || failedReady.contains(cur_offset)) {
numMessages += 1;
_pending.add(cur_offset);
_waitingToEmit.add(new MessageAndRealOffset(msg.message(), cur_offset));
@@ -191,6 +208,7 @@ public class PartitionManager {
_pending.headSet(offset - _spoutConfig.maxOffsetBehind).clear();
}
_pending.remove(offset);
+ retryRecords.remove(offset);
numberAcked++;
}
@@ -204,6 +222,8 @@ public class PartitionManager {
} else {
LOG.debug("failing at offset=" + offset + " with _pending.size()=" + _pending.size() + " pending and _emittedToOffset=" + _emittedToOffset);
failed.add(offset);
+ MessageRetryRecord retryRecord = retryRecords.get(offset);
+ retryRecords.put(offset, retryRecord == null ? new MessageRetryRecord() : retryRecord.retryAgainRecord());
numberFailed++;
if (numberAcked == 0 && numberFailed > _spoutConfig.maxOffsetBehind) {
throw new RuntimeException("Too many tuple failures");
@@ -261,4 +281,42 @@ public class PartitionManager {
this.offset = offset;
}
}
+
+ /**
+ * A MessageRetryRecord holds the data of how many times a message has
+ * failed and been retried, and when the last failure occurred. It can
+ * determine whether it is ready to be retried by employing an exponential
+ * back-off calculation using config values stored in SpoutConfig:
+ * <ul>
+ * <li>retryInitialDelayMs - time to delay before the first retry</li>
+ * <li>retryDelayMultiplier - multiplier by which to increase the delay for each subsequent retry</li>
+ * <li>retryMaxDelayMs - maximum retry delay (once this delay time is reached, subsequent retries will
+ * delay for this amount of time every time)
+ * </li>
+ * </ul>
+ */
+ class MessageRetryRecord {
+ private final long failTimeUTC;
+ private final int attemptsAlreadyPerformed;
+
+ private MessageRetryRecord(int attemptsAlreadyPerformed) {
+ this.failTimeUTC = new Date().getTime();
+ this.attemptsAlreadyPerformed = attemptsAlreadyPerformed;
+ }
+
+ public MessageRetryRecord() {
+ this(1);
+ }
+
+ public MessageRetryRecord retryAgainRecord() {
+ return new MessageRetryRecord(this.attemptsAlreadyPerformed + 1);
+ }
+
+ public boolean isReadyForRetry() {
+ double delayMultiplier = Math.pow(_spoutConfig.retryDelayMultiplier, this.attemptsAlreadyPerformed - 1);
+ long delayThisRetryMs = (long) (_spoutConfig.retryInitialDelayMs * delayMultiplier);
+ delayThisRetryMs = Math.min(delayThisRetryMs, _spoutConfig.retryMaxDelayMs);
+ return new Date().getTime() - this.failTimeUTC > delayThisRetryMs;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/cac70e77/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java b/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
index 1b66026..d8ca7eb 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
@@ -27,6 +27,9 @@ public class SpoutConfig extends KafkaConfig implements Serializable {
public String zkRoot = null;
public String id = null;
public long stateUpdateIntervalMs = 2000;
+ public long retryInitialDelayMs = 10 * 1000;
+ public double retryDelayMultiplier = 1.5;
+ public long retryMaxDelayMs = 5 * 60 * 1000;
public SpoutConfig(BrokerHosts hosts, String topic, String zkRoot, String id) {
super(hosts, topic);
[24/44] storm git commit: update storm-kafka README.md
Posted by bo...@apache.org.
update storm-kafka README.md
Update storm-kafka README.md with info about retry settings.
Also update the sentence on what SpoutConfig provides over KafkaConfig.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/77c2df28
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/77c2df28
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/77c2df28
Branch: refs/heads/master
Commit: 77c2df28517f9b9acd87724116d9b7b3573be4fc
Parents: 8c78c13
Author: Rick Kilgore <ri...@hbo.com>
Authored: Thu Sep 11 17:29:05 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Thu Sep 11 17:29:05 2014 -0700
----------------------------------------------------------------------
external/storm-kafka/README.md | 22 +++++++++++++++-----
.../src/jvm/storm/kafka/SpoutConfig.java | 9 ++++++--
2 files changed, 24 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/77c2df28/external/storm-kafka/README.md
----------------------------------------------------------------------
diff --git a/external/storm-kafka/README.md b/external/storm-kafka/README.md
index 73cf334..283a950 100644
--- a/external/storm-kafka/README.md
+++ b/external/storm-kafka/README.md
@@ -51,17 +51,29 @@ The optional ClientId is used as a part of the zookeeper path where the spout's
There are 2 extensions of KafkaConfig currently in use.
-Spoutconfig is an extension of KafkaConfig that supports 2 additional fields, zkroot and id. The Zkroot will be used
-as root to store your consumer's offset. The id should uniquely identify your spout.
+Spoutconfig is an extension of KafkaConfig that supports additional fields with ZooKeeper connection info and for controlling
+behavior specific to KafkaSpout. The Zkroot will be used as root to store your consumer's offset. The id should uniquely
+identify your spout.
```java
public SpoutConfig(BrokerHosts hosts, String topic, String zkRoot, String id);
+public SpoutConfig(BrokerHosts hosts, String topic, String id);
+```
+In addition to these parameters, SpoutConfig contains the following fields that control how KafkaSpout behaves:
+```java
+ // setting for how often to save the current kafka offset to ZooKeeper
+ public long stateUpdateIntervalMs = 2000;
+
+ // Exponential back-off retry settings. These are used when retrying messages after a bolt
+ // calls OutputCollector.fail().
+ // Note: be sure to set backtype.storm.Config.MESSAGE_TIMEOUT_SECS appropriately to prevent
+ // resubmitting the message while still retrying.
+ public long retryInitialDelayMs = 0;
+ public double retryDelayMultiplier = 1.0;
+ public long retryDelayMaxMs = 60 * 1000;
```
Core KafkaSpout only accepts an instance of SpoutConfig.
TridentKafkaConfig is another extension of KafkaConfig.
-```java
-public SpoutConfig(BrokerHosts hosts, String topic, String id);
-```
TridentKafkaEmitter only accepts TridentKafkaConfig.
The KafkaConfig class also has bunch of public variables that controls your application's behavior. Here are defaults:
http://git-wip-us.apache.org/repos/asf/storm/blob/77c2df28/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java b/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
index fbc887f..1c40ddf 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
@@ -26,10 +26,15 @@ public class SpoutConfig extends KafkaConfig implements Serializable {
public Integer zkPort = null;
public String zkRoot = null;
public String id = null;
+
+ // setting for how often to save the current kafka offset to ZooKeeper
public long stateUpdateIntervalMs = 2000;
- // Exponential back-off retry settings - note: be sure to set backtype.storm.Config.MESSAGE_TIMEOUT_SECS
- // appropriately to prevent resubmitting the message while still retrying
+ // Exponential back-off retry settings. These are used when retrying messages after a bolt
+ // calls OutputCollector.fail().
+ //
+ // Note: be sure to set backtype.storm.Config.MESSAGE_TIMEOUT_SECS appropriately to prevent
+ // resubmitting the message while still retrying.
public long retryInitialDelayMs = 0;
public double retryDelayMultiplier = 1.0;
public long retryDelayMaxMs = 60 * 1000;
[12/44] storm git commit: remove garbage - I don't know where it came
from
Posted by bo...@apache.org.
remove garbage - I don't know where it came from
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ea0989d8
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ea0989d8
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ea0989d8
Branch: refs/heads/master
Commit: ea0989d8fb46ac213d813ec58d9c0ee07b75a6be
Parents: 1e1e572
Author: Rick Kilgore <ri...@hbo.com>
Authored: Tue Sep 9 14:54:25 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Tue Sep 9 14:54:25 2014 -0700
----------------------------------------------------------------------
storm-core/dependency-reduced-pom.xml | 417 -----------------------------
1 file changed, 417 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0989d8/storm-core/dependency-reduced-pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/dependency-reduced-pom.xml b/storm-core/dependency-reduced-pom.xml
deleted file mode 100644
index ec123dd..0000000
--- a/storm-core/dependency-reduced-pom.xml
+++ /dev/null
@@ -1,417 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <parent>
- <artifactId>storm</artifactId>
- <groupId>com.hbo.hurley</groupId>
- <version>0.9.3-incubating-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
- <groupId>com.hbo.hurley</groupId>
- <artifactId>storm-core</artifactId>
- <name>Storm Core</name>
- <description>Storm Core Java API and Clojure implementation.</description>
- <build>
- <sourceDirectory>src/jvm</sourceDirectory>
- <testSourceDirectory>test/jvm</testSourceDirectory>
- <resources>
- <resource>
- <directory>../conf</directory>
- </resource>
- <resource>
- <targetPath>META-INF</targetPath>
- <directory>../</directory>
- <includes>
- <include>NOTICE</include>
- </includes>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/dev</directory>
- </testResource>
- <testResource>
- <directory>test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>com.theoryinpractise</groupId>
- <artifactId>clojure-maven-plugin</artifactId>
- <extensions>true</extensions>
- <executions>
- <execution>
- <id>compile-clojure</id>
- <phase>compile</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
- <execution>
- <id>test-clojure</id>
- <phase>test</phase>
- <goals>
- <goal>test-with-junit</goal>
- </goals>
- <configuration>
- <vmargs>${test.extra.args}</vmargs>
- </configuration>
- </execution>
- </executions>
- <configuration>
- <sourceDirectories>
- <sourceDirectory>src/clj</sourceDirectory>
- </sourceDirectories>
- <testSourceDirectories>
- <testSourceDirectory>test/clj</testSourceDirectory>
- </testSourceDirectories>
- <warnOnReflection>false</warnOnReflection>
- <copyDeclaredNamespaceOnly>true</copyDeclaredNamespaceOnly>
- <copiedNamespaces>
- <copiedNamespace>none</copiedNamespace>
- </copiedNamespaces>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-surefire-report-plugin</artifactId>
- <configuration>
- <reportsDirectories>
- <file>${project.build.directory}/test-reports</file>
- </reportsDirectories>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.2</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- </execution>
- </executions>
- <dependencies>
- <dependency>
- <groupId>com.hbo.hurley</groupId>
- <artifactId>maven-shade-clojure-transformer</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- <configuration>
- <keepDependenciesWithProvidedScope>true</keepDependenciesWithProvidedScope>
- <promoteTransitiveDependencies>false</promoteTransitiveDependencies>
- <createDependencyReducedPom>true</createDependencyReducedPom>
- <minimizeJar>false</minimizeJar>
- <artifactSet>
- <includes>
- <include>org.apache.thrift:*</include>
- <include>io.netty:netty</include>
- <include>com.google.guava:guava</include>
- <include>org.apache.httpcomponents:http*</include>
- <include>org.apache.zookeeper:zookeeper</include>
- <include>org.apache.curator:*</include>
- </includes>
- </artifactSet>
- <relocations>
- <relocation>
- <pattern>org.apache.thrift</pattern>
- <shadedPattern>org.apache.thrift7</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.jboss.netty</pattern>
- <shadedPattern>org.apache.storm.netty</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google.common</pattern>
- <shadedPattern>org.apache.storm.guava</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.http</pattern>
- <shadedPattern>org.apache.storm.http</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.zookeeper</pattern>
- <shadedPattern>org.apache.storm.zookeeper</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.curator</pattern>
- <shadedPattern>org.apache.storm.curator</shadedPattern>
- </relocation>
- </relocations>
- <transformers>
- <transformer />
- </transformers>
- <filters>
- <filter>
- <artifact>org.apache.thrift:*</artifact>
- <excludes>
- <exclude>META-INF/LICENSE.txt</exclude>
- <exclude>META-INF/NOTICE.txt</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>io.netty:netty</artifact>
- <excludes>
- <exclude>META-INF/LICENSE.txt</exclude>
- <exclude>META-INF/NOTICE.txt</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>commons-httpclient:commons-httpclient</artifact>
- <excludes>
- <exclude>META-INF/LICENSE.txt</exclude>
- <exclude>META-INF/NOTICE.txt</exclude>
- <exclude>META-INF/README.txt</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.zookeeper:zookeeper</artifact>
- <excludes>
- <exclude>LICENSE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- </plugin>
- </plugins>
- </build>
- <dependencies>
- <dependency>
- <groupId>org.clojure</groupId>
- <artifactId>clojure</artifactId>
- <version>1.5.1</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>clj-time</groupId>
- <artifactId>clj-time</artifactId>
- <version>0.4.1</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>compojure</groupId>
- <artifactId>compojure</artifactId>
- <version>1.1.3</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>hiccup</groupId>
- <artifactId>hiccup</artifactId>
- <version>0.3.6</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>ring</groupId>
- <artifactId>ring-devel</artifactId>
- <version>0.3.11</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>ring</groupId>
- <artifactId>ring-jetty-adapter</artifactId>
- <version>0.3.11</version>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>org.mortbay.jetty</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.clojure</groupId>
- <artifactId>tools.logging</artifactId>
- <version>0.2.3</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.clojure</groupId>
- <artifactId>math.numeric-tower</artifactId>
- <version>0.0.1</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.clojure</groupId>
- <artifactId>tools.cli</artifactId>
- <version>0.2.4</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.clojure</groupId>
- <artifactId>tools.nrepl</artifactId>
- <version>0.2.3</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <artifactId>clojure</artifactId>
- <groupId>org.clojure</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>clojure-complete</groupId>
- <artifactId>clojure-complete</artifactId>
- <version>0.2.3</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <artifactId>clojure</artifactId>
- <groupId>org.clojure</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>2.4</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-exec</artifactId>
- <version>1.1</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>2.5</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- <version>0.7.0</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>slf4j-api</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- <version>2.5.0</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>log4j</artifactId>
- <groupId>log4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>slf4j-log4j12</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.googlecode.json-simple</groupId>
- <artifactId>json-simple</artifactId>
- <version>1.1</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>com.twitter</groupId>
- <artifactId>carbonite</artifactId>
- <version>1.4.0</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.yaml</groupId>
- <artifactId>snakeyaml</artifactId>
- <version>1.11</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.3.3</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.lmax</groupId>
- <artifactId>disruptor</artifactId>
- <version>3.2.1</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.jgrapht</groupId>
- <artifactId>jgrapht-core</artifactId>
- <version>0.9.0</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>16.0.1</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-classic</artifactId>
- <version>1.0.13</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>log4j-over-slf4j</artifactId>
- <version>1.6.6</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <version>3.9.0.Final</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>1.9.5</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.clojars.runa</groupId>
- <artifactId>conjure</artifactId>
- <version>2.1.3</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.1</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>reply</groupId>
- <artifactId>reply</artifactId>
- <version>0.3.0</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>cd-client</artifactId>
- <groupId>org.thnetos</groupId>
- </exclusion>
- <exclusion>
- <artifactId>drawbridge</artifactId>
- <groupId>com.cemerick</groupId>
- </exclusion>
- <exclusion>
- <artifactId>versioneer</artifactId>
- <groupId>trptcolin</groupId>
- </exclusion>
- <exclusion>
- <artifactId>sjacket</artifactId>
- <groupId>org.clojars.trptcolin</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
-</project>
-
[21/44] storm git commit: Merge pull request #2 from
HBOCodeLabs/hboRepo
Posted by bo...@apache.org.
Merge pull request #2 from HBOCodeLabs/hboRepo
exponential backoff for failed messages
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/36c5dc51
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/36c5dc51
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/36c5dc51
Branch: refs/heads/master
Commit: 36c5dc5169555325e9e7680939b2f87d6de44487
Parents: 5aae768 d010cb8
Author: Rick Kilgore <ri...@hbo.com>
Authored: Thu Sep 11 14:03:37 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Thu Sep 11 14:03:37 2014 -0700
----------------------------------------------------------------------
README.markdown | 3 +
examples/storm-starter/pom.xml | 6 +-
external/storm-hdfs/pom.xml | 6 +-
external/storm-kafka/pom.xml | 20 ++++--
.../src/jvm/storm/kafka/PartitionManager.java | 71 ++++++++++++++++++--
.../src/jvm/storm/kafka/SpoutConfig.java | 6 ++
pom.xml | 23 ++++++-
.../maven-shade-clojure-transformer/pom.xml | 6 +-
storm-core/pom.xml | 6 +-
storm-dist/binary/pom.xml | 6 +-
storm-dist/source/pom.xml | 6 +-
11 files changed, 131 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
[31/44] storm git commit: remove artifact
Posted by bo...@apache.org.
remove artifact
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/96dfa412
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/96dfa412
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/96dfa412
Branch: refs/heads/master
Commit: 96dfa412663722f31b493bb8cd8ec7829d148472
Parents: 2bf7f6f
Author: Rick Kilgore <ri...@hbo.com>
Authored: Wed Oct 1 14:19:19 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Wed Oct 1 14:19:19 2014 -0700
----------------------------------------------------------------------
storm-core/dependency-reduced-pom.xml | 417 -----------------------------
1 file changed, 417 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/96dfa412/storm-core/dependency-reduced-pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/dependency-reduced-pom.xml b/storm-core/dependency-reduced-pom.xml
deleted file mode 100644
index ec123dd..0000000
--- a/storm-core/dependency-reduced-pom.xml
+++ /dev/null
@@ -1,417 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <parent>
- <artifactId>storm</artifactId>
- <groupId>com.hbo.hurley</groupId>
- <version>0.9.3-incubating-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
- <groupId>com.hbo.hurley</groupId>
- <artifactId>storm-core</artifactId>
- <name>Storm Core</name>
- <description>Storm Core Java API and Clojure implementation.</description>
- <build>
- <sourceDirectory>src/jvm</sourceDirectory>
- <testSourceDirectory>test/jvm</testSourceDirectory>
- <resources>
- <resource>
- <directory>../conf</directory>
- </resource>
- <resource>
- <targetPath>META-INF</targetPath>
- <directory>../</directory>
- <includes>
- <include>NOTICE</include>
- </includes>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/dev</directory>
- </testResource>
- <testResource>
- <directory>test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>com.theoryinpractise</groupId>
- <artifactId>clojure-maven-plugin</artifactId>
- <extensions>true</extensions>
- <executions>
- <execution>
- <id>compile-clojure</id>
- <phase>compile</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
- <execution>
- <id>test-clojure</id>
- <phase>test</phase>
- <goals>
- <goal>test-with-junit</goal>
- </goals>
- <configuration>
- <vmargs>${test.extra.args}</vmargs>
- </configuration>
- </execution>
- </executions>
- <configuration>
- <sourceDirectories>
- <sourceDirectory>src/clj</sourceDirectory>
- </sourceDirectories>
- <testSourceDirectories>
- <testSourceDirectory>test/clj</testSourceDirectory>
- </testSourceDirectories>
- <warnOnReflection>false</warnOnReflection>
- <copyDeclaredNamespaceOnly>true</copyDeclaredNamespaceOnly>
- <copiedNamespaces>
- <copiedNamespace>none</copiedNamespace>
- </copiedNamespaces>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-surefire-report-plugin</artifactId>
- <configuration>
- <reportsDirectories>
- <file>${project.build.directory}/test-reports</file>
- </reportsDirectories>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.2</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- </execution>
- </executions>
- <dependencies>
- <dependency>
- <groupId>com.hbo.hurley</groupId>
- <artifactId>maven-shade-clojure-transformer</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- <configuration>
- <keepDependenciesWithProvidedScope>true</keepDependenciesWithProvidedScope>
- <promoteTransitiveDependencies>false</promoteTransitiveDependencies>
- <createDependencyReducedPom>true</createDependencyReducedPom>
- <minimizeJar>false</minimizeJar>
- <artifactSet>
- <includes>
- <include>org.apache.thrift:*</include>
- <include>io.netty:netty</include>
- <include>com.google.guava:guava</include>
- <include>org.apache.httpcomponents:http*</include>
- <include>org.apache.zookeeper:zookeeper</include>
- <include>org.apache.curator:*</include>
- </includes>
- </artifactSet>
- <relocations>
- <relocation>
- <pattern>org.apache.thrift</pattern>
- <shadedPattern>org.apache.thrift7</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.jboss.netty</pattern>
- <shadedPattern>org.apache.storm.netty</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google.common</pattern>
- <shadedPattern>org.apache.storm.guava</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.http</pattern>
- <shadedPattern>org.apache.storm.http</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.zookeeper</pattern>
- <shadedPattern>org.apache.storm.zookeeper</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.curator</pattern>
- <shadedPattern>org.apache.storm.curator</shadedPattern>
- </relocation>
- </relocations>
- <transformers>
- <transformer />
- </transformers>
- <filters>
- <filter>
- <artifact>org.apache.thrift:*</artifact>
- <excludes>
- <exclude>META-INF/LICENSE.txt</exclude>
- <exclude>META-INF/NOTICE.txt</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>io.netty:netty</artifact>
- <excludes>
- <exclude>META-INF/LICENSE.txt</exclude>
- <exclude>META-INF/NOTICE.txt</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>commons-httpclient:commons-httpclient</artifact>
- <excludes>
- <exclude>META-INF/LICENSE.txt</exclude>
- <exclude>META-INF/NOTICE.txt</exclude>
- <exclude>META-INF/README.txt</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.zookeeper:zookeeper</artifact>
- <excludes>
- <exclude>LICENSE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- </plugin>
- </plugins>
- </build>
- <dependencies>
- <dependency>
- <groupId>org.clojure</groupId>
- <artifactId>clojure</artifactId>
- <version>1.5.1</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>clj-time</groupId>
- <artifactId>clj-time</artifactId>
- <version>0.4.1</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>compojure</groupId>
- <artifactId>compojure</artifactId>
- <version>1.1.3</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>hiccup</groupId>
- <artifactId>hiccup</artifactId>
- <version>0.3.6</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>ring</groupId>
- <artifactId>ring-devel</artifactId>
- <version>0.3.11</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>ring</groupId>
- <artifactId>ring-jetty-adapter</artifactId>
- <version>0.3.11</version>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>org.mortbay.jetty</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.clojure</groupId>
- <artifactId>tools.logging</artifactId>
- <version>0.2.3</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.clojure</groupId>
- <artifactId>math.numeric-tower</artifactId>
- <version>0.0.1</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.clojure</groupId>
- <artifactId>tools.cli</artifactId>
- <version>0.2.4</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.clojure</groupId>
- <artifactId>tools.nrepl</artifactId>
- <version>0.2.3</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <artifactId>clojure</artifactId>
- <groupId>org.clojure</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>clojure-complete</groupId>
- <artifactId>clojure-complete</artifactId>
- <version>0.2.3</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <artifactId>clojure</artifactId>
- <groupId>org.clojure</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>2.4</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-exec</artifactId>
- <version>1.1</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>2.5</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- <version>0.7.0</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>slf4j-api</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- <version>2.5.0</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>log4j</artifactId>
- <groupId>log4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>slf4j-log4j12</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.googlecode.json-simple</groupId>
- <artifactId>json-simple</artifactId>
- <version>1.1</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>com.twitter</groupId>
- <artifactId>carbonite</artifactId>
- <version>1.4.0</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.yaml</groupId>
- <artifactId>snakeyaml</artifactId>
- <version>1.11</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.3.3</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.lmax</groupId>
- <artifactId>disruptor</artifactId>
- <version>3.2.1</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.jgrapht</groupId>
- <artifactId>jgrapht-core</artifactId>
- <version>0.9.0</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>16.0.1</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-classic</artifactId>
- <version>1.0.13</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>log4j-over-slf4j</artifactId>
- <version>1.6.6</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <version>3.9.0.Final</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>1.9.5</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.clojars.runa</groupId>
- <artifactId>conjure</artifactId>
- <version>2.1.3</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.1</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>reply</groupId>
- <artifactId>reply</artifactId>
- <version>0.3.0</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>cd-client</artifactId>
- <groupId>org.thnetos</groupId>
- </exclusion>
- <exclusion>
- <artifactId>drawbridge</artifactId>
- <groupId>com.cemerick</groupId>
- </exclusion>
- <exclusion>
- <artifactId>versioneer</artifactId>
- <groupId>trptcolin</groupId>
- </exclusion>
- <exclusion>
- <artifactId>sjacket</artifactId>
- <groupId>org.clojars.trptcolin</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
-</project>
-
[25/44] storm git commit: Merge remote-tracking branch 'apache/master'
Posted by bo...@apache.org.
Merge remote-tracking branch 'apache/master'
Conflicts:
README.markdown
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ed593480
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ed593480
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ed593480
Branch: refs/heads/master
Commit: ed593480ba543127c9491b5beab68ca49c6bd9a1
Parents: 36c5dc5 a56ccc7
Author: Rick Kilgore <ri...@hbo.com>
Authored: Fri Sep 26 16:10:49 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Fri Sep 26 16:10:49 2014 -0700
----------------------------------------------------------------------
CHANGELOG.md | 6 +
DEVELOPER.md | 28 +-
LICENSE | 25 ++
README.markdown | 3 +
dev-tools/jira-github-join.py | 2 +-
.../multilang/resources/asyncSplitsentence.js | 32 ++
.../multilang/resources/randomsentence.js | 75 ++++
.../multilang/resources/splitsentence.js | 26 ++
.../storm-starter/multilang/resources/storm.js | 349 +++++++++++++++++++
.../storm/starter/WordCountTopologyNode.java | 121 +++++++
.../jvm/storm/kafka/DynamicBrokersReader.java | 2 +-
.../storm/kafka/DynamicBrokersReaderTest.java | 1 +
.../clj/backtype/storm/daemon/supervisor.clj | 4 +
storm-core/src/dev/resources/storm.js | 349 +++++++++++++++++++
storm-core/src/dev/resources/tester_bolt.js | 27 ++
storm-core/src/dev/resources/tester_spout.js | 58 +++
.../src/jvm/backtype/storm/StormSubmitter.java | 16 +
storm-core/src/ui/public/component.html | 5 +
storm-core/src/ui/public/images/spinner.gif | Bin 0 -> 23470 bytes
storm-core/src/ui/public/index.html | 5 +
.../src/ui/public/js/jquery.blockUI.min.js | 6 +
storm-core/src/ui/public/js/script.js | 11 +
storm-core/src/ui/public/js/visualization.js | 16 +-
storm-core/src/ui/public/topology.html | 5 +
.../test/clj/backtype/storm/multilang_test.clj | 55 ++-
.../test/clj/backtype/storm/supervisor_test.clj | 2 +
26 files changed, 1179 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ed593480/README.markdown
----------------------------------------------------------------------
diff --cc README.markdown
index ff6f8c7,5fe638e..d0cfd69
--- a/README.markdown
+++ b/README.markdown
@@@ -173,7 -172,8 +174,9 @@@ under the License
* Dave Golombek ([@dgolombek](https://github.com/dgolombek))
* Edison Xu ([@EdisonXu](https://github.com/EdisonXu))
* Ariel Valentin ([@arielvalentin](http://blog.arielvalentin.com/))
+ * Dane Hammer ([@danehammer](https://github.com/danehammer))
+ * Christophe Carre' ([@chrisz](https://github.com/chrisz))
+* Rick Kilgore ([@rick-kilgore](http://github.com/rick-kilgore))
## Acknowledgements
[03/44] storm git commit: Merge branch 'master' into retryDelay
Posted by bo...@apache.org.
Merge branch 'master' into retryDelay
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6138e945
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6138e945
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6138e945
Branch: refs/heads/master
Commit: 6138e945894c2e0b95666909af51fd336862ce6d
Parents: f9d2948 5aae768
Author: Rick Kilgore <ri...@hbo.com>
Authored: Fri Sep 5 23:57:17 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Fri Sep 5 23:57:17 2014 -0700
----------------------------------------------------------------------
CHANGELOG.md | 2 ++
README.markdown | 4 +++-
pom.xml | 2 +-
storm-core/src/ui/public/component.html | 1 +
storm-core/src/ui/public/index.html | 1 +
storm-core/src/ui/public/topology.html | 1 +
6 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[30/44] storm git commit: back out whitepsace change
Posted by bo...@apache.org.
back out whitepsace change
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/2bf7f6fa
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/2bf7f6fa
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/2bf7f6fa
Branch: refs/heads/master
Commit: 2bf7f6fa1321ca720f2f03d34b638fa502a7e61b
Parents: fabd682
Author: Rick Kilgore <ri...@hbo.com>
Authored: Wed Oct 1 14:18:16 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Wed Oct 1 14:18:16 2014 -0700
----------------------------------------------------------------------
external/storm-kafka/pom.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/2bf7f6fa/external/storm-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-kafka/pom.xml b/external/storm-kafka/pom.xml
index e94ea12..4972619 100644
--- a/external/storm-kafka/pom.xml
+++ b/external/storm-kafka/pom.xml
@@ -31,6 +31,7 @@
<description>Storm Spouts for Apache Kafka</description>
<build>
<plugins>
+
</plugins>
<sourceDirectory>src/jvm</sourceDirectory>
<testSourceDirectory>src/test</testSourceDirectory>
[22/44] storm git commit: add me
Posted by bo...@apache.org.
add me
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8c78c13d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8c78c13d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8c78c13d
Branch: refs/heads/master
Commit: 8c78c13de01002fd39536430a32da5a0230184d6
Parents: 26af8da
Author: Rick Kilgore <ri...@hbo.com>
Authored: Thu Sep 11 14:05:37 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Thu Sep 11 14:05:37 2014 -0700
----------------------------------------------------------------------
README.markdown | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/8c78c13d/README.markdown
----------------------------------------------------------------------
diff --git a/README.markdown b/README.markdown
index d65d632..f765645 100644
--- a/README.markdown
+++ b/README.markdown
@@ -171,6 +171,7 @@ under the License.
* Dave Golombek ([@dgolombek](https://github.com/dgolombek))
* Edison Xu ([@EdisonXu](https://github.com/EdisonXu))
* Ariel Valentin ([@arielvalentin](http://blog.arielvalentin.com/))
+* Rick Kilgore ([@rick-kilgore](http://github.com/rick-kilgore))
## Acknowledgements
[17/44] storm git commit: silly fix
Posted by bo...@apache.org.
silly fix
use member var, don't pass it
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f4fa76c6
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f4fa76c6
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f4fa76c6
Branch: refs/heads/master
Commit: f4fa76c63efefa987ba124dd7dc8f895f9fe6dcc
Parents: 2aa0346
Author: Rick Kilgore <ri...@hbo.com>
Authored: Tue Sep 9 16:06:56 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Tue Sep 9 16:06:56 2014 -0700
----------------------------------------------------------------------
external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f4fa76c6/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
index c66545e..36a0e85 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
@@ -307,15 +307,15 @@ public class PartitionManager {
private MessageRetryRecord(int retryNum) {
this.retryNum = retryNum;
- this.retryTimeUTC = new Date().getTime() + calculateRetryDelay(this.retryNum);
+ this.retryTimeUTC = new Date().getTime() + calculateRetryDelay();
}
public MessageRetryRecord createNextRetryRecord() {
return new MessageRetryRecord(this.retryNum + 1);
}
- private long calculateRetryDelay(int retryNum) {
- double delayMultiplier = Math.pow(_spoutConfig.retryDelayMultiplier, retryNum - 1);
+ private long calculateRetryDelay() {
+ double delayMultiplier = Math.pow(_spoutConfig.retryDelayMultiplier, this.retryNum - 1);
long delayThisRetryMs = (long) (_spoutConfig.retryInitialDelayMs * delayMultiplier);
return Math.min(delayThisRetryMs, _spoutConfig.retryDelayMaxMs);
}
[10/44] storm git commit: add myself to list of contributors
Posted by bo...@apache.org.
add myself to list of contributors
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/1e1e5728
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/1e1e5728
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/1e1e5728
Branch: refs/heads/master
Commit: 1e1e57289fe4d1317fcb4cc6e4819c18178b742b
Parents: ae65a40
Author: Rick Kilgore <ri...@hbo.com>
Authored: Tue Sep 9 11:51:18 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Tue Sep 9 11:51:18 2014 -0700
----------------------------------------------------------------------
README.markdown | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/1e1e5728/README.markdown
----------------------------------------------------------------------
diff --git a/README.markdown b/README.markdown
index 93c534d..ff6f8c7 100644
--- a/README.markdown
+++ b/README.markdown
@@ -173,6 +173,7 @@ under the License.
* Dave Golombek ([@dgolombek](https://github.com/dgolombek))
* Edison Xu ([@EdisonXu](https://github.com/EdisonXu))
* Ariel Valentin ([@arielvalentin](http://blog.arielvalentin.com/))
+* Rick Kilgore ([@rick-kilgore](http://github.com/rick-kilgore))
## Acknowledgements
[09/44] storm git commit: add link to apache project in
README.markdown file
Posted by bo...@apache.org.
add link to apache project in README.markdown file
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ae65a404
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ae65a404
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ae65a404
Branch: refs/heads/master
Commit: ae65a40473158aaaa546981943e922131aa28b36
Parents: b95b6d6
Author: Rick Kilgore <ri...@hbo.com>
Authored: Tue Sep 9 11:45:04 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Tue Sep 9 11:45:04 2014 -0700
----------------------------------------------------------------------
README.markdown | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ae65a404/README.markdown
----------------------------------------------------------------------
diff --git a/README.markdown b/README.markdown
index d65d632..93c534d 100644
--- a/README.markdown
+++ b/README.markdown
@@ -1,3 +1,5 @@
+This is a forked version of incubator-storm maintained by [HBOCodeLabs](https://github.com/HBOCodeLabs). The main apache project is [here](https://github.com/apache/incubator-storm).
+
Storm is a distributed realtime computation system. Similar to how Hadoop provides a set of general primitives for doing batch processing, Storm provides a set of general primitives for doing realtime computation. Storm is simple, can be used with any programming language, [is used by many companies](http://storm-project.net/documentation/Powered-By.html), and is a lot of fun to use!
The [Rationale page](http://storm-project.net/documentation/Rationale.html) explains what Storm is and why it was built. [This presentation](http://vimeo.com/40972420) is also a good introduction to the project.
[06/44] storm git commit: default settings = no retry delay
Posted by bo...@apache.org.
default settings = no retry delay
default settings = no retry delay - i.e., preserve legacy behavior
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/05336764
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/05336764
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/05336764
Branch: refs/heads/master
Commit: 05336764408921f3ebcc4ac5b3e82da10e713bea
Parents: 04b64bc
Author: rick-kilgore <ri...@gmail.com>
Authored: Mon Sep 8 20:58:01 2014 +0000
Committer: rick-kilgore <ri...@gmail.com>
Committed: Mon Sep 8 20:58:01 2014 +0000
----------------------------------------------------------------------
external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/05336764/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java b/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
index 9a5c311..5a5d6f4 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
@@ -30,9 +30,9 @@ public class SpoutConfig extends KafkaConfig implements Serializable {
// Exponential back-off retry settings - note: be sure to set backtype.storm.Config.MESSAGE_TIMEOUT_SECS
// appropriately to prevent resubmitting the message while still retrying
- public long retryInitialDelayMs = 1000;
+ public long retryInitialDelayMs = 0;
public double retryDelayMultiplier = 1.0;
- public long retryMaxDelayMs = 1000;
+ public long retryMaxDelayMs = 60 * 1000;
public SpoutConfig(BrokerHosts hosts, String topic, String zkRoot, String id) {
super(hosts, topic);
[37/44] storm git commit: Merge branch 'master' into retries
Posted by bo...@apache.org.
Merge branch 'master' into retries
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a95a1883
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a95a1883
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a95a1883
Branch: refs/heads/master
Commit: a95a18838b0d17b604ea5e2ba44007d7c3c10318
Parents: 02bffc6 e6bed26
Author: Rick Kilgore <ri...@hbo.com>
Authored: Fri Nov 21 18:25:26 2014 -0800
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Fri Nov 21 18:25:26 2014 -0800
----------------------------------------------------------------------
CHANGELOG.md | 26 +-
DEVELOPER.md | 4 +
LICENSE | 34 +-
README.markdown | 2 +
SECURITY.md | 356 ++-
STORM-UI-REST-API.md | 17 +-
bin/storm | 16 +-
bin/storm.cmd | 11 +-
conf/defaults.yaml | 45 +-
conf/jaas_digest.conf | 8 +-
conf/jaas_kerberos.conf | 15 +
examples/storm-starter/README.markdown | 9 +-
.../storm-starter/multilang/resources/storm.js | 20 +-
.../storm-starter/multilang/resources/storm.py | 87 +-
.../storm-starter/multilang/resources/storm.rb | 90 +-
examples/storm-starter/pom.xml | 2 +-
.../src/jvm/storm/starter/util/StormRunner.java | 3 +-
external/storm-hbase/pom.xml | 4 +-
.../storm/hbase/bolt/HBaseLookupBolt.java | 2 +-
.../hbase/bolt/mapper/HBaseValueMapper.java | 4 +-
.../storm/hbase/trident/state/HBaseState.java | 6 +-
.../hbase/topology/WordCountValueMapper.java | 3 +-
external/storm-hdfs/pom.xml | 4 +-
external/storm-kafka/README.md | 7 +-
external/storm-kafka/pom.xml | 2 +-
.../src/jvm/storm/kafka/KafkaUtils.java | 63 +-
.../src/jvm/storm/kafka/PartitionManager.java | 11 +-
.../jvm/storm/kafka/UpdateOffsetException.java | 5 +
.../src/test/storm/kafka/KafkaUtilsTest.java | 7 +-
logback/cluster.xml | 4 +-
logback/worker.xml | 41 +
pom.xml | 39 +-
.../maven-shade-clojure-transformer/pom.xml | 4 +-
storm-core/pom.xml | 311 +-
.../src/clj/backtype/storm/LocalCluster.clj | 4 +
storm-core/src/clj/backtype/storm/LocalDRPC.clj | 4 +-
.../src/clj/backtype/storm/MockAutoCred.clj | 58 +
storm-core/src/clj/backtype/storm/bootstrap.clj | 5 +-
storm-core/src/clj/backtype/storm/cluster.clj | 167 +-
.../storm/command/upload_credentials.clj | 35 +
storm-core/src/clj/backtype/storm/config.clj | 40 +-
.../backtype/storm/daemon/builtin_metrics.clj | 21 +-
.../src/clj/backtype/storm/daemon/common.clj | 18 +-
.../src/clj/backtype/storm/daemon/drpc.clj | 174 +-
.../src/clj/backtype/storm/daemon/executor.clj | 137 +-
.../src/clj/backtype/storm/daemon/logviewer.clj | 271 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 245 +-
.../clj/backtype/storm/daemon/supervisor.clj | 181 +-
.../src/clj/backtype/storm/daemon/task.clj | 7 +-
.../src/clj/backtype/storm/daemon/worker.clj | 109 +-
storm-core/src/clj/backtype/storm/disruptor.clj | 11 +-
.../src/clj/backtype/storm/messaging/loader.clj | 13 +-
storm-core/src/clj/backtype/storm/testing.clj | 56 +-
storm-core/src/clj/backtype/storm/testing4j.clj | 5 +-
storm-core/src/clj/backtype/storm/thrift.clj | 28 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 213 +-
.../src/clj/backtype/storm/ui/helpers.clj | 69 +-
storm-core/src/clj/backtype/storm/util.clj | 95 +-
storm-core/src/clj/backtype/storm/zookeeper.clj | 19 +-
storm-core/src/clj/storm/trident/testing.clj | 1 +
.../src/dev/drpc-simple-acl-test-scenario.yaml | 11 +
storm-core/src/dev/resources/storm.js | 6 +
storm-core/src/dev/resources/storm.py | 261 +-
storm-core/src/dev/resources/storm.rb | 237 +-
storm-core/src/jvm/backtype/storm/Config.java | 537 +++-
.../jvm/backtype/storm/ConfigValidation.java | 156 +-
.../src/jvm/backtype/storm/Constants.java | 3 +-
.../backtype/storm/ICredentialsListener.java | 32 +
.../src/jvm/backtype/storm/ILocalCluster.java | 2 +
.../src/jvm/backtype/storm/StormSubmitter.java | 164 +-
.../storm/drpc/DRPCInvocationsClient.java | 91 +-
.../src/jvm/backtype/storm/drpc/DRPCSpout.java | 100 +-
.../jvm/backtype/storm/drpc/ReturnResults.java | 35 +-
.../storm/generated/AuthorizationException.java | 345 ++
.../backtype/storm/generated/Credentials.java | 390 +++
.../storm/generated/DistributedRPC.java | 110 +-
.../generated/DistributedRPCInvocations.java | 352 +-
.../jvm/backtype/storm/generated/Nimbus.java | 3006 +++++++++++++++++-
.../backtype/storm/generated/SubmitOptions.java | 98 +-
.../backtype/storm/generated/TopologyInfo.java | 192 +-
.../storm/generated/TopologySummary.java | 192 +-
.../backtype/storm/messaging/netty/Client.java | 73 +-
.../storm/messaging/netty/ControlMessage.java | 4 +-
.../storm/messaging/netty/MessageDecoder.java | 32 +-
.../storm/messaging/netty/MessageEncoder.java | 4 +
.../storm/messaging/netty/SaslMessageToken.java | 99 +
.../storm/messaging/netty/SaslNettyClient.java | 166 +
.../messaging/netty/SaslNettyClientState.java | 31 +
.../storm/messaging/netty/SaslNettyServer.java | 165 +
.../messaging/netty/SaslNettyServerState.java | 31 +
.../messaging/netty/SaslStormClientHandler.java | 158 +
.../netty/SaslStormServerAuthorizeHandler.java | 83 +
.../messaging/netty/SaslStormServerHandler.java | 155 +
.../storm/messaging/netty/SaslUtils.java | 74 +
.../backtype/storm/messaging/netty/Server.java | 104 +-
.../netty/StormClientPipelineFactory.java | 12 +-
.../messaging/netty/StormServerHandler.java | 2 +-
.../netty/StormServerPipelineFactory.java | 20 +-
.../jvm/backtype/storm/scheduler/Cluster.java | 13 +
.../scheduler/multitenant/DefaultPool.java | 219 ++
.../storm/scheduler/multitenant/FreePool.java | 125 +
.../scheduler/multitenant/IsolatedPool.java | 346 ++
.../multitenant/MultitenantScheduler.java | 98 +
.../storm/scheduler/multitenant/Node.java | 343 ++
.../storm/scheduler/multitenant/NodePool.java | 296 ++
.../storm/security/INimbusCredentialPlugin.java | 47 +
.../backtype/storm/security/auth/AuthUtils.java | 228 +-
.../auth/DefaultHttpCredentialsPlugin.java | 87 +
.../security/auth/DefaultPrincipalToLocal.java | 43 +
.../storm/security/auth/IAutoCredentials.java | 55 +
.../security/auth/ICredentialsRenewer.java | 41 +
.../auth/IGroupMappingServiceProvider.java | 42 +
.../security/auth/IHttpCredentialsPlugin.java | 50 +
.../storm/security/auth/IPrincipalToLocal.java | 41 +
.../storm/security/auth/ITransportPlugin.java | 14 +-
.../security/auth/KerberosPrincipalToLocal.java | 45 +
.../storm/security/auth/ReqContext.java | 10 +-
.../security/auth/SaslTransportPlugin.java | 44 +-
.../security/auth/ShellBasedGroupsMapping.java | 94 +
.../security/auth/SimpleTransportPlugin.java | 61 +-
.../security/auth/SingleUserPrincipal.java | 56 +
.../storm/security/auth/TBackoffConnect.java | 77 +
.../storm/security/auth/ThriftClient.java | 85 +-
.../security/auth/ThriftConnectionType.java | 77 +
.../storm/security/auth/ThriftServer.java | 19 +-
.../auth/authorizer/DRPCAuthorizerBase.java | 46 +
.../authorizer/DRPCSimpleACLAuthorizer.java | 157 +
.../auth/authorizer/DenyAuthorizer.java | 4 +-
.../auth/authorizer/NoopAuthorizer.java | 6 +-
.../auth/authorizer/SimpleACLAuthorizer.java | 131 +
.../authorizer/SimpleWhitelistAuthorizer.java | 70 +
.../auth/digest/DigestSaslTransportPlugin.java | 1 +
.../storm/security/auth/hadoop/AutoHDFS.java | 262 ++
.../storm/security/auth/kerberos/AutoTGT.java | 281 ++
.../auth/kerberos/AutoTGTKrb5LoginModule.java | 108 +
.../kerberos/AutoTGTKrb5LoginModuleTest.java | 44 +
.../auth/kerberos/ClientCallbackHandler.java | 104 +
.../kerberos/KerberosSaslTransportPlugin.java | 206 ++
.../auth/kerberos/ServerCallbackHandler.java | 86 +
.../auth/kerberos/jaas_kerberos_cluster.conf | 31 +
.../auth/kerberos/jaas_kerberos_launcher.conf | 12 +
.../jvm/backtype/storm/spout/ShellSpout.java | 65 +-
.../src/jvm/backtype/storm/task/ShellBolt.java | 211 +-
.../testing/ForwardingMetricsConsumer.java | 95 +
.../testing/SingleUserSimpleTransport.java | 37 +
.../state/TestTransactionalState.java | 47 +
.../transactional/state/TransactionalState.java | 56 +-
.../jvm/backtype/storm/utils/DRPCClient.java | 63 +-
.../backtype/storm/utils/DisruptorQueue.java | 56 +-
.../jvm/backtype/storm/utils/LocalState.java | 44 +-
.../jvm/backtype/storm/utils/NimbusClient.java | 11 +-
.../jvm/backtype/storm/utils/ShellUtils.java | 498 +++
.../src/jvm/backtype/storm/utils/TestUtils.java | 34 +
.../src/jvm/backtype/storm/utils/Utils.java | 130 +-
.../backtype/storm/utils/ZookeeperAuthInfo.java | 9 +-
.../storm/utils/ZookeeperServerCnxnFactory.java | 84 +
.../trident/drpc/ReturnResultsReducer.java | 13 +-
.../topology/state/TestTransactionalState.java | 47 +
.../topology/state/TransactionalState.java | 58 +-
storm-core/src/multilang/js/storm.js | 349 ++
storm-core/src/multilang/py/storm.py | 25 +-
storm-core/src/multilang/rb/storm.rb | 55 +-
.../src/native/worker-launcher/.autom4te.cfg | 42 +
.../worker-launcher/.deps/worker-launcher.Po | 1 +
.../src/native/worker-launcher/Makefile.am | 32 +
.../src/native/worker-launcher/configure.ac | 50 +
.../native/worker-launcher/impl/configuration.c | 340 ++
.../native/worker-launcher/impl/configuration.h | 45 +
.../src/native/worker-launcher/impl/main.c | 210 ++
.../worker-launcher/impl/worker-launcher.c | 779 +++++
.../worker-launcher/impl/worker-launcher.h | 129 +
.../worker-launcher/test/test-worker-launcher.c | 340 ++
storm-core/src/py/__init__.py | 16 +
storm-core/src/py/storm/DistributedRPC-remote | 18 +
storm-core/src/py/storm/DistributedRPC.py | 37 +-
.../py/storm/DistributedRPCInvocations-remote | 18 +
.../src/py/storm/DistributedRPCInvocations.py | 96 +-
storm-core/src/py/storm/Nimbus-remote | 25 +
storm-core/src/py/storm/Nimbus.py | 652 +++-
storm-core/src/py/storm/__init__.py | 16 +
storm-core/src/py/storm/constants.py | 16 +
storm-core/src/py/storm/ttypes.py | 1243 +++++++-
storm-core/src/storm.thrift | 58 +-
storm-core/src/ui/public/component.html | 29 +-
storm-core/src/ui/public/css/style.css | 9 +
storm-core/src/ui/public/index.html | 12 +-
storm-core/src/ui/public/js/purl.js | 267 --
storm-core/src/ui/public/js/script.js | 6 +-
storm-core/src/ui/public/js/url.min.js | 1 +
storm-core/src/ui/public/js/visualization.js | 2 +-
.../public/templates/anti-forgery-template.html | 19 +
.../templates/component-page-template.html | 6 +-
.../public/templates/index-page-template.html | 14 +-
.../templates/topology-page-template.html | 29 +-
.../src/ui/public/templates/user-template.html | 25 +
storm-core/src/ui/public/topology.html | 29 +-
.../test/clj/backtype/storm/cluster_test.clj | 93 +-
.../test/clj/backtype/storm/config_test.clj | 11 +
.../test/clj/backtype/storm/drpc_test.clj | 14 +-
.../clj/backtype/storm/local_state_test.clj | 14 +-
.../test/clj/backtype/storm/logviewer_test.clj | 187 ++
.../storm/messaging/netty_integration_test.clj | 4 +-
.../storm/messaging/netty_unit_test.clj | 32 +-
.../test/clj/backtype/storm/nimbus_test.clj | 377 ++-
.../scheduler/multitenant_scheduler_test.clj | 831 +++++
.../storm/security/auth/AuthUtils_test.clj | 16 +-
.../auth/DefaultHttpCredentialsPlugin_test.clj | 40 +
.../storm/security/auth/ThriftClient_test.clj | 28 +-
.../storm/security/auth/ThriftServer_test.clj | 8 +-
.../backtype/storm/security/auth/auth_test.clj | 374 ++-
.../authorizer/DRPCSimpleACLAuthorizer_test.clj | 226 ++
.../security/auth/auto_login_module_test.clj | 91 +
.../storm/security/auth/drpc-auth-alice.jaas | 5 +
.../storm/security/auth/drpc-auth-bob.jaas | 5 +
.../storm/security/auth/drpc-auth-charlie.jaas | 5 +
.../storm/security/auth/drpc-auth-server.jaas | 6 +
.../storm/security/auth/drpc_auth_test.clj | 315 ++
.../storm/security/auth/nimbus_auth_test.clj | 181 ++
.../test/clj/backtype/storm/submitter_test.clj | 75 +
.../test/clj/backtype/storm/supervisor_test.clj | 186 +-
.../test/clj/backtype/storm/testing4j_test.clj | 32 +
.../clj/backtype/storm/transactional_test.clj | 27 +-
.../utils/ZookeeperServerCnxnFactory_test.clj | 35 +
.../test/clj/backtype/storm/utils_test.clj | 58 +-
.../test/clj/storm/trident/state_test.clj | 25 +-
.../storm/utils/DisruptorQueueTest.java | 25 +-
storm-dist/binary/LICENSE | 25 +-
storm-dist/binary/pom.xml | 2 +-
storm-dist/binary/src/main/assembly/binary.xml | 4 +
storm-dist/source/pom.xml | 2 +-
230 files changed, 22991 insertions(+), 1952 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/a95a1883/external/storm-kafka/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/a95a1883/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
----------------------------------------------------------------------
[26/44] storm git commit: Merge branch 'master' into retries
Posted by bo...@apache.org.
Merge branch 'master' into retries
Conflicts:
README.markdown
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3cda45d5
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3cda45d5
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3cda45d5
Branch: refs/heads/master
Commit: 3cda45d5045eafc35a9aaa4136681fe5628360de
Parents: 77c2df2 ed59348
Author: Rick Kilgore <ri...@hbo.com>
Authored: Wed Oct 1 13:29:19 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Wed Oct 1 13:29:19 2014 -0700
----------------------------------------------------------------------
CHANGELOG.md | 6 +
DEVELOPER.md | 28 +-
LICENSE | 25 ++
README.markdown | 5 +
dev-tools/jira-github-join.py | 2 +-
.../multilang/resources/asyncSplitsentence.js | 32 ++
.../multilang/resources/randomsentence.js | 75 ++++
.../multilang/resources/splitsentence.js | 26 ++
.../storm-starter/multilang/resources/storm.js | 349 +++++++++++++++++++
examples/storm-starter/pom.xml | 6 +-
.../storm/starter/WordCountTopologyNode.java | 121 +++++++
external/storm-hdfs/pom.xml | 6 +-
external/storm-kafka/pom.xml | 20 +-
.../jvm/storm/kafka/DynamicBrokersReader.java | 2 +-
.../storm/kafka/DynamicBrokersReaderTest.java | 1 +
pom.xml | 23 +-
.../maven-shade-clojure-transformer/pom.xml | 6 +-
storm-core/pom.xml | 6 +-
.../clj/backtype/storm/daemon/supervisor.clj | 4 +
storm-core/src/dev/resources/storm.js | 349 +++++++++++++++++++
storm-core/src/dev/resources/tester_bolt.js | 27 ++
storm-core/src/dev/resources/tester_spout.js | 58 +++
.../src/jvm/backtype/storm/StormSubmitter.java | 16 +
storm-core/src/ui/public/component.html | 5 +
storm-core/src/ui/public/images/spinner.gif | Bin 0 -> 23470 bytes
storm-core/src/ui/public/index.html | 5 +
.../src/ui/public/js/jquery.blockUI.min.js | 6 +
storm-core/src/ui/public/js/script.js | 11 +
storm-core/src/ui/public/js/visualization.js | 16 +-
storm-core/src/ui/public/topology.html | 5 +
.../test/clj/backtype/storm/multilang_test.clj | 55 ++-
.../test/clj/backtype/storm/supervisor_test.clj | 2 +
storm-dist/binary/pom.xml | 6 +-
storm-dist/source/pom.xml | 6 +-
34 files changed, 1236 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
[28/44] storm git commit: Merge branch 'master' into retries
Posted by bo...@apache.org.
Merge branch 'master' into retries
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d49d014c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d49d014c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d49d014c
Branch: refs/heads/master
Commit: d49d014c0c540cebd0a985ace52c57fc5ee88ec7
Parents: 3cda45d 9faf431
Author: Rick Kilgore <ri...@hbo.com>
Authored: Wed Oct 1 13:41:45 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Wed Oct 1 13:41:45 2014 -0700
----------------------------------------------------------------------
README.markdown | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[14/44] storm git commit: remove garbage file
Posted by bo...@apache.org.
remove garbage file
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/49f91dc3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/49f91dc3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/49f91dc3
Branch: refs/heads/master
Commit: 49f91dc3e644643d774afa159cf521faa9062e42
Parents: 5c1029a
Author: Rick Kilgore <ri...@hbo.com>
Authored: Tue Sep 9 15:32:39 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Tue Sep 9 15:32:39 2014 -0700
----------------------------------------------------------------------
external/storm-kafka/gs | 7 -------
1 file changed, 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/49f91dc3/external/storm-kafka/gs
----------------------------------------------------------------------
diff --git a/external/storm-kafka/gs b/external/storm-kafka/gs
deleted file mode 100644
index 9f6e837..0000000
--- a/external/storm-kafka/gs
+++ /dev/null
@@ -1,7 +0,0 @@
-changes made since retries:
-
- 45.9% external/storm-kafka/
-
- external/storm-kafka/pom.xml | 18 [1;32m+++++++++++++++[m[1;31m---[m
- pom.xml | 21 [1;32m++++++++++++++++++++[m[1;31m-[m
- 2 files changed, 35 insertions(+), 4 deletions(-)
[33/44] storm git commit: Merge remote-tracking branch 'apache/master'
Posted by bo...@apache.org.
Merge remote-tracking branch 'apache/master'
Conflicts:
README.markdown
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3c699e37
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3c699e37
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3c699e37
Branch: refs/heads/master
Commit: 3c699e371a1aba32bdbcd1853edca80f4a00b7e9
Parents: ae71054 38367b0
Author: Rick Kilgore <ri...@hbo.com>
Authored: Wed Oct 1 17:08:14 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Wed Oct 1 17:08:14 2014 -0700
----------------------------------------------------------------------
CHANGELOG.md | 4 +
README.markdown | 1 +
bin/storm | 2 +-
examples/storm-starter/pom.xml | 2 +-
external/storm-hbase/LICENSE | 202 ++++++++++++++++
external/storm-hbase/README.md | 215 +++++++++++++++++
external/storm-hbase/pom.xml | 78 +++++++
.../storm/hbase/bolt/AbstractHBaseBolt.java | 69 ++++++
.../org/apache/storm/hbase/bolt/HBaseBolt.java | 77 ++++++
.../storm/hbase/bolt/HBaseLookupBolt.java | 83 +++++++
.../storm/hbase/bolt/mapper/HBaseMapper.java | 48 ++++
.../bolt/mapper/HBaseProjectionCriteria.java | 85 +++++++
.../hbase/bolt/mapper/HBaseValueMapper.java | 41 ++++
.../hbase/bolt/mapper/SimpleHBaseMapper.java | 90 +++++++
.../apache/storm/hbase/common/ColumnList.java | 202 ++++++++++++++++
.../apache/storm/hbase/common/HBaseClient.java | 135 +++++++++++
.../org/apache/storm/hbase/common/IColumn.java | 30 +++
.../org/apache/storm/hbase/common/ICounter.java | 29 +++
.../org/apache/storm/hbase/common/Utils.java | 69 ++++++
.../storm/hbase/security/HBaseSecurityUtil.java | 52 +++++
.../mapper/SimpleTridentHBaseMapper.java | 89 +++++++
.../trident/mapper/TridentHBaseMapper.java | 31 +++
.../hbase/trident/state/HBaseMapState.java | 233 +++++++++++++++++++
.../storm/hbase/trident/state/HBaseQuery.java | 40 ++++
.../storm/hbase/trident/state/HBaseState.java | 162 +++++++++++++
.../hbase/trident/state/HBaseStateFactory.java | 40 ++++
.../storm/hbase/trident/state/HBaseUpdater.java | 32 +++
.../storm/hbase/topology/LookupWordCount.java | 79 +++++++
.../hbase/topology/PersistentWordCount.java | 81 +++++++
.../storm/hbase/topology/TotalWordCounter.java | 70 ++++++
.../storm/hbase/topology/WordCountClient.java | 57 +++++
.../hbase/topology/WordCountValueMapper.java | 69 ++++++
.../storm/hbase/topology/WordCounter.java | 59 +++++
.../apache/storm/hbase/topology/WordSpout.java | 88 +++++++
.../storm/hbase/trident/PrintFunction.java | 40 ++++
.../storm/hbase/trident/WordCountTrident.java | 104 +++++++++
external/storm-hdfs/pom.xml | 2 +-
external/storm-kafka/pom.xml | 2 +-
.../src/test/storm/kafka/TridentKafkaTest.java | 9 +-
pom.xml | 27 +--
.../maven-shade-clojure-transformer/pom.xml | 2 +-
storm-core/pom.xml | 2 +-
.../src/jvm/backtype/storm/tuple/ITuple.java | 136 +++++++++++
.../src/jvm/backtype/storm/tuple/Tuple.java | 113 +--------
.../storm/trident/testing/MockTridentTuple.java | 166 -------------
.../jvm/storm/trident/tuple/TridentTuple.java | 46 +---
.../storm/trident/tuple/TridentTupleView.java | 35 ++-
.../jvm/storm/trident/tuple/ValuePointer.java | 1 -
storm-core/src/ui/public/topology.html | 4 +-
.../test/clj/storm/trident/tuple_test.clj | 28 ++-
storm-dist/binary/pom.xml | 2 +-
storm-dist/binary/src/main/assembly/binary.xml | 14 ++
storm-dist/source/pom.xml | 2 +-
53 files changed, 3018 insertions(+), 361 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/3c699e37/README.markdown
----------------------------------------------------------------------
diff --cc README.markdown
index e45f63a,2f90a4e..05d29bc
--- a/README.markdown
+++ b/README.markdown
@@@ -174,7 -174,7 +174,8 @@@ under the License
* Ariel Valentin ([@arielvalentin](http://blog.arielvalentin.com/))
* Dane Hammer ([@danehammer](https://github.com/danehammer))
* Christophe Carre' ([@chrisz](https://github.com/chrisz))
+ * Anya Tchernishov ([@anyatch](https://github.com/anyatch))
+* Rick Kilgore ([@rick-kilgore](http://github.com/rick-kilgore))
## Acknowledgements
[07/44] storm git commit: publish artifacts to HBO archiva repository
Posted by bo...@apache.org.
publish artifacts to HBO archiva repository
mvn deploy publishes artifacts to HBO archiva repository
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/cbbea7c5
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/cbbea7c5
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/cbbea7c5
Branch: refs/heads/master
Commit: cbbea7c5b2d014e0cb76b120223e6c257a96248b
Parents: 0533676
Author: rick-kilgore <ri...@gmail.com>
Authored: Mon Sep 8 22:44:22 2014 +0000
Committer: rick-kilgore <ri...@gmail.com>
Committed: Mon Sep 8 22:44:22 2014 +0000
----------------------------------------------------------------------
external/storm-kafka/pom.xml | 19 ++++++++++++++++---
pom.xml | 21 ++++++++++++++++++++-
2 files changed, 36 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/cbbea7c5/external/storm-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-kafka/pom.xml b/external/storm-kafka/pom.xml
index 4972619..4f960f3 100644
--- a/external/storm-kafka/pom.xml
+++ b/external/storm-kafka/pom.xml
@@ -26,16 +26,29 @@
</parent>
<packaging>jar</packaging>
- <artifactId>storm-kafka</artifactId>
- <name>storm-kafka</name>
+ <groupId>com.hbo.hurley</groupId>
+ <artifactId>storm-kafka-hurley</artifactId>
+ <name>storm-kafka-hurley</name>
<description>Storm Spouts for Apache Kafka</description>
<build>
<plugins>
-
</plugins>
<sourceDirectory>src/jvm</sourceDirectory>
<testSourceDirectory>src/test</testSourceDirectory>
</build>
+
+ <distributionManagement>
+ <repository>
+ <id>archiva.internal</id>
+ <name>Hurley Maven Repository</name>
+ <url>http://maven.hurley.hbo.com:8000/repository/internal/</url>
+ </repository>
+ <snapshotRepository>
+ <id>archiva.snapshots</id>
+ <name>Hurley Maven Snapshot Repository</name>
+ <url>http://maven.hurley.hbo.com:8000/repository/snapshots/</url>
+ </snapshotRepository>
+ </distributionManagement>
<dependencies>
<dependency>
<groupId>org.mockito</groupId>
http://git-wip-us.apache.org/repos/asf/storm/blob/cbbea7c5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 111672c..825cc38 100644
--- a/pom.xml
+++ b/pom.xml
@@ -287,6 +287,16 @@
<name>Storm Website</name>
<url>file:///tmp/site</url>
</site>
+ <repository>
+ <id>archiva.internal</id>
+ <name>Hurley Maven Repository</name>
+ <url>http://maven.hurley.hbo.com:8000/repository/internal/</url>
+ </repository>
+ <snapshotRepository>
+ <id>archiva.snapshots</id>
+ <name>Hurley Maven Snapshot Repository</name>
+ <url>http://maven.hurley.hbo.com:8000/repository/snapshots/</url>
+ </snapshotRepository>
</distributionManagement>
<dependencyManagement>
@@ -516,6 +526,16 @@
<enabled>true</enabled>
</releases>
<snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ <id>hurley</id>
+ <url>http://maven.hurley.hbo.com:8000/repository/snapshots/</url>
+ </repository>
+ <repository>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
<enabled>false</enabled>
</snapshots>
<id>central</id>
@@ -534,7 +554,6 @@
</repositories>
<build>
-
<pluginManagement>
<plugins>
<plugin>
[20/44] storm git commit: Merge branch 'retries' into hboRepo
Posted by bo...@apache.org.
Merge branch 'retries' into hboRepo
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d010cb8f
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d010cb8f
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d010cb8f
Branch: refs/heads/master
Commit: d010cb8f25b4a4e0a7882391fc74566c657dbcd1
Parents: e54fae3 26af8da
Author: Rick Kilgore <ri...@hbo.com>
Authored: Wed Sep 10 17:18:53 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Wed Sep 10 17:18:53 2014 -0700
----------------------------------------------------------------------
external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[23/44] storm git commit: added to README the need to run mvn deploy
Posted by bo...@apache.org.
added to README the need to run mvn deploy
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/40ab899f
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/40ab899f
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/40ab899f
Branch: refs/heads/master
Commit: 40ab899fe3d67e77c9f1678dbc70d19dd2faf065
Parents: 36c5dc5
Author: Rick Kilgore <ri...@hbo.com>
Authored: Thu Sep 11 14:16:05 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Thu Sep 11 14:16:05 2014 -0700
----------------------------------------------------------------------
README.markdown | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/40ab899f/README.markdown
----------------------------------------------------------------------
diff --git a/README.markdown b/README.markdown
index ff6f8c7..2eaeacf 100644
--- a/README.markdown
+++ b/README.markdown
@@ -1,4 +1,4 @@
-This is a forked version of incubator-storm maintained by [HBOCodeLabs](https://github.com/HBOCodeLabs). The main apache project is [here](https://github.com/apache/incubator-storm).
+This is a forked version of incubator-storm maintained by [HBOCodeLabs](https://github.com/HBOCodeLabs). The main apache project is [here](https://github.com/apache/incubator-storm). In order to make changes take effect within HBO Hurley, you need to run "mvn deploy" in the root project folder from the branch that you want to be exported.
Storm is a distributed realtime computation system. Similar to how Hadoop provides a set of general primitives for doing batch processing, Storm provides a set of general primitives for doing realtime computation. Storm is simple, can be used with any programming language, [is used by many companies](http://storm-project.net/documentation/Powered-By.html), and is a lot of fun to use!
[15/44] storm git commit: do the time calc only once
Posted by bo...@apache.org.
do the time calc only once
do the due time calculation for a MessageRetryRecord when one is
created, not every time isReadyForRetry() is calculated
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/2aa03468
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/2aa03468
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/2aa03468
Branch: refs/heads/master
Commit: 2aa03468db3971a11d07048628908074f2a0d3b3
Parents: 79e3efe
Author: Rick Kilgore <ri...@hbo.com>
Authored: Tue Sep 9 16:03:34 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Tue Sep 9 16:03:34 2014 -0700
----------------------------------------------------------------------
.../src/jvm/storm/kafka/PartitionManager.java | 33 +++++++++++---------
1 file changed, 19 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/2aa03468/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
index 02edf31..c66545e 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
@@ -223,7 +223,9 @@ public class PartitionManager {
LOG.debug("failing at offset=" + offset + " with _pending.size()=" + _pending.size() + " pending and _emittedToOffset=" + _emittedToOffset);
failed.add(offset);
MessageRetryRecord retryRecord = retryRecords.get(offset);
- retryRecords.put(offset, retryRecord == null ? new MessageRetryRecord() : retryRecord.retryAgainRecord());
+ retryRecords.put(offset, retryRecord == null
+ ? new MessageRetryRecord()
+ : retryRecord.createNextRetryRecord());
numberFailed++;
if (numberAcked == 0 && numberFailed > _spoutConfig.maxOffsetBehind) {
throw new RuntimeException("Too many tuple failures");
@@ -296,27 +298,30 @@ public class PartitionManager {
* </ul>
*/
class MessageRetryRecord {
- private final long failTimeUTC;
- private final int attemptsAlreadyPerformed;
-
- private MessageRetryRecord(int attemptsAlreadyPerformed) {
- this.failTimeUTC = new Date().getTime();
- this.attemptsAlreadyPerformed = attemptsAlreadyPerformed;
- }
+ private final int retryNum;
+ private final long retryTimeUTC;
public MessageRetryRecord() {
this(1);
}
- public MessageRetryRecord retryAgainRecord() {
- return new MessageRetryRecord(this.attemptsAlreadyPerformed + 1);
+ private MessageRetryRecord(int retryNum) {
+ this.retryNum = retryNum;
+ this.retryTimeUTC = new Date().getTime() + calculateRetryDelay(this.retryNum);
}
- public boolean isReadyForRetry() {
- double delayMultiplier = Math.pow(_spoutConfig.retryDelayMultiplier, this.attemptsAlreadyPerformed - 1);
+ public MessageRetryRecord createNextRetryRecord() {
+ return new MessageRetryRecord(this.retryNum + 1);
+ }
+
+ private long calculateRetryDelay(int retryNum) {
+ double delayMultiplier = Math.pow(_spoutConfig.retryDelayMultiplier, retryNum - 1);
long delayThisRetryMs = (long) (_spoutConfig.retryInitialDelayMs * delayMultiplier);
- delayThisRetryMs = Math.min(delayThisRetryMs, _spoutConfig.retryDelayMaxMs);
- return new Date().getTime() - this.failTimeUTC > delayThisRetryMs;
+ return Math.min(delayThisRetryMs, _spoutConfig.retryDelayMaxMs);
+ }
+
+ public boolean isReadyForRetry() {
+ return new Date().getTime() > this.retryTimeUTC;
}
}
}
[36/44] storm git commit: Merge remote-tracking branch 'apache/master'
Posted by bo...@apache.org.
Merge remote-tracking branch 'apache/master'
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e6bed26b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e6bed26b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e6bed26b
Branch: refs/heads/master
Commit: e6bed26b0ef6247f31a7220bed32b77adef80eb9
Parents: 882dfc5 f8bce22
Author: Rick Kilgore <ri...@hbo.com>
Authored: Fri Nov 21 18:24:36 2014 -0800
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Fri Nov 21 18:24:36 2014 -0800
----------------------------------------------------------------------
CHANGELOG.md | 26 +-
DEVELOPER.md | 4 +
LICENSE | 34 +-
README.markdown | 2 +
SECURITY.md | 356 ++-
STORM-UI-REST-API.md | 17 +-
bin/storm | 16 +-
bin/storm.cmd | 11 +-
conf/defaults.yaml | 45 +-
conf/jaas_digest.conf | 8 +-
conf/jaas_kerberos.conf | 15 +
examples/storm-starter/README.markdown | 9 +-
.../storm-starter/multilang/resources/storm.js | 20 +-
.../storm-starter/multilang/resources/storm.py | 87 +-
.../storm-starter/multilang/resources/storm.rb | 90 +-
examples/storm-starter/pom.xml | 2 +-
.../src/jvm/storm/starter/util/StormRunner.java | 3 +-
external/storm-hbase/pom.xml | 4 +-
.../storm/hbase/bolt/HBaseLookupBolt.java | 2 +-
.../hbase/bolt/mapper/HBaseValueMapper.java | 4 +-
.../storm/hbase/trident/state/HBaseState.java | 6 +-
.../hbase/topology/WordCountValueMapper.java | 3 +-
external/storm-hdfs/pom.xml | 4 +-
external/storm-kafka/README.md | 7 +-
external/storm-kafka/pom.xml | 2 +-
.../src/jvm/storm/kafka/KafkaUtils.java | 63 +-
.../src/jvm/storm/kafka/PartitionManager.java | 11 +-
.../jvm/storm/kafka/UpdateOffsetException.java | 5 +
.../src/test/storm/kafka/KafkaUtilsTest.java | 7 +-
logback/cluster.xml | 4 +-
logback/worker.xml | 41 +
pom.xml | 39 +-
.../maven-shade-clojure-transformer/pom.xml | 4 +-
storm-core/pom.xml | 311 +-
.../src/clj/backtype/storm/LocalCluster.clj | 4 +
storm-core/src/clj/backtype/storm/LocalDRPC.clj | 4 +-
.../src/clj/backtype/storm/MockAutoCred.clj | 58 +
storm-core/src/clj/backtype/storm/bootstrap.clj | 5 +-
storm-core/src/clj/backtype/storm/cluster.clj | 167 +-
.../storm/command/upload_credentials.clj | 35 +
storm-core/src/clj/backtype/storm/config.clj | 40 +-
.../backtype/storm/daemon/builtin_metrics.clj | 21 +-
.../src/clj/backtype/storm/daemon/common.clj | 18 +-
.../src/clj/backtype/storm/daemon/drpc.clj | 174 +-
.../src/clj/backtype/storm/daemon/executor.clj | 137 +-
.../src/clj/backtype/storm/daemon/logviewer.clj | 271 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 245 +-
.../clj/backtype/storm/daemon/supervisor.clj | 181 +-
.../src/clj/backtype/storm/daemon/task.clj | 7 +-
.../src/clj/backtype/storm/daemon/worker.clj | 109 +-
storm-core/src/clj/backtype/storm/disruptor.clj | 11 +-
.../src/clj/backtype/storm/messaging/loader.clj | 13 +-
storm-core/src/clj/backtype/storm/testing.clj | 56 +-
storm-core/src/clj/backtype/storm/testing4j.clj | 5 +-
storm-core/src/clj/backtype/storm/thrift.clj | 28 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 213 +-
.../src/clj/backtype/storm/ui/helpers.clj | 69 +-
storm-core/src/clj/backtype/storm/util.clj | 95 +-
storm-core/src/clj/backtype/storm/zookeeper.clj | 19 +-
storm-core/src/clj/storm/trident/testing.clj | 1 +
.../src/dev/drpc-simple-acl-test-scenario.yaml | 11 +
storm-core/src/dev/resources/storm.js | 6 +
storm-core/src/dev/resources/storm.py | 261 +-
storm-core/src/dev/resources/storm.rb | 237 +-
storm-core/src/jvm/backtype/storm/Config.java | 537 +++-
.../jvm/backtype/storm/ConfigValidation.java | 156 +-
.../src/jvm/backtype/storm/Constants.java | 3 +-
.../backtype/storm/ICredentialsListener.java | 32 +
.../src/jvm/backtype/storm/ILocalCluster.java | 2 +
.../src/jvm/backtype/storm/StormSubmitter.java | 164 +-
.../storm/drpc/DRPCInvocationsClient.java | 91 +-
.../src/jvm/backtype/storm/drpc/DRPCSpout.java | 100 +-
.../jvm/backtype/storm/drpc/ReturnResults.java | 35 +-
.../storm/generated/AuthorizationException.java | 345 ++
.../backtype/storm/generated/Credentials.java | 390 +++
.../storm/generated/DistributedRPC.java | 110 +-
.../generated/DistributedRPCInvocations.java | 352 +-
.../jvm/backtype/storm/generated/Nimbus.java | 3006 +++++++++++++++++-
.../backtype/storm/generated/SubmitOptions.java | 98 +-
.../backtype/storm/generated/TopologyInfo.java | 192 +-
.../storm/generated/TopologySummary.java | 192 +-
.../backtype/storm/messaging/netty/Client.java | 73 +-
.../storm/messaging/netty/ControlMessage.java | 4 +-
.../storm/messaging/netty/MessageDecoder.java | 32 +-
.../storm/messaging/netty/MessageEncoder.java | 4 +
.../storm/messaging/netty/SaslMessageToken.java | 99 +
.../storm/messaging/netty/SaslNettyClient.java | 166 +
.../messaging/netty/SaslNettyClientState.java | 31 +
.../storm/messaging/netty/SaslNettyServer.java | 165 +
.../messaging/netty/SaslNettyServerState.java | 31 +
.../messaging/netty/SaslStormClientHandler.java | 158 +
.../netty/SaslStormServerAuthorizeHandler.java | 83 +
.../messaging/netty/SaslStormServerHandler.java | 155 +
.../storm/messaging/netty/SaslUtils.java | 74 +
.../backtype/storm/messaging/netty/Server.java | 104 +-
.../netty/StormClientPipelineFactory.java | 12 +-
.../messaging/netty/StormServerHandler.java | 2 +-
.../netty/StormServerPipelineFactory.java | 20 +-
.../jvm/backtype/storm/scheduler/Cluster.java | 13 +
.../scheduler/multitenant/DefaultPool.java | 219 ++
.../storm/scheduler/multitenant/FreePool.java | 125 +
.../scheduler/multitenant/IsolatedPool.java | 346 ++
.../multitenant/MultitenantScheduler.java | 98 +
.../storm/scheduler/multitenant/Node.java | 343 ++
.../storm/scheduler/multitenant/NodePool.java | 296 ++
.../storm/security/INimbusCredentialPlugin.java | 47 +
.../backtype/storm/security/auth/AuthUtils.java | 228 +-
.../auth/DefaultHttpCredentialsPlugin.java | 87 +
.../security/auth/DefaultPrincipalToLocal.java | 43 +
.../storm/security/auth/IAutoCredentials.java | 55 +
.../security/auth/ICredentialsRenewer.java | 41 +
.../auth/IGroupMappingServiceProvider.java | 42 +
.../security/auth/IHttpCredentialsPlugin.java | 50 +
.../storm/security/auth/IPrincipalToLocal.java | 41 +
.../storm/security/auth/ITransportPlugin.java | 14 +-
.../security/auth/KerberosPrincipalToLocal.java | 45 +
.../storm/security/auth/ReqContext.java | 10 +-
.../security/auth/SaslTransportPlugin.java | 44 +-
.../security/auth/ShellBasedGroupsMapping.java | 94 +
.../security/auth/SimpleTransportPlugin.java | 61 +-
.../security/auth/SingleUserPrincipal.java | 56 +
.../storm/security/auth/TBackoffConnect.java | 77 +
.../storm/security/auth/ThriftClient.java | 85 +-
.../security/auth/ThriftConnectionType.java | 77 +
.../storm/security/auth/ThriftServer.java | 19 +-
.../auth/authorizer/DRPCAuthorizerBase.java | 46 +
.../authorizer/DRPCSimpleACLAuthorizer.java | 157 +
.../auth/authorizer/DenyAuthorizer.java | 4 +-
.../auth/authorizer/NoopAuthorizer.java | 6 +-
.../auth/authorizer/SimpleACLAuthorizer.java | 131 +
.../authorizer/SimpleWhitelistAuthorizer.java | 70 +
.../auth/digest/DigestSaslTransportPlugin.java | 1 +
.../storm/security/auth/hadoop/AutoHDFS.java | 262 ++
.../storm/security/auth/kerberos/AutoTGT.java | 281 ++
.../auth/kerberos/AutoTGTKrb5LoginModule.java | 108 +
.../kerberos/AutoTGTKrb5LoginModuleTest.java | 44 +
.../auth/kerberos/ClientCallbackHandler.java | 104 +
.../kerberos/KerberosSaslTransportPlugin.java | 206 ++
.../auth/kerberos/ServerCallbackHandler.java | 86 +
.../auth/kerberos/jaas_kerberos_cluster.conf | 31 +
.../auth/kerberos/jaas_kerberos_launcher.conf | 12 +
.../jvm/backtype/storm/spout/ShellSpout.java | 65 +-
.../src/jvm/backtype/storm/task/ShellBolt.java | 211 +-
.../testing/ForwardingMetricsConsumer.java | 95 +
.../testing/SingleUserSimpleTransport.java | 37 +
.../state/TestTransactionalState.java | 47 +
.../transactional/state/TransactionalState.java | 56 +-
.../jvm/backtype/storm/utils/DRPCClient.java | 63 +-
.../backtype/storm/utils/DisruptorQueue.java | 56 +-
.../jvm/backtype/storm/utils/LocalState.java | 44 +-
.../jvm/backtype/storm/utils/NimbusClient.java | 11 +-
.../jvm/backtype/storm/utils/ShellUtils.java | 498 +++
.../src/jvm/backtype/storm/utils/TestUtils.java | 34 +
.../src/jvm/backtype/storm/utils/Utils.java | 130 +-
.../backtype/storm/utils/ZookeeperAuthInfo.java | 9 +-
.../storm/utils/ZookeeperServerCnxnFactory.java | 84 +
.../trident/drpc/ReturnResultsReducer.java | 13 +-
.../topology/state/TestTransactionalState.java | 47 +
.../topology/state/TransactionalState.java | 58 +-
storm-core/src/multilang/js/storm.js | 349 ++
storm-core/src/multilang/py/storm.py | 25 +-
storm-core/src/multilang/rb/storm.rb | 55 +-
.../src/native/worker-launcher/.autom4te.cfg | 42 +
.../worker-launcher/.deps/worker-launcher.Po | 1 +
.../src/native/worker-launcher/Makefile.am | 32 +
.../src/native/worker-launcher/configure.ac | 50 +
.../native/worker-launcher/impl/configuration.c | 340 ++
.../native/worker-launcher/impl/configuration.h | 45 +
.../src/native/worker-launcher/impl/main.c | 210 ++
.../worker-launcher/impl/worker-launcher.c | 779 +++++
.../worker-launcher/impl/worker-launcher.h | 129 +
.../worker-launcher/test/test-worker-launcher.c | 340 ++
storm-core/src/py/__init__.py | 16 +
storm-core/src/py/storm/DistributedRPC-remote | 18 +
storm-core/src/py/storm/DistributedRPC.py | 37 +-
.../py/storm/DistributedRPCInvocations-remote | 18 +
.../src/py/storm/DistributedRPCInvocations.py | 96 +-
storm-core/src/py/storm/Nimbus-remote | 25 +
storm-core/src/py/storm/Nimbus.py | 652 +++-
storm-core/src/py/storm/__init__.py | 16 +
storm-core/src/py/storm/constants.py | 16 +
storm-core/src/py/storm/ttypes.py | 1243 +++++++-
storm-core/src/storm.thrift | 58 +-
storm-core/src/ui/public/component.html | 29 +-
storm-core/src/ui/public/css/style.css | 9 +
storm-core/src/ui/public/index.html | 12 +-
storm-core/src/ui/public/js/purl.js | 267 --
storm-core/src/ui/public/js/script.js | 6 +-
storm-core/src/ui/public/js/url.min.js | 1 +
storm-core/src/ui/public/js/visualization.js | 2 +-
.../public/templates/anti-forgery-template.html | 19 +
.../templates/component-page-template.html | 6 +-
.../public/templates/index-page-template.html | 14 +-
.../templates/topology-page-template.html | 29 +-
.../src/ui/public/templates/user-template.html | 25 +
storm-core/src/ui/public/topology.html | 29 +-
.../test/clj/backtype/storm/cluster_test.clj | 93 +-
.../test/clj/backtype/storm/config_test.clj | 11 +
.../test/clj/backtype/storm/drpc_test.clj | 14 +-
.../clj/backtype/storm/local_state_test.clj | 14 +-
.../test/clj/backtype/storm/logviewer_test.clj | 187 ++
.../storm/messaging/netty_integration_test.clj | 4 +-
.../storm/messaging/netty_unit_test.clj | 32 +-
.../test/clj/backtype/storm/nimbus_test.clj | 377 ++-
.../scheduler/multitenant_scheduler_test.clj | 831 +++++
.../storm/security/auth/AuthUtils_test.clj | 16 +-
.../auth/DefaultHttpCredentialsPlugin_test.clj | 40 +
.../storm/security/auth/ThriftClient_test.clj | 28 +-
.../storm/security/auth/ThriftServer_test.clj | 8 +-
.../backtype/storm/security/auth/auth_test.clj | 374 ++-
.../authorizer/DRPCSimpleACLAuthorizer_test.clj | 226 ++
.../security/auth/auto_login_module_test.clj | 91 +
.../storm/security/auth/drpc-auth-alice.jaas | 5 +
.../storm/security/auth/drpc-auth-bob.jaas | 5 +
.../storm/security/auth/drpc-auth-charlie.jaas | 5 +
.../storm/security/auth/drpc-auth-server.jaas | 6 +
.../storm/security/auth/drpc_auth_test.clj | 315 ++
.../storm/security/auth/nimbus_auth_test.clj | 181 ++
.../test/clj/backtype/storm/submitter_test.clj | 75 +
.../test/clj/backtype/storm/supervisor_test.clj | 186 +-
.../test/clj/backtype/storm/testing4j_test.clj | 32 +
.../clj/backtype/storm/transactional_test.clj | 27 +-
.../utils/ZookeeperServerCnxnFactory_test.clj | 35 +
.../test/clj/backtype/storm/utils_test.clj | 58 +-
.../test/clj/storm/trident/state_test.clj | 25 +-
.../storm/utils/DisruptorQueueTest.java | 25 +-
storm-dist/binary/LICENSE | 25 +-
storm-dist/binary/pom.xml | 2 +-
storm-dist/binary/src/main/assembly/binary.xml | 4 +
storm-dist/source/pom.xml | 2 +-
230 files changed, 22991 insertions(+), 1952 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e6bed26b/README.markdown
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e6bed26b/external/storm-kafka/README.md
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e6bed26b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
----------------------------------------------------------------------
[41/44] storm git commit: Merge branch 'master' into retries
Posted by bo...@apache.org.
Merge branch 'master' into retries
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f99fe8ee
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f99fe8ee
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f99fe8ee
Branch: refs/heads/master
Commit: f99fe8ee38249951688d11c8dc93d39d24c6b0ac
Parents: 4be9e52 55c2a84
Author: Rick Kilgore <ri...@hbo.com>
Authored: Wed Dec 3 11:59:37 2014 -0800
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Wed Dec 3 11:59:37 2014 -0800
----------------------------------------------------------------------
CHANGELOG.md | 7 +++++-
README.markdown | 2 ++
conf/defaults.yaml | 1 +
doap_Storm.rdf | 4 ++--
.../multilang/resources/asyncSplitsentence.js | 18 +++++++++++++++
.../multilang/resources/randomsentence.js | 18 +++++++++++++++
.../multilang/resources/splitsentence.js | 18 +++++++++++++++
.../storm-starter/multilang/resources/storm.js | 18 +++++++++++++++
external/storm-hbase/pom.xml | 1 -
.../trident/mapper/TridentHBaseMapper.java | 18 +++++++++++++++
.../hbase/trident/state/HBaseMapState.java | 17 ++++++++++++++
.../hdfs/bolt/format/DefaultSequenceFormat.java | 17 ++++++++++++++
.../storm/hdfs/bolt/format/SequenceFormat.java | 17 ++++++++++++++
.../hdfs/bolt/rotation/TimedRotationPolicy.java | 17 ++++++++++++++
.../hdfs/common/rotation/MoveFileAction.java | 17 ++++++++++++++
.../hdfs/common/rotation/RotationAction.java | 17 ++++++++++++++
.../apache/storm/hdfs/trident/HdfsState.java | 17 ++++++++++++++
.../storm/hdfs/trident/HdfsStateFactory.java | 17 ++++++++++++++
.../apache/storm/hdfs/trident/HdfsUpdater.java | 17 ++++++++++++++
.../trident/format/DefaultSequenceFormat.java | 17 ++++++++++++++
.../hdfs/trident/format/SequenceFormat.java | 17 ++++++++++++++
.../trident/rotation/TimedRotationPolicy.java | 17 ++++++++++++++
.../storm/hdfs/trident/FixedBatchSpout.java | 17 ++++++++++++++
.../storm/hdfs/trident/TridentFileTopology.java | 17 ++++++++++++++
.../hdfs/trident/TridentSequenceTopology.java | 17 ++++++++++++++
external/storm-kafka/README.md | 2 +-
.../jvm/storm/kafka/UpdateOffsetException.java | 17 ++++++++++++++
pom.xml | 11 ++++++---
.../src/clj/backtype/storm/daemon/executor.clj | 4 ++--
storm-core/src/clj/backtype/storm/testing4j.clj | 23 +++++++++++--------
storm-core/src/dev/resources/storm.js | 18 +++++++++++++++
storm-core/src/dev/resources/tester_bolt.js | 17 ++++++++++++++
storm-core/src/dev/resources/tester_spout.js | 17 ++++++++++++++
storm-core/src/jvm/backtype/storm/Config.java | 10 ++++++++
.../auth/authorizer/SimpleACLAuthorizer.java | 2 +-
.../storm/testing/CompleteTopologyParam.java | 22 +++++++++++++++++-
.../storm/testing/PythonShellMetricsBolt.java | 17 ++++++++++++++
.../storm/testing/PythonShellMetricsSpout.java | 17 ++++++++++++++
.../storm/ui/InvalidRequestException.java | 17 ++++++++++++++
storm-core/src/multilang/js/storm.js | 19 ++++++++++++++-
storm-core/src/ui/public/favicon.ico | Bin 0 -> 18280 bytes
41 files changed, 550 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
[29/44] storm git commit: back out deploy to HBO maven server
Posted by bo...@apache.org.
back out deploy to HBO maven server
back out deploy to HBO maven server
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/fabd682a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/fabd682a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/fabd682a
Branch: refs/heads/master
Commit: fabd682a2880cc77ff025cb7c09ad19834d47539
Parents: d49d014
Author: Rick Kilgore <ri...@hbo.com>
Authored: Wed Oct 1 14:16:26 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Wed Oct 1 14:16:26 2014 -0700
----------------------------------------------------------------------
README.markdown | 2 -
examples/storm-starter/pom.xml | 6 +-
external/storm-hdfs/pom.xml | 4 +-
external/storm-kafka/pom.xml | 19 +-
pom.xml | 23 +-
.../maven-shade-clojure-transformer/pom.xml | 4 +-
storm-core/dependency-reduced-pom.xml | 417 +++++++++++++++++++
storm-core/pom.xml | 6 +-
storm-dist/binary/pom.xml | 6 +-
storm-dist/source/pom.xml | 6 +-
10 files changed, 438 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/fabd682a/README.markdown
----------------------------------------------------------------------
diff --git a/README.markdown b/README.markdown
index 8064862..e45f63a 100644
--- a/README.markdown
+++ b/README.markdown
@@ -1,5 +1,3 @@
-This is a forked version of incubator-storm maintained by [HBOCodeLabs](https://github.com/HBOCodeLabs). The main apache project is [here](https://github.com/apache/incubator-storm). In order to make changes take effect within HBO Hurley, you need to run "mvn deploy" in the root project folder from the branch that you want to be exported.
-
Storm is a distributed realtime computation system. Similar to how Hadoop provides a set of general primitives for doing batch processing, Storm provides a set of general primitives for doing realtime computation. Storm is simple, can be used with any programming language, [is used by many companies](http://storm-project.net/documentation/Powered-By.html), and is a lot of fun to use!
The [Rationale page](http://storm-project.net/documentation/Rationale.html) explains what Storm is and why it was built. [This presentation](http://vimeo.com/40972420) is also a good introduction to the project.
http://git-wip-us.apache.org/repos/asf/storm/blob/fabd682a/examples/storm-starter/pom.xml
----------------------------------------------------------------------
diff --git a/examples/storm-starter/pom.xml b/examples/storm-starter/pom.xml
index ccd2770..903c6e7 100644
--- a/examples/storm-starter/pom.xml
+++ b/examples/storm-starter/pom.xml
@@ -19,12 +19,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>storm</artifactId>
- <groupId>com.hbo.hurley</groupId>
+ <groupId>org.apache.storm</groupId>
<version>0.9.3-incubating-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <groupId>com.hbo.hurley</groupId>
+ <groupId>org.apache.storm</groupId>
<artifactId>storm-starter</artifactId>
<packaging>jar</packaging>
@@ -66,7 +66,7 @@
<version>3.0.3</version>
</dependency>
<dependency>
- <groupId>com.hbo.hurley</groupId>
+ <groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>${project.version}</version>
<!-- keep storm out of the jar-with-dependencies -->
http://git-wip-us.apache.org/repos/asf/storm/blob/fabd682a/external/storm-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/pom.xml b/external/storm-hdfs/pom.xml
index 2fc6f54..0d08cae 100644
--- a/external/storm-hdfs/pom.xml
+++ b/external/storm-hdfs/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>storm</artifactId>
- <groupId>com.hbo.hurley</groupId>
+ <groupId>org.apache.storm</groupId>
<version>0.9.3-incubating-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
@@ -37,7 +37,7 @@
<dependencies>
<dependency>
- <groupId>com.hbo.hurley</groupId>
+ <groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
http://git-wip-us.apache.org/repos/asf/storm/blob/fabd682a/external/storm-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-kafka/pom.xml b/external/storm-kafka/pom.xml
index 21a79f1..e94ea12 100644
--- a/external/storm-kafka/pom.xml
+++ b/external/storm-kafka/pom.xml
@@ -20,14 +20,14 @@
<parent>
<artifactId>storm</artifactId>
- <groupId>com.hbo.hurley</groupId>
+ <groupId>org.apache.storm</groupId>
<version>0.9.3-incubating-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<packaging>jar</packaging>
<artifactId>storm-kafka</artifactId>
- <name>storm-kafka-hurley</name>
+ <name>storm-kafka</name>
<description>Storm Spouts for Apache Kafka</description>
<build>
<plugins>
@@ -35,19 +35,6 @@
<sourceDirectory>src/jvm</sourceDirectory>
<testSourceDirectory>src/test</testSourceDirectory>
</build>
-
- <distributionManagement>
- <repository>
- <id>archiva.internal</id>
- <name>Hurley Maven Repository</name>
- <url>http://maven.hurley.hbo.com:8000/repository/internal/</url>
- </repository>
- <snapshotRepository>
- <id>archiva.snapshots</id>
- <name>Hurley Maven Snapshot Repository</name>
- <url>http://maven.hurley.hbo.com:8000/repository/snapshots/</url>
- </snapshotRepository>
- </distributionManagement>
<dependencies>
<dependency>
<groupId>org.mockito</groupId>
@@ -122,7 +109,7 @@
</exclusions>
</dependency>
<dependency>
- <groupId>com.hbo.hurley</groupId>
+ <groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
http://git-wip-us.apache.org/repos/asf/storm/blob/fabd682a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9b54055..111672c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
</parent>
- <groupId>com.hbo.hurley</groupId>
+ <groupId>org.apache.storm</groupId>
<artifactId>storm</artifactId>
<version>0.9.3-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
@@ -287,16 +287,6 @@
<name>Storm Website</name>
<url>file:///tmp/site</url>
</site>
- <repository>
- <id>archiva.internal</id>
- <name>Hurley Maven Repository</name>
- <url>http://maven.hurley.hbo.com:8000/repository/internal/</url>
- </repository>
- <snapshotRepository>
- <id>archiva.snapshots</id>
- <name>Hurley Maven Snapshot Repository</name>
- <url>http://maven.hurley.hbo.com:8000/repository/snapshots/</url>
- </snapshotRepository>
</distributionManagement>
<dependencyManagement>
@@ -526,16 +516,6 @@
<enabled>true</enabled>
</releases>
<snapshots>
- <enabled>true</enabled>
- </snapshots>
- <id>hurley</id>
- <url>http://maven.hurley.hbo.com:8000/repository/snapshots/</url>
- </repository>
- <repository>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
<enabled>false</enabled>
</snapshots>
<id>central</id>
@@ -554,6 +534,7 @@
</repositories>
<build>
+
<pluginManagement>
<plugins>
<plugin>
http://git-wip-us.apache.org/repos/asf/storm/blob/fabd682a/storm-buildtools/maven-shade-clojure-transformer/pom.xml
----------------------------------------------------------------------
diff --git a/storm-buildtools/maven-shade-clojure-transformer/pom.xml b/storm-buildtools/maven-shade-clojure-transformer/pom.xml
index e540355..efd4659 100644
--- a/storm-buildtools/maven-shade-clojure-transformer/pom.xml
+++ b/storm-buildtools/maven-shade-clojure-transformer/pom.xml
@@ -20,12 +20,12 @@
<parent>
<artifactId>storm</artifactId>
- <groupId>com.hbo.hurley</groupId>
+ <groupId>org.apache.storm</groupId>
<version>0.9.3-incubating-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <groupId>com.hbo.hurley</groupId>
+ <groupId>org.apache.storm</groupId>
<artifactId>maven-shade-clojure-transformer</artifactId>
<dependencies>
http://git-wip-us.apache.org/repos/asf/storm/blob/fabd682a/storm-core/dependency-reduced-pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/dependency-reduced-pom.xml b/storm-core/dependency-reduced-pom.xml
new file mode 100644
index 0000000..ec123dd
--- /dev/null
+++ b/storm-core/dependency-reduced-pom.xml
@@ -0,0 +1,417 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <parent>
+ <artifactId>storm</artifactId>
+ <groupId>com.hbo.hurley</groupId>
+ <version>0.9.3-incubating-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>com.hbo.hurley</groupId>
+ <artifactId>storm-core</artifactId>
+ <name>Storm Core</name>
+ <description>Storm Core Java API and Clojure implementation.</description>
+ <build>
+ <sourceDirectory>src/jvm</sourceDirectory>
+ <testSourceDirectory>test/jvm</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>../conf</directory>
+ </resource>
+ <resource>
+ <targetPath>META-INF</targetPath>
+ <directory>../</directory>
+ <includes>
+ <include>NOTICE</include>
+ </includes>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/dev</directory>
+ </testResource>
+ <testResource>
+ <directory>test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <groupId>com.theoryinpractise</groupId>
+ <artifactId>clojure-maven-plugin</artifactId>
+ <extensions>true</extensions>
+ <executions>
+ <execution>
+ <id>compile-clojure</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>test-clojure</id>
+ <phase>test</phase>
+ <goals>
+ <goal>test-with-junit</goal>
+ </goals>
+ <configuration>
+ <vmargs>${test.extra.args}</vmargs>
+ </configuration>
+ </execution>
+ </executions>
+ <configuration>
+ <sourceDirectories>
+ <sourceDirectory>src/clj</sourceDirectory>
+ </sourceDirectories>
+ <testSourceDirectories>
+ <testSourceDirectory>test/clj</testSourceDirectory>
+ </testSourceDirectories>
+ <warnOnReflection>false</warnOnReflection>
+ <copyDeclaredNamespaceOnly>true</copyDeclaredNamespaceOnly>
+ <copiedNamespaces>
+ <copiedNamespace>none</copiedNamespace>
+ </copiedNamespaces>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-surefire-report-plugin</artifactId>
+ <configuration>
+ <reportsDirectories>
+ <file>${project.build.directory}/test-reports</file>
+ </reportsDirectories>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.2</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>com.hbo.hurley</groupId>
+ <artifactId>maven-shade-clojure-transformer</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <configuration>
+ <keepDependenciesWithProvidedScope>true</keepDependenciesWithProvidedScope>
+ <promoteTransitiveDependencies>false</promoteTransitiveDependencies>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <minimizeJar>false</minimizeJar>
+ <artifactSet>
+ <includes>
+ <include>org.apache.thrift:*</include>
+ <include>io.netty:netty</include>
+ <include>com.google.guava:guava</include>
+ <include>org.apache.httpcomponents:http*</include>
+ <include>org.apache.zookeeper:zookeeper</include>
+ <include>org.apache.curator:*</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.thrift</pattern>
+ <shadedPattern>org.apache.thrift7</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.jboss.netty</pattern>
+ <shadedPattern>org.apache.storm.netty</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <shadedPattern>org.apache.storm.guava</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.http</pattern>
+ <shadedPattern>org.apache.storm.http</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.zookeeper</pattern>
+ <shadedPattern>org.apache.storm.zookeeper</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.curator</pattern>
+ <shadedPattern>org.apache.storm.curator</shadedPattern>
+ </relocation>
+ </relocations>
+ <transformers>
+ <transformer />
+ </transformers>
+ <filters>
+ <filter>
+ <artifact>org.apache.thrift:*</artifact>
+ <excludes>
+ <exclude>META-INF/LICENSE.txt</exclude>
+ <exclude>META-INF/NOTICE.txt</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>io.netty:netty</artifact>
+ <excludes>
+ <exclude>META-INF/LICENSE.txt</exclude>
+ <exclude>META-INF/NOTICE.txt</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>commons-httpclient:commons-httpclient</artifact>
+ <excludes>
+ <exclude>META-INF/LICENSE.txt</exclude>
+ <exclude>META-INF/NOTICE.txt</exclude>
+ <exclude>META-INF/README.txt</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>org.apache.zookeeper:zookeeper</artifact>
+ <excludes>
+ <exclude>LICENSE.txt</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>org.clojure</groupId>
+ <artifactId>clojure</artifactId>
+ <version>1.5.1</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>clj-time</groupId>
+ <artifactId>clj-time</artifactId>
+ <version>0.4.1</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>compojure</groupId>
+ <artifactId>compojure</artifactId>
+ <version>1.1.3</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>hiccup</groupId>
+ <artifactId>hiccup</artifactId>
+ <version>0.3.6</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>ring</groupId>
+ <artifactId>ring-devel</artifactId>
+ <version>0.3.11</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>ring</groupId>
+ <artifactId>ring-jetty-adapter</artifactId>
+ <version>0.3.11</version>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>org.mortbay.jetty</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.clojure</groupId>
+ <artifactId>tools.logging</artifactId>
+ <version>0.2.3</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.clojure</groupId>
+ <artifactId>math.numeric-tower</artifactId>
+ <version>0.0.1</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.clojure</groupId>
+ <artifactId>tools.cli</artifactId>
+ <version>0.2.4</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.clojure</groupId>
+ <artifactId>tools.nrepl</artifactId>
+ <version>0.2.3</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>clojure</artifactId>
+ <groupId>org.clojure</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>clojure-complete</groupId>
+ <artifactId>clojure-complete</artifactId>
+ <version>0.2.3</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>clojure</artifactId>
+ <groupId>org.clojure</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.4</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-exec</artifactId>
+ <version>1.1</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.5</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ <version>0.7.0</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>slf4j-api</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>2.5.0</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>log4j</artifactId>
+ <groupId>log4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.googlecode.json-simple</groupId>
+ <artifactId>json-simple</artifactId>
+ <version>1.1</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>carbonite</artifactId>
+ <version>1.4.0</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <version>1.11</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.3.3</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.lmax</groupId>
+ <artifactId>disruptor</artifactId>
+ <version>3.2.1</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.jgrapht</groupId>
+ <artifactId>jgrapht-core</artifactId>
+ <version>0.9.0</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>16.0.1</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>1.0.13</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <version>1.6.6</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ <version>3.9.0.Final</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>1.9.5</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.clojars.runa</groupId>
+ <artifactId>conjure</artifactId>
+ <version>2.1.3</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>reply</groupId>
+ <artifactId>reply</artifactId>
+ <version>0.3.0</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>cd-client</artifactId>
+ <groupId>org.thnetos</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>drawbridge</artifactId>
+ <groupId>com.cemerick</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>versioneer</artifactId>
+ <groupId>trptcolin</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>sjacket</artifactId>
+ <groupId>org.clojars.trptcolin</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+</project>
+
http://git-wip-us.apache.org/repos/asf/storm/blob/fabd682a/storm-core/pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/pom.xml b/storm-core/pom.xml
index 49c0f5a..4c72498 100644
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@ -19,10 +19,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>storm</artifactId>
- <groupId>com.hbo.hurley</groupId>
+ <groupId>org.apache.storm</groupId>
<version>0.9.3-incubating-SNAPSHOT</version>
</parent>
- <groupId>com.hbo.hurley</groupId>
+ <groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<packaging>jar</packaging>
<name>Storm Core</name>
@@ -355,7 +355,7 @@
</configuration>
<dependencies>
<dependency>
- <groupId>com.hbo.hurley</groupId>
+ <groupId>org.apache.storm</groupId>
<artifactId>maven-shade-clojure-transformer</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/storm/blob/fabd682a/storm-dist/binary/pom.xml
----------------------------------------------------------------------
diff --git a/storm-dist/binary/pom.xml b/storm-dist/binary/pom.xml
index 288f05d..0d97c0b 100644
--- a/storm-dist/binary/pom.xml
+++ b/storm-dist/binary/pom.xml
@@ -20,11 +20,11 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>storm</artifactId>
- <groupId>com.hbo.hurley</groupId>
+ <groupId>org.apache.storm</groupId>
<version>0.9.3-incubating-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <groupId>com.hbo.hurley</groupId>
+ <groupId>org.apache.storm</groupId>
<artifactId>apache-storm-bin</artifactId>
<packaging>pom</packaging>
<name>Storm Binary Distribution</name>
@@ -32,7 +32,7 @@
<dependencies>
<dependency>
- <groupId>com.hbo.hurley</groupId>
+ <groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/storm/blob/fabd682a/storm-dist/source/pom.xml
----------------------------------------------------------------------
diff --git a/storm-dist/source/pom.xml b/storm-dist/source/pom.xml
index e61f2df..4fedefb 100644
--- a/storm-dist/source/pom.xml
+++ b/storm-dist/source/pom.xml
@@ -20,11 +20,11 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>storm</artifactId>
- <groupId>com.hbo.hurley</groupId>
+ <groupId>org.apache.storm</groupId>
<version>0.9.3-incubating-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <groupId>com.hbo.hurley</groupId>
+ <groupId>org.apache.storm</groupId>
<artifactId>apache-storm-source</artifactId>
<packaging>pom</packaging>
<name>Storm Source Distribution</name>
@@ -32,7 +32,7 @@
<dependencies>
<dependency>
- <groupId>com.hbo.hurley</groupId>
+ <groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>${project.version}</version>
</dependency>
[44/44] storm git commit: ADDED STORM-495 to Changelog
Posted by bo...@apache.org.
ADDED STORM-495 to Changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/94b96087
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/94b96087
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/94b96087
Branch: refs/heads/master
Commit: 94b96087ba78853b97770a0968a5e9bea63811d4
Parents: 273fa63
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Wed Jan 14 09:58:49 2015 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Wed Jan 14 09:58:49 2015 -0600
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/94b96087/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 44f7bab..857c3fb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,6 +25,7 @@
* STORM-552: Add netty socket backlog config
* STORM-578: Calls to submit-mocked-assignment in supervisor-test use invalid executor-id format
* STORM-600: upgrade jacoco plugin to support jdk8
+ * STORM-495: KafkaSpout retries with exponential backoff
## 0.9.3-rc2
* STORM-558: change "swap!" to "reset!" to fix assignment-versions in supervisor
[18/44] storm git commit: Merge branch 'retries' into hboRepo
Posted by bo...@apache.org.
Merge branch 'retries' into hboRepo
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e54fae31
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e54fae31
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e54fae31
Branch: refs/heads/master
Commit: e54fae316d6fc97f0e35a442901adc5ce20166ab
Parents: 6f6ece9 f4fa76c
Author: Rick Kilgore <ri...@hbo.com>
Authored: Tue Sep 9 16:07:15 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Tue Sep 9 16:07:15 2014 -0700
----------------------------------------------------------------------
external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[05/44] storm git commit: add comment and different default for retry
Posted by bo...@apache.org.
add comment and different default for retry
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/04b64bc6
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/04b64bc6
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/04b64bc6
Branch: refs/heads/master
Commit: 04b64bc66d5c29ba77c455f0125200ed8335376f
Parents: 1c748a7
Author: rick-kilgore <ri...@gmail.com>
Authored: Mon Sep 8 20:49:18 2014 +0000
Committer: rick-kilgore <ri...@gmail.com>
Committed: Mon Sep 8 20:49:18 2014 +0000
----------------------------------------------------------------------
external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/04b64bc6/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java b/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
index d8ca7eb..9a5c311 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
@@ -27,9 +27,12 @@ public class SpoutConfig extends KafkaConfig implements Serializable {
public String zkRoot = null;
public String id = null;
public long stateUpdateIntervalMs = 2000;
- public long retryInitialDelayMs = 10 * 1000;
- public double retryDelayMultiplier = 1.5;
- public long retryMaxDelayMs = 5 * 60 * 1000;
+
+ // Exponential back-off retry settings - note: be sure to set backtype.storm.Config.MESSAGE_TIMEOUT_SECS
+ // appropriately to prevent resubmitting the message while still retrying
+ public long retryInitialDelayMs = 1000;
+ public double retryDelayMultiplier = 1.0;
+ public long retryMaxDelayMs = 1000;
public SpoutConfig(BrokerHosts hosts, String topic, String zkRoot, String id) {
super(hosts, topic);
[11/44] storm git commit: make config names more consistent with each
other
Posted by bo...@apache.org.
make config names more consistent with each other
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/79e3efe2
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/79e3efe2
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/79e3efe2
Branch: refs/heads/master
Commit: 79e3efe2558d252f2122facdb0ce5e69c9c62104
Parents: 0533676
Author: Rick Kilgore <ri...@hbo.com>
Authored: Tue Sep 9 14:52:37 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Tue Sep 9 14:52:37 2014 -0700
----------------------------------------------------------------------
external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java | 4 ++--
external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/79e3efe2/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
index e80b19a..02edf31 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
@@ -290,7 +290,7 @@ public class PartitionManager {
* <ul>
* <li>retryInitialDelayMs - time to delay before the first retry</li>
* <li>retryDelayMultiplier - multiplier by which to increase the delay for each subsequent retry</li>
- * <li>retryMaxDelayMs - maximum retry delay (once this delay time is reached, subsequent retries will
+ * <li>retryDelayMaxMs - maximum retry delay (once this delay time is reached, subsequent retries will
* delay for this amount of time every time)
* </li>
* </ul>
@@ -315,7 +315,7 @@ public class PartitionManager {
public boolean isReadyForRetry() {
double delayMultiplier = Math.pow(_spoutConfig.retryDelayMultiplier, this.attemptsAlreadyPerformed - 1);
long delayThisRetryMs = (long) (_spoutConfig.retryInitialDelayMs * delayMultiplier);
- delayThisRetryMs = Math.min(delayThisRetryMs, _spoutConfig.retryMaxDelayMs);
+ delayThisRetryMs = Math.min(delayThisRetryMs, _spoutConfig.retryDelayMaxMs);
return new Date().getTime() - this.failTimeUTC > delayThisRetryMs;
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/79e3efe2/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java b/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
index 5a5d6f4..fbc887f 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
@@ -32,7 +32,7 @@ public class SpoutConfig extends KafkaConfig implements Serializable {
// appropriately to prevent resubmitting the message while still retrying
public long retryInitialDelayMs = 0;
public double retryDelayMultiplier = 1.0;
- public long retryMaxDelayMs = 60 * 1000;
+ public long retryDelayMaxMs = 60 * 1000;
public SpoutConfig(BrokerHosts hosts, String topic, String zkRoot, String id) {
super(hosts, topic);
[13/44] storm git commit: Merge branch 'retries' into hboRepo
Posted by bo...@apache.org.
Merge branch 'retries' into hboRepo
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/5c1029a9
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/5c1029a9
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/5c1029a9
Branch: refs/heads/master
Commit: 5c1029a921483bc86c0b01fec37bfdaba13a206d
Parents: ea0989d 79e3efe
Author: Rick Kilgore <ri...@hbo.com>
Authored: Tue Sep 9 14:56:00 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Tue Sep 9 14:56:00 2014 -0700
----------------------------------------------------------------------
external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java | 4 ++--
external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[43/44] storm git commit: Merge branch 'retries' of
http://github.com/HBOCodeLabs/incubator-storm into STORM-495
Posted by bo...@apache.org.
Merge branch 'retries' of http://github.com/HBOCodeLabs/incubator-storm into STORM-495
STORM-495: KafkaSpout retries with exponential backoff
Conflicts:
README.markdown
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/273fa637
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/273fa637
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/273fa637
Branch: refs/heads/master
Commit: 273fa637a193f3aa4c50dd901ed50d5a48e25a9e
Parents: fca7c76 fd06698
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Wed Jan 14 09:58:01 2015 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Wed Jan 14 09:58:01 2015 -0600
----------------------------------------------------------------------
README.markdown | 1 +
external/storm-kafka/README.md | 22 ++-
external/storm-kafka/pom.xml | 21 +-
.../ExponentialBackoffMsgRetryManager.java | 167 ++++++++++++++++
.../jvm/storm/kafka/FailedMsgRetryManager.java | 26 +++
.../src/jvm/storm/kafka/PartitionManager.java | 42 ++--
.../src/jvm/storm/kafka/SpoutConfig.java | 8 +
.../ExponentialBackoffMsgRetryManagerTest.java | 194 +++++++++++++++++++
8 files changed, 460 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/273fa637/README.markdown
----------------------------------------------------------------------
diff --cc README.markdown
index 26e773b,946bf28..7eee582
--- a/README.markdown
+++ b/README.markdown
@@@ -179,10 -179,7 +179,11 @@@ under the License
* Anya Tchernishov ([@anyatch](https://github.com/anyatch))
* Ahmet DAL ([@javrasya](https://github.com/javrasya))
* Kai Sasaki ([@Lewuathe](https://github.com/Lewuathe))
+* Matthew Siegel ([@msiegel07](https://github.com/msiegel07))
+* Jun Xia ([@tedxia](https://github.com/tedxia))
+* caofangkun ([@caofangkun](https://github.com/caofangkun))
+* Mansheng Yang ([@lightyang](https://github.com/lightyang))
+ * Rick Kilgore ([@rick-kilgore](http://github.com/rick-kilgore))
## Acknowledgements
http://git-wip-us.apache.org/repos/asf/storm/blob/273fa637/external/storm-kafka/README.md
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/273fa637/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
----------------------------------------------------------------------
[04/44] storm git commit: Merge pull request #1 from
rick-kilgore/retryDelay
Posted by bo...@apache.org.
Merge pull request #1 from rick-kilgore/retryDelay
exponential backoff for failed messages
moving repo to HBOCodeLabs
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/1c748a7f
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/1c748a7f
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/1c748a7f
Branch: refs/heads/master
Commit: 1c748a7fde11ebc7d209c16ad2b30bf19a316395
Parents: 5aae768 6138e94
Author: Rick Kilgore <ri...@hbo.com>
Authored: Sat Sep 6 00:08:12 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Sat Sep 6 00:08:12 2014 -0700
----------------------------------------------------------------------
.../src/jvm/storm/kafka/PartitionManager.java | 66 ++++++++++++++++++--
.../src/jvm/storm/kafka/SpoutConfig.java | 3 +
2 files changed, 65 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[34/44] storm git commit: stop if retries hit
TOPOLOGY_MESSAGE_TIMEOUT_SECS
Posted by bo...@apache.org.
stop if retries hit TOPOLOGY_MESSAGE_TIMEOUT_SECS
print a loud error message with stack trace if the retries run up
against the TOPOLOGY_MESSAGE_TIMEOUT_SECS timeout, and stop retrying
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/882dfc5f
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/882dfc5f
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/882dfc5f
Branch: refs/heads/master
Commit: 882dfc5fb7b3b26a17e4a1ee5f91220cd80fc4dd
Parents: 3c699e3
Author: Rick Kilgore <ri...@hbo.com>
Authored: Thu Oct 2 23:12:01 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Thu Oct 2 23:12:01 2014 -0700
----------------------------------------------------------------------
.../src/jvm/storm/kafka/PartitionManager.java | 96 +++++++++++++++++---
1 file changed, 81 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/882dfc5f/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
index 6b0930b..cff6df0 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
@@ -38,13 +38,24 @@ import java.util.*;
public class PartitionManager {
public static final Logger LOG = LoggerFactory.getLogger(PartitionManager.class);
+ private static final String TIMES_UP_MSG =
+ "Retry logic in your topology is taking longer to complete than is allowed by your"
+ +" Storm Config setting TOPOLOGY_MESSAGE_TIMEOUT_SECS (%s seconds). (i.e., you have"
+ +" called OutputCollector.fail() too many times for this message). KafkaSpout has"
+ +" aborted next retry attempt (retry %s) for the Kafka message at offset %s since it"
+ +" would occur after this timeout.";
+ private static final long TIMEOUT_RESET_VALUE = -1L;
+
private final CombinedMetric _fetchAPILatencyMax;
private final ReducedMetric _fetchAPILatencyMean;
private final CountMetric _fetchAPICallCount;
private final CountMetric _fetchAPIMessageCount;
Long _emittedToOffset;
- SortedSet<Long> _pending = new TreeSet<Long>();
+ // _pending key = Kafka offset, value = time at which the message was first submitted to the topology
+ private SortedMap<Long,Long> _pending = new TreeMap<Long,Long>();
private SortedSet<Long> failed = new TreeSet<Long>();
+
+ // retryRecords key = Kafka offset, value = retry info for the given message
private Map<Long,MessageRetryRecord> retryRecords = new HashMap<Long,MessageRetryRecord>();
Long _committedTo;
LinkedList<MessageAndRealOffset> _waitingToEmit = new LinkedList<MessageAndRealOffset>();
@@ -190,7 +201,9 @@ public class PartitionManager {
}
if (!had_failed || failedReady.contains(cur_offset)) {
numMessages += 1;
- _pending.add(cur_offset);
+ if (!_pending.containsKey(cur_offset)) {
+ _pending.put(cur_offset, System.currentTimeMillis());
+ }
_waitingToEmit.add(new MessageAndRealOffset(msg.message(), cur_offset));
_emittedToOffset = Math.max(msg.nextOffset(), _emittedToOffset);
if (had_failed) {
@@ -203,9 +216,9 @@ public class PartitionManager {
}
public void ack(Long offset) {
- if (!_pending.isEmpty() && _pending.first() < offset - _spoutConfig.maxOffsetBehind) {
+ if (!_pending.isEmpty() && _pending.firstKey() < offset - _spoutConfig.maxOffsetBehind) {
// Too many things pending!
- _pending.headSet(offset - _spoutConfig.maxOffsetBehind).clear();
+ _pending.headMap(offset - _spoutConfig.maxOffsetBehind).clear();
}
_pending.remove(offset);
retryRecords.remove(offset);
@@ -221,15 +234,23 @@ public class PartitionManager {
);
} else {
LOG.debug("failing at offset=" + offset + " with _pending.size()=" + _pending.size() + " pending and _emittedToOffset=" + _emittedToOffset);
- failed.add(offset);
- MessageRetryRecord retryRecord = retryRecords.get(offset);
- retryRecords.put(offset, retryRecord == null
- ? new MessageRetryRecord()
- : retryRecord.createNextRetryRecord());
numberFailed++;
if (numberAcked == 0 && numberFailed > _spoutConfig.maxOffsetBehind) {
throw new RuntimeException("Too many tuple failures");
}
+
+ try {
+ MessageRetryRecord retryRecord = retryRecords.get(offset);
+ retryRecord = retryRecord == null
+ ? new MessageRetryRecord(offset)
+ : retryRecord.createNextRetryRecord();
+
+ retryRecords.put(offset, retryRecord);
+ failed.add(offset);
+
+ } catch (MessageRetryRecord.AvailableRetryTimeExceededException e) {
+ LOG.error("cannot retry", e);
+ }
}
}
@@ -262,7 +283,7 @@ public class PartitionManager {
if (_pending.isEmpty()) {
return _emittedToOffset;
} else {
- return _pending.first();
+ return _pending.firstKey();
}
}
@@ -298,20 +319,59 @@ public class PartitionManager {
* </ul>
*/
class MessageRetryRecord {
+ private final long offset;
private final int retryNum;
private final long retryTimeUTC;
- public MessageRetryRecord() {
- this(1);
+ public MessageRetryRecord(long offset) throws AvailableRetryTimeExceededException {
+ this(offset, 1);
}
- private MessageRetryRecord(int retryNum) {
+ private MessageRetryRecord(long offset, int retryNum) throws AvailableRetryTimeExceededException {
+ this.offset = offset;
this.retryNum = retryNum;
this.retryTimeUTC = System.currentTimeMillis() + calculateRetryDelay();
+ validateRetryTime();
+ }
+
+ /**
+ * Create a MessageRetryRecord for the next retry that should occur after this one.
+ * @return MessageRetryRecord with the next retry time, or null to indicate that another
+ * retry should not be performed. The latter case can happen if we are about to
+ * run into the backtype.storm.Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS in the Storm
+ * configuration.
+ */
+ public MessageRetryRecord createNextRetryRecord() throws AvailableRetryTimeExceededException {
+ return new MessageRetryRecord(this.offset, this.retryNum + 1);
}
- public MessageRetryRecord createNextRetryRecord() {
- return new MessageRetryRecord(this.retryNum + 1);
+ private void validateRetryTime() throws AvailableRetryTimeExceededException {
+ long stormStartTime = PartitionManager.this._pending.get(this.offset);
+
+ if (stormStartTime == TIMEOUT_RESET_VALUE) {
+ // This is a resubmission from the Storm framework after Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS
+ // has elapsed. Restart my timer.
+ PartitionManager.this._pending.put(this.offset, System.currentTimeMillis());
+
+ } else {
+ int timeoutSeconds = Utils.getInt(_stormConf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS));
+ if (this.retryTimeUTC - stormStartTime > timeoutSeconds * 1000) {
+
+ // Prepare for when the Storm framework calls fail()
+ _pending.put(this.offset, TIMEOUT_RESET_VALUE);
+
+ throw new AvailableRetryTimeExceededException(String.format(TIMES_UP_MSG,
+ timeoutSeconds,
+ this.retryNum,
+ this.offset));
+
+ } else {
+ LOG.warn(String.format("allowing another retry: start=%s, retryTime=%s, timeoutSeconds=%s",
+ (stormStartTime / 1000) % 1000,
+ (this.retryTimeUTC / 1000) % 1000,
+ timeoutSeconds));
+ }
+ }
}
private long calculateRetryDelay() {
@@ -323,5 +383,11 @@ public class PartitionManager {
public boolean isReadyForRetry() {
return System.currentTimeMillis() > this.retryTimeUTC;
}
+
+ class AvailableRetryTimeExceededException extends Exception {
+ public AvailableRetryTimeExceededException(String msg) {
+ super(msg);
+ }
+ }
}
}
[02/44] storm git commit: Merge branch 'master' into retryDelay
Posted by bo...@apache.org.
Merge branch 'master' into retryDelay
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f9d2948a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f9d2948a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f9d2948a
Branch: refs/heads/master
Commit: f9d2948a9e2dadbbb99128fd6570038e2096eabd
Parents: cac70e7 58f6582
Author: Rick Kilgore <ri...@hbo.com>
Authored: Wed Sep 3 15:04:32 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Wed Sep 3 15:04:32 2014 -0700
----------------------------------------------------------------------
CHANGELOG.md | 20 +
README.markdown | 2 +
STORM-UI-REST-API.md | 5 +-
bin/storm.cmd | 10 +-
conf/defaults.yaml | 1 +
.../src/jvm/storm/kafka/KafkaUtils.java | 5 +-
storm-core/pom.xml | 53 +-
.../src/clj/backtype/storm/daemon/common.clj | 3 +
.../src/clj/backtype/storm/daemon/nimbus.clj | 4 +-
.../clj/backtype/storm/daemon/supervisor.clj | 24 +-
storm-core/src/clj/backtype/storm/event.clj | 3 +
.../backtype/storm/scheduler/EvenScheduler.clj | 2 +-
storm-core/src/clj/backtype/storm/testing.clj | 9 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 42 +-
storm-core/src/clj/backtype/storm/util.clj | 2 +-
storm-core/src/jvm/backtype/storm/Config.java | 38 +-
.../backtype/storm/messaging/netty/Client.java | 27 +-
.../storm/multilang/JsonSerializer.java | 7 +-
.../DefaultSerializationDelegate.java | 61 ++
.../GzipBridgeSerializationDelegate.java | 64 ++
.../GzipSerializationDelegate.java | 68 +++
.../serialization/SerializationDelegate.java | 35 ++
.../src/jvm/backtype/storm/task/ShellBolt.java | 7 +-
.../StormBoundedExponentialBackoffRetry.java | 74 +++
.../src/jvm/backtype/storm/utils/Utils.java | 91 ++-
.../templates/component-page-template.html | 593 ++++++++++++++-----
.../public/templates/index-page-template.html | 219 +++++--
.../templates/topology-page-template.html | 195 +++++-
.../test/clj/backtype/storm/scheduler_test.clj | 21 +
.../test/clj/backtype/storm/supervisor_test.clj | 60 ++
.../test/clj/backtype/storm/utils_test.clj | 3 +-
.../GzipBridgeSerializationDelegateTest.java | 82 +++
...StormBoundedExponentialBackoffRetryTest.java | 101 ++++
33 files changed, 1615 insertions(+), 316 deletions(-)
----------------------------------------------------------------------
[27/44] storm git commit: Merge branch 'master' into readme
Posted by bo...@apache.org.
Merge branch 'master' into readme
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/9faf431f
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/9faf431f
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/9faf431f
Branch: refs/heads/master
Commit: 9faf431ff347549380e5a1cf34273018daa815ac
Parents: 40ab899 ed59348
Author: Rick Kilgore <ri...@hbo.com>
Authored: Wed Oct 1 13:29:41 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Wed Oct 1 13:29:41 2014 -0700
----------------------------------------------------------------------
CHANGELOG.md | 6 +
DEVELOPER.md | 28 +-
LICENSE | 25 ++
README.markdown | 3 +
dev-tools/jira-github-join.py | 2 +-
.../multilang/resources/asyncSplitsentence.js | 32 ++
.../multilang/resources/randomsentence.js | 75 ++++
.../multilang/resources/splitsentence.js | 26 ++
.../storm-starter/multilang/resources/storm.js | 349 +++++++++++++++++++
.../storm/starter/WordCountTopologyNode.java | 121 +++++++
.../jvm/storm/kafka/DynamicBrokersReader.java | 2 +-
.../storm/kafka/DynamicBrokersReaderTest.java | 1 +
.../clj/backtype/storm/daemon/supervisor.clj | 4 +
storm-core/src/dev/resources/storm.js | 349 +++++++++++++++++++
storm-core/src/dev/resources/tester_bolt.js | 27 ++
storm-core/src/dev/resources/tester_spout.js | 58 +++
.../src/jvm/backtype/storm/StormSubmitter.java | 16 +
storm-core/src/ui/public/component.html | 5 +
storm-core/src/ui/public/images/spinner.gif | Bin 0 -> 23470 bytes
storm-core/src/ui/public/index.html | 5 +
.../src/ui/public/js/jquery.blockUI.min.js | 6 +
storm-core/src/ui/public/js/script.js | 11 +
storm-core/src/ui/public/js/visualization.js | 16 +-
storm-core/src/ui/public/topology.html | 5 +
.../test/clj/backtype/storm/multilang_test.clj | 55 ++-
.../test/clj/backtype/storm/supervisor_test.clj | 2 +
26 files changed, 1179 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/9faf431f/README.markdown
----------------------------------------------------------------------
[32/44] storm git commit: remove whitespace changes
Posted by bo...@apache.org.
remove whitespace changes
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ae710546
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ae710546
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ae710546
Branch: refs/heads/master
Commit: ae710546c4295f0b2a21f8fa993c7f7310855787
Parents: 96dfa41
Author: Rick Kilgore <ri...@hbo.com>
Authored: Wed Oct 1 15:52:00 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Wed Oct 1 15:52:00 2014 -0700
----------------------------------------------------------------------
external/storm-hdfs/pom.xml | 2 +-
storm-buildtools/maven-shade-clojure-transformer/pom.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ae710546/external/storm-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/pom.xml b/external/storm-hdfs/pom.xml
index 0d08cae..2aadac0 100644
--- a/external/storm-hdfs/pom.xml
+++ b/external/storm-hdfs/pom.xml
@@ -65,4 +65,4 @@
</exclusions>
</dependency>
</dependencies>
-</project>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/ae710546/storm-buildtools/maven-shade-clojure-transformer/pom.xml
----------------------------------------------------------------------
diff --git a/storm-buildtools/maven-shade-clojure-transformer/pom.xml b/storm-buildtools/maven-shade-clojure-transformer/pom.xml
index efd4659..a6fbad1 100644
--- a/storm-buildtools/maven-shade-clojure-transformer/pom.xml
+++ b/storm-buildtools/maven-shade-clojure-transformer/pom.xml
@@ -36,4 +36,4 @@
<scope>provided</scope>
</dependency>
</dependencies>
-</project>
+</project>
\ No newline at end of file
[19/44] storm git commit: fixes from review
Posted by bo...@apache.org.
fixes from review
replace new Date().getTime() with System.currentTimeMillis()
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/26af8daa
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/26af8daa
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/26af8daa
Branch: refs/heads/master
Commit: 26af8daa3e16e2199828c7760a68849dbbd27af1
Parents: f4fa76c
Author: Rick Kilgore <ri...@hbo.com>
Authored: Wed Sep 10 16:05:36 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Wed Sep 10 16:05:36 2014 -0700
----------------------------------------------------------------------
external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/26af8daa/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
index 36a0e85..6b0930b 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
@@ -307,7 +307,7 @@ public class PartitionManager {
private MessageRetryRecord(int retryNum) {
this.retryNum = retryNum;
- this.retryTimeUTC = new Date().getTime() + calculateRetryDelay();
+ this.retryTimeUTC = System.currentTimeMillis() + calculateRetryDelay();
}
public MessageRetryRecord createNextRetryRecord() {
@@ -321,7 +321,7 @@ public class PartitionManager {
}
public boolean isReadyForRetry() {
- return new Date().getTime() > this.retryTimeUTC;
+ return System.currentTimeMillis() > this.retryTimeUTC;
}
}
}
[08/44] storm git commit: change everything to com.hbo.hurley
Posted by bo...@apache.org.
change everything to com.hbo.hurley
change everything to com.hbo.hurley so relative links work
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b95b6d68
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b95b6d68
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b95b6d68
Branch: refs/heads/master
Commit: b95b6d68c54b3e80431c73609870c6935b6d56ce
Parents: cbbea7c
Author: rick-kilgore <ri...@gmail.com>
Authored: Tue Sep 9 00:17:29 2014 +0000
Committer: rick-kilgore <ri...@gmail.com>
Committed: Tue Sep 9 00:17:29 2014 +0000
----------------------------------------------------------------------
examples/storm-starter/pom.xml | 6 +-
external/storm-hdfs/pom.xml | 6 +-
external/storm-kafka/gs | 7 +
external/storm-kafka/pom.xml | 7 +-
pom.xml | 2 +-
.../maven-shade-clojure-transformer/pom.xml | 6 +-
storm-core/dependency-reduced-pom.xml | 417 +++++++++++++++++++
storm-core/pom.xml | 6 +-
storm-dist/binary/pom.xml | 6 +-
storm-dist/source/pom.xml | 6 +-
10 files changed, 446 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b95b6d68/examples/storm-starter/pom.xml
----------------------------------------------------------------------
diff --git a/examples/storm-starter/pom.xml b/examples/storm-starter/pom.xml
index 903c6e7..ccd2770 100644
--- a/examples/storm-starter/pom.xml
+++ b/examples/storm-starter/pom.xml
@@ -19,12 +19,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>storm</artifactId>
- <groupId>org.apache.storm</groupId>
+ <groupId>com.hbo.hurley</groupId>
<version>0.9.3-incubating-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <groupId>org.apache.storm</groupId>
+ <groupId>com.hbo.hurley</groupId>
<artifactId>storm-starter</artifactId>
<packaging>jar</packaging>
@@ -66,7 +66,7 @@
<version>3.0.3</version>
</dependency>
<dependency>
- <groupId>org.apache.storm</groupId>
+ <groupId>com.hbo.hurley</groupId>
<artifactId>storm-core</artifactId>
<version>${project.version}</version>
<!-- keep storm out of the jar-with-dependencies -->
http://git-wip-us.apache.org/repos/asf/storm/blob/b95b6d68/external/storm-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/pom.xml b/external/storm-hdfs/pom.xml
index 2aadac0..2fc6f54 100644
--- a/external/storm-hdfs/pom.xml
+++ b/external/storm-hdfs/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>storm</artifactId>
- <groupId>org.apache.storm</groupId>
+ <groupId>com.hbo.hurley</groupId>
<version>0.9.3-incubating-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
@@ -37,7 +37,7 @@
<dependencies>
<dependency>
- <groupId>org.apache.storm</groupId>
+ <groupId>com.hbo.hurley</groupId>
<artifactId>storm-core</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
@@ -65,4 +65,4 @@
</exclusions>
</dependency>
</dependencies>
-</project>
\ No newline at end of file
+</project>
http://git-wip-us.apache.org/repos/asf/storm/blob/b95b6d68/external/storm-kafka/gs
----------------------------------------------------------------------
diff --git a/external/storm-kafka/gs b/external/storm-kafka/gs
new file mode 100644
index 0000000..9f6e837
--- /dev/null
+++ b/external/storm-kafka/gs
@@ -0,0 +1,7 @@
+changes made since retries:
+
+ 45.9% external/storm-kafka/
+
+ external/storm-kafka/pom.xml | 18 [1;32m+++++++++++++++[m[1;31m---[m
+ pom.xml | 21 [1;32m++++++++++++++++++++[m[1;31m-[m
+ 2 files changed, 35 insertions(+), 4 deletions(-)
http://git-wip-us.apache.org/repos/asf/storm/blob/b95b6d68/external/storm-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-kafka/pom.xml b/external/storm-kafka/pom.xml
index 4f960f3..21a79f1 100644
--- a/external/storm-kafka/pom.xml
+++ b/external/storm-kafka/pom.xml
@@ -20,14 +20,13 @@
<parent>
<artifactId>storm</artifactId>
- <groupId>org.apache.storm</groupId>
+ <groupId>com.hbo.hurley</groupId>
<version>0.9.3-incubating-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<packaging>jar</packaging>
- <groupId>com.hbo.hurley</groupId>
- <artifactId>storm-kafka-hurley</artifactId>
+ <artifactId>storm-kafka</artifactId>
<name>storm-kafka-hurley</name>
<description>Storm Spouts for Apache Kafka</description>
<build>
@@ -123,7 +122,7 @@
</exclusions>
</dependency>
<dependency>
- <groupId>org.apache.storm</groupId>
+ <groupId>com.hbo.hurley</groupId>
<artifactId>storm-core</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
http://git-wip-us.apache.org/repos/asf/storm/blob/b95b6d68/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 825cc38..9b54055 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
</parent>
- <groupId>org.apache.storm</groupId>
+ <groupId>com.hbo.hurley</groupId>
<artifactId>storm</artifactId>
<version>0.9.3-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/storm/blob/b95b6d68/storm-buildtools/maven-shade-clojure-transformer/pom.xml
----------------------------------------------------------------------
diff --git a/storm-buildtools/maven-shade-clojure-transformer/pom.xml b/storm-buildtools/maven-shade-clojure-transformer/pom.xml
index a6fbad1..e540355 100644
--- a/storm-buildtools/maven-shade-clojure-transformer/pom.xml
+++ b/storm-buildtools/maven-shade-clojure-transformer/pom.xml
@@ -20,12 +20,12 @@
<parent>
<artifactId>storm</artifactId>
- <groupId>org.apache.storm</groupId>
+ <groupId>com.hbo.hurley</groupId>
<version>0.9.3-incubating-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <groupId>org.apache.storm</groupId>
+ <groupId>com.hbo.hurley</groupId>
<artifactId>maven-shade-clojure-transformer</artifactId>
<dependencies>
@@ -36,4 +36,4 @@
<scope>provided</scope>
</dependency>
</dependencies>
-</project>
\ No newline at end of file
+</project>
http://git-wip-us.apache.org/repos/asf/storm/blob/b95b6d68/storm-core/dependency-reduced-pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/dependency-reduced-pom.xml b/storm-core/dependency-reduced-pom.xml
new file mode 100644
index 0000000..ec123dd
--- /dev/null
+++ b/storm-core/dependency-reduced-pom.xml
@@ -0,0 +1,417 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <parent>
+ <artifactId>storm</artifactId>
+ <groupId>com.hbo.hurley</groupId>
+ <version>0.9.3-incubating-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>com.hbo.hurley</groupId>
+ <artifactId>storm-core</artifactId>
+ <name>Storm Core</name>
+ <description>Storm Core Java API and Clojure implementation.</description>
+ <build>
+ <sourceDirectory>src/jvm</sourceDirectory>
+ <testSourceDirectory>test/jvm</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>../conf</directory>
+ </resource>
+ <resource>
+ <targetPath>META-INF</targetPath>
+ <directory>../</directory>
+ <includes>
+ <include>NOTICE</include>
+ </includes>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/dev</directory>
+ </testResource>
+ <testResource>
+ <directory>test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <groupId>com.theoryinpractise</groupId>
+ <artifactId>clojure-maven-plugin</artifactId>
+ <extensions>true</extensions>
+ <executions>
+ <execution>
+ <id>compile-clojure</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>test-clojure</id>
+ <phase>test</phase>
+ <goals>
+ <goal>test-with-junit</goal>
+ </goals>
+ <configuration>
+ <vmargs>${test.extra.args}</vmargs>
+ </configuration>
+ </execution>
+ </executions>
+ <configuration>
+ <sourceDirectories>
+ <sourceDirectory>src/clj</sourceDirectory>
+ </sourceDirectories>
+ <testSourceDirectories>
+ <testSourceDirectory>test/clj</testSourceDirectory>
+ </testSourceDirectories>
+ <warnOnReflection>false</warnOnReflection>
+ <copyDeclaredNamespaceOnly>true</copyDeclaredNamespaceOnly>
+ <copiedNamespaces>
+ <copiedNamespace>none</copiedNamespace>
+ </copiedNamespaces>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-surefire-report-plugin</artifactId>
+ <configuration>
+ <reportsDirectories>
+ <file>${project.build.directory}/test-reports</file>
+ </reportsDirectories>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.2</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>com.hbo.hurley</groupId>
+ <artifactId>maven-shade-clojure-transformer</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <configuration>
+ <keepDependenciesWithProvidedScope>true</keepDependenciesWithProvidedScope>
+ <promoteTransitiveDependencies>false</promoteTransitiveDependencies>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <minimizeJar>false</minimizeJar>
+ <artifactSet>
+ <includes>
+ <include>org.apache.thrift:*</include>
+ <include>io.netty:netty</include>
+ <include>com.google.guava:guava</include>
+ <include>org.apache.httpcomponents:http*</include>
+ <include>org.apache.zookeeper:zookeeper</include>
+ <include>org.apache.curator:*</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.thrift</pattern>
+ <shadedPattern>org.apache.thrift7</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.jboss.netty</pattern>
+ <shadedPattern>org.apache.storm.netty</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <shadedPattern>org.apache.storm.guava</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.http</pattern>
+ <shadedPattern>org.apache.storm.http</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.zookeeper</pattern>
+ <shadedPattern>org.apache.storm.zookeeper</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.curator</pattern>
+ <shadedPattern>org.apache.storm.curator</shadedPattern>
+ </relocation>
+ </relocations>
+ <transformers>
+ <transformer />
+ </transformers>
+ <filters>
+ <filter>
+ <artifact>org.apache.thrift:*</artifact>
+ <excludes>
+ <exclude>META-INF/LICENSE.txt</exclude>
+ <exclude>META-INF/NOTICE.txt</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>io.netty:netty</artifact>
+ <excludes>
+ <exclude>META-INF/LICENSE.txt</exclude>
+ <exclude>META-INF/NOTICE.txt</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>commons-httpclient:commons-httpclient</artifact>
+ <excludes>
+ <exclude>META-INF/LICENSE.txt</exclude>
+ <exclude>META-INF/NOTICE.txt</exclude>
+ <exclude>META-INF/README.txt</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>org.apache.zookeeper:zookeeper</artifact>
+ <excludes>
+ <exclude>LICENSE.txt</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>org.clojure</groupId>
+ <artifactId>clojure</artifactId>
+ <version>1.5.1</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>clj-time</groupId>
+ <artifactId>clj-time</artifactId>
+ <version>0.4.1</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>compojure</groupId>
+ <artifactId>compojure</artifactId>
+ <version>1.1.3</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>hiccup</groupId>
+ <artifactId>hiccup</artifactId>
+ <version>0.3.6</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>ring</groupId>
+ <artifactId>ring-devel</artifactId>
+ <version>0.3.11</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>ring</groupId>
+ <artifactId>ring-jetty-adapter</artifactId>
+ <version>0.3.11</version>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>org.mortbay.jetty</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.clojure</groupId>
+ <artifactId>tools.logging</artifactId>
+ <version>0.2.3</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.clojure</groupId>
+ <artifactId>math.numeric-tower</artifactId>
+ <version>0.0.1</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.clojure</groupId>
+ <artifactId>tools.cli</artifactId>
+ <version>0.2.4</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.clojure</groupId>
+ <artifactId>tools.nrepl</artifactId>
+ <version>0.2.3</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>clojure</artifactId>
+ <groupId>org.clojure</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>clojure-complete</groupId>
+ <artifactId>clojure-complete</artifactId>
+ <version>0.2.3</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>clojure</artifactId>
+ <groupId>org.clojure</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.4</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-exec</artifactId>
+ <version>1.1</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.5</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ <version>0.7.0</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>slf4j-api</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>2.5.0</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>log4j</artifactId>
+ <groupId>log4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.googlecode.json-simple</groupId>
+ <artifactId>json-simple</artifactId>
+ <version>1.1</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>carbonite</artifactId>
+ <version>1.4.0</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <version>1.11</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.3.3</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.lmax</groupId>
+ <artifactId>disruptor</artifactId>
+ <version>3.2.1</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.jgrapht</groupId>
+ <artifactId>jgrapht-core</artifactId>
+ <version>0.9.0</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>16.0.1</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>1.0.13</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <version>1.6.6</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ <version>3.9.0.Final</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>1.9.5</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.clojars.runa</groupId>
+ <artifactId>conjure</artifactId>
+ <version>2.1.3</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>reply</groupId>
+ <artifactId>reply</artifactId>
+ <version>0.3.0</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>cd-client</artifactId>
+ <groupId>org.thnetos</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>drawbridge</artifactId>
+ <groupId>com.cemerick</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>versioneer</artifactId>
+ <groupId>trptcolin</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>sjacket</artifactId>
+ <groupId>org.clojars.trptcolin</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+</project>
+
http://git-wip-us.apache.org/repos/asf/storm/blob/b95b6d68/storm-core/pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/pom.xml b/storm-core/pom.xml
index 4c72498..49c0f5a 100644
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@ -19,10 +19,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>storm</artifactId>
- <groupId>org.apache.storm</groupId>
+ <groupId>com.hbo.hurley</groupId>
<version>0.9.3-incubating-SNAPSHOT</version>
</parent>
- <groupId>org.apache.storm</groupId>
+ <groupId>com.hbo.hurley</groupId>
<artifactId>storm-core</artifactId>
<packaging>jar</packaging>
<name>Storm Core</name>
@@ -355,7 +355,7 @@
</configuration>
<dependencies>
<dependency>
- <groupId>org.apache.storm</groupId>
+ <groupId>com.hbo.hurley</groupId>
<artifactId>maven-shade-clojure-transformer</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/storm/blob/b95b6d68/storm-dist/binary/pom.xml
----------------------------------------------------------------------
diff --git a/storm-dist/binary/pom.xml b/storm-dist/binary/pom.xml
index 0d97c0b..288f05d 100644
--- a/storm-dist/binary/pom.xml
+++ b/storm-dist/binary/pom.xml
@@ -20,11 +20,11 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>storm</artifactId>
- <groupId>org.apache.storm</groupId>
+ <groupId>com.hbo.hurley</groupId>
<version>0.9.3-incubating-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <groupId>org.apache.storm</groupId>
+ <groupId>com.hbo.hurley</groupId>
<artifactId>apache-storm-bin</artifactId>
<packaging>pom</packaging>
<name>Storm Binary Distribution</name>
@@ -32,7 +32,7 @@
<dependencies>
<dependency>
- <groupId>org.apache.storm</groupId>
+ <groupId>com.hbo.hurley</groupId>
<artifactId>storm-core</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/storm/blob/b95b6d68/storm-dist/source/pom.xml
----------------------------------------------------------------------
diff --git a/storm-dist/source/pom.xml b/storm-dist/source/pom.xml
index 4fedefb..e61f2df 100644
--- a/storm-dist/source/pom.xml
+++ b/storm-dist/source/pom.xml
@@ -20,11 +20,11 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>storm</artifactId>
- <groupId>org.apache.storm</groupId>
+ <groupId>com.hbo.hurley</groupId>
<version>0.9.3-incubating-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <groupId>org.apache.storm</groupId>
+ <groupId>com.hbo.hurley</groupId>
<artifactId>apache-storm-source</artifactId>
<packaging>pom</packaging>
<name>Storm Source Distribution</name>
@@ -32,7 +32,7 @@
<dependencies>
<dependency>
- <groupId>org.apache.storm</groupId>
+ <groupId>com.hbo.hurley</groupId>
<artifactId>storm-core</artifactId>
<version>${project.version}</version>
</dependency>
[40/44] storm git commit: Merge remote-tracking branch 'apache/master'
Posted by bo...@apache.org.
Merge remote-tracking branch 'apache/master'
Conflicts:
README.markdown
external/storm-kafka/README.md
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/55c2a84d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/55c2a84d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/55c2a84d
Branch: refs/heads/master
Commit: 55c2a84db3cb4f2a2fd7edb8c8b48f19cadd90f2
Parents: 9583fea c064dba
Author: Rick Kilgore <ri...@hbo.com>
Authored: Wed Dec 3 11:59:19 2014 -0800
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Wed Dec 3 11:59:19 2014 -0800
----------------------------------------------------------------------
CHANGELOG.md | 7 +++++-
README.markdown | 2 ++
conf/defaults.yaml | 1 +
doap_Storm.rdf | 4 ++--
.../multilang/resources/asyncSplitsentence.js | 18 +++++++++++++++
.../multilang/resources/randomsentence.js | 18 +++++++++++++++
.../multilang/resources/splitsentence.js | 18 +++++++++++++++
.../storm-starter/multilang/resources/storm.js | 18 +++++++++++++++
external/storm-hbase/pom.xml | 1 -
.../trident/mapper/TridentHBaseMapper.java | 18 +++++++++++++++
.../hbase/trident/state/HBaseMapState.java | 17 ++++++++++++++
.../hdfs/bolt/format/DefaultSequenceFormat.java | 17 ++++++++++++++
.../storm/hdfs/bolt/format/SequenceFormat.java | 17 ++++++++++++++
.../hdfs/bolt/rotation/TimedRotationPolicy.java | 17 ++++++++++++++
.../hdfs/common/rotation/MoveFileAction.java | 17 ++++++++++++++
.../hdfs/common/rotation/RotationAction.java | 17 ++++++++++++++
.../apache/storm/hdfs/trident/HdfsState.java | 17 ++++++++++++++
.../storm/hdfs/trident/HdfsStateFactory.java | 17 ++++++++++++++
.../apache/storm/hdfs/trident/HdfsUpdater.java | 17 ++++++++++++++
.../trident/format/DefaultSequenceFormat.java | 17 ++++++++++++++
.../hdfs/trident/format/SequenceFormat.java | 17 ++++++++++++++
.../trident/rotation/TimedRotationPolicy.java | 17 ++++++++++++++
.../storm/hdfs/trident/FixedBatchSpout.java | 17 ++++++++++++++
.../storm/hdfs/trident/TridentFileTopology.java | 17 ++++++++++++++
.../hdfs/trident/TridentSequenceTopology.java | 17 ++++++++++++++
external/storm-kafka/README.md | 2 +-
.../jvm/storm/kafka/UpdateOffsetException.java | 17 ++++++++++++++
pom.xml | 11 ++++++---
.../src/clj/backtype/storm/daemon/executor.clj | 4 ++--
storm-core/src/clj/backtype/storm/testing4j.clj | 23 +++++++++++--------
storm-core/src/dev/resources/storm.js | 18 +++++++++++++++
storm-core/src/dev/resources/tester_bolt.js | 17 ++++++++++++++
storm-core/src/dev/resources/tester_spout.js | 17 ++++++++++++++
storm-core/src/jvm/backtype/storm/Config.java | 10 ++++++++
.../auth/authorizer/SimpleACLAuthorizer.java | 2 +-
.../storm/testing/CompleteTopologyParam.java | 22 +++++++++++++++++-
.../storm/testing/PythonShellMetricsBolt.java | 17 ++++++++++++++
.../storm/testing/PythonShellMetricsSpout.java | 17 ++++++++++++++
.../storm/ui/InvalidRequestException.java | 17 ++++++++++++++
storm-core/src/multilang/js/storm.js | 19 ++++++++++++++-
storm-core/src/ui/public/favicon.ico | Bin 0 -> 18280 bytes
41 files changed, 550 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/55c2a84d/README.markdown
----------------------------------------------------------------------
diff --cc README.markdown
index 337cd49,1077bd6..946bf28
--- a/README.markdown
+++ b/README.markdown
@@@ -177,7 -177,8 +177,9 @@@ under the License
* Dane Hammer ([@danehammer](https://github.com/danehammer))
* Christophe Carre' ([@chrisz](https://github.com/chrisz))
* Anya Tchernishov ([@anyatch](https://github.com/anyatch))
+ * Ahmet DAL ([@javrasya](https://github.com/javrasya))
+ * Kai Sasaki ([@Lewuathe](https://github.com/Lewuathe))
+* Rick Kilgore ([@rick-kilgore](http://github.com/rick-kilgore))
## Acknowledgements
http://git-wip-us.apache.org/repos/asf/storm/blob/55c2a84d/external/storm-kafka/README.md
----------------------------------------------------------------------
[39/44] storm git commit: Merge branch 'master' into retries
Posted by bo...@apache.org.
Merge branch 'master' into retries
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/4be9e52b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/4be9e52b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/4be9e52b
Branch: refs/heads/master
Commit: 4be9e52b235c390b6746ef290c0422aec104fb00
Parents: a95a188 9583fea
Author: Rick Kilgore <ri...@hbo.com>
Authored: Mon Nov 24 22:39:40 2014 -0800
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Mon Nov 24 22:39:40 2014 -0800
----------------------------------------------------------------------
doap_Storm.rdf | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 57 insertions(+)
----------------------------------------------------------------------
[38/44] storm git commit: Merge remote-tracking branch 'apache/master'
Posted by bo...@apache.org.
Merge remote-tracking branch 'apache/master'
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/9583feaf
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/9583feaf
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/9583feaf
Branch: refs/heads/master
Commit: 9583feaf87538e4e9aca9aaf405e02834e5ee9bf
Parents: e6bed26 3c4133c
Author: Rick Kilgore <ri...@hbo.com>
Authored: Mon Nov 24 22:39:23 2014 -0800
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Mon Nov 24 22:39:23 2014 -0800
----------------------------------------------------------------------
doap_Storm.rdf | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 57 insertions(+)
----------------------------------------------------------------------
[16/44] storm git commit: Merge branch 'retries' into hboRepo
Posted by bo...@apache.org.
Merge branch 'retries' into hboRepo
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6f6ece94
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6f6ece94
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6f6ece94
Branch: refs/heads/master
Commit: 6f6ece94743eb5432f079db636f4c76dc880f076
Parents: 49f91dc 2aa0346
Author: Rick Kilgore <ri...@hbo.com>
Authored: Tue Sep 9 16:04:30 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Tue Sep 9 16:04:30 2014 -0700
----------------------------------------------------------------------
.../src/jvm/storm/kafka/PartitionManager.java | 33 +++++++++++---------
1 file changed, 19 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
[42/44] storm git commit: fixes from Bobby Evans review
Posted by bo...@apache.org.
fixes from Bobby Evans review
fix potential long overflow bug
employ a use-case-specific Comparator instead of an ill-advised, general
Comparable implementation for sorting retry records in the PriorityQueue
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/fd066985
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/fd066985
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/fd066985
Branch: refs/heads/master
Commit: fd066985a74c3140f61a465cfc49c83a6ddfa713
Parents: f99fe8e
Author: Rick Kilgore <ri...@hbo.com>
Authored: Fri Jan 9 13:34:29 2015 -0800
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Fri Jan 9 13:34:29 2015 -0800
----------------------------------------------------------------------
.../ExponentialBackoffMsgRetryManager.java | 29 ++++++++++++++------
1 file changed, 21 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/fd066985/external/storm-kafka/src/jvm/storm/kafka/ExponentialBackoffMsgRetryManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/ExponentialBackoffMsgRetryManager.java b/external/storm-kafka/src/jvm/storm/kafka/ExponentialBackoffMsgRetryManager.java
index cace9cd..9b2c904 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/ExponentialBackoffMsgRetryManager.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/ExponentialBackoffMsgRetryManager.java
@@ -17,10 +17,11 @@
*/
package storm.kafka;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
import java.util.PriorityQueue;
import java.util.Queue;
-import java.util.SortedMap;
-import java.util.TreeMap;
public class ExponentialBackoffMsgRetryManager implements FailedMsgRetryManager {
@@ -28,8 +29,8 @@ public class ExponentialBackoffMsgRetryManager implements FailedMsgRetryManager
private final double retryDelayMultiplier;
private final long retryDelayMaxMs;
- private Queue<MessageRetryRecord> waiting = new PriorityQueue<MessageRetryRecord>();
- private SortedMap<Long,MessageRetryRecord> records = new TreeMap<Long,MessageRetryRecord>();
+ private Queue<MessageRetryRecord> waiting = new PriorityQueue<MessageRetryRecord>(11, new RetryTimeComparator());
+ private Map<Long,MessageRetryRecord> records = new HashMap<Long,MessageRetryRecord>();
public ExponentialBackoffMsgRetryManager(long retryInitialDelayMs, double retryDelayMultiplier, long retryDelayMaxMs) {
this.retryInitialDelayMs = retryInitialDelayMs;
@@ -103,7 +104,7 @@ public class ExponentialBackoffMsgRetryManager implements FailedMsgRetryManager
* </li>
* </ul>
*/
- class MessageRetryRecord implements Comparable<MessageRetryRecord> {
+ class MessageRetryRecord {
private final long offset;
private final int retryNum;
private final long retryTimeUTC;
@@ -131,7 +132,11 @@ public class ExponentialBackoffMsgRetryManager implements FailedMsgRetryManager
private long calculateRetryDelay() {
double delayMultiplier = Math.pow(retryDelayMultiplier, this.retryNum - 1);
- long delayThisRetryMs = (long) (retryInitialDelayMs * delayMultiplier);
+ double delay = retryInitialDelayMs * delayMultiplier;
+ Long maxLong = Long.MAX_VALUE;
+ long delayThisRetryMs = delay >= maxLong.doubleValue()
+ ? maxLong
+ : (long) delay;
return Math.min(delayThisRetryMs, retryDelayMaxMs);
}
@@ -145,10 +150,18 @@ public class ExponentialBackoffMsgRetryManager implements FailedMsgRetryManager
public int hashCode() {
return Long.valueOf(this.offset).hashCode();
}
+ }
+
+ class RetryTimeComparator implements Comparator<MessageRetryRecord> {
+
+ @Override
+ public int compare(MessageRetryRecord record1, MessageRetryRecord record2) {
+ return Long.compare(record1.retryTimeUTC, record2.retryTimeUTC);
+ }
@Override
- public int compareTo(MessageRetryRecord other) {
- return Long.compare(this.retryTimeUTC, other.retryTimeUTC);
+ public boolean equals(Object obj) {
+ return false;
}
}
}
[35/44] storm git commit: separate retry logic into manager class
Posted by bo...@apache.org.
separate retry logic into manager class
separate retry logic into manager class ExponentialBackoffMsgRetryManagerTest
fixed logic with regards to TOPOLOGY_MESSAGE_TIMEOUT_SECS (this is not a max
time across retries, but rather a max time within a retry - so there is
no conflict and no need to account for it in the retry logic).
tests for ExponentialBackoffMsgRetryManagerTest
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/02bffc60
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/02bffc60
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/02bffc60
Branch: refs/heads/master
Commit: 02bffc601e71d89948a7c5c214d1ae69ccc8ef1d
Parents: 882dfc5
Author: Rick Kilgore <ri...@hbo.com>
Authored: Fri Nov 21 18:00:51 2014 -0800
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Fri Nov 21 18:00:51 2014 -0800
----------------------------------------------------------------------
external/storm-kafka/pom.xml | 21 +-
.../ExponentialBackoffMsgRetryManager.java | 154 +++++++++++++++
.../jvm/storm/kafka/FailedMsgRetryManager.java | 26 +++
.../src/jvm/storm/kafka/PartitionManager.java | 146 ++------------
.../src/jvm/storm/kafka/SpoutConfig.java | 3 -
.../ExponentialBackoffMsgRetryManagerTest.java | 194 +++++++++++++++++++
6 files changed, 408 insertions(+), 136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/02bffc60/external/storm-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-kafka/pom.xml b/external/storm-kafka/pom.xml
index 05bb117..f17af74 100644
--- a/external/storm-kafka/pom.xml
+++ b/external/storm-kafka/pom.xml
@@ -31,7 +31,26 @@
<description>Storm Spouts for Apache Kafka</description>
<build>
<plugins>
-
+ <plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ <version>0.7.2.201409121644</version>
+ <executions>
+ <execution>
+ <id>jacoco-initialize</id>
+ <goals>
+ <goal>prepare-agent</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>jacoco-report</id>
+ <phase>test</phase>
+ <goals>
+ <goal>report</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
<sourceDirectory>src/jvm</sourceDirectory>
<testSourceDirectory>src/test</testSourceDirectory>
http://git-wip-us.apache.org/repos/asf/storm/blob/02bffc60/external/storm-kafka/src/jvm/storm/kafka/ExponentialBackoffMsgRetryManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/ExponentialBackoffMsgRetryManager.java b/external/storm-kafka/src/jvm/storm/kafka/ExponentialBackoffMsgRetryManager.java
new file mode 100644
index 0000000..cace9cd
--- /dev/null
+++ b/external/storm-kafka/src/jvm/storm/kafka/ExponentialBackoffMsgRetryManager.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package storm.kafka;
+
+import java.util.PriorityQueue;
+import java.util.Queue;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+public class ExponentialBackoffMsgRetryManager implements FailedMsgRetryManager {
+
+ private final long retryInitialDelayMs;
+ private final double retryDelayMultiplier;
+ private final long retryDelayMaxMs;
+
+ private Queue<MessageRetryRecord> waiting = new PriorityQueue<MessageRetryRecord>();
+ private SortedMap<Long,MessageRetryRecord> records = new TreeMap<Long,MessageRetryRecord>();
+
+ public ExponentialBackoffMsgRetryManager(long retryInitialDelayMs, double retryDelayMultiplier, long retryDelayMaxMs) {
+ this.retryInitialDelayMs = retryInitialDelayMs;
+ this.retryDelayMultiplier = retryDelayMultiplier;
+ this.retryDelayMaxMs = retryDelayMaxMs;
+ }
+
+ @Override
+ public void failed(Long offset) {
+ MessageRetryRecord oldRecord = this.records.get(offset);
+ MessageRetryRecord newRecord = oldRecord == null ?
+ new MessageRetryRecord(offset) :
+ oldRecord.createNextRetryRecord();
+ this.records.put(offset, newRecord);
+ this.waiting.add(newRecord);
+ }
+
+ @Override
+ public void acked(Long offset) {
+ MessageRetryRecord record = this.records.remove(offset);
+ if (record != null) {
+ this.waiting.remove(record);
+ }
+ }
+
+ @Override
+ public void retryStarted(Long offset) {
+ MessageRetryRecord record = this.records.get(offset);
+ if (record == null || !this.waiting.contains(record)) {
+ throw new IllegalStateException("cannot retry a message that has not failed");
+ } else {
+ this.waiting.remove(record);
+ }
+ }
+
+ @Override
+ public Long nextFailedMessageToRetry() {
+ if (this.waiting.size() > 0) {
+ MessageRetryRecord first = this.waiting.peek();
+ if (System.currentTimeMillis() >= first.retryTimeUTC) {
+ if (this.records.containsKey(first.offset)) {
+ return first.offset;
+ } else {
+ // defensive programming - should be impossible
+ this.waiting.remove(first);
+ return nextFailedMessageToRetry();
+ }
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public boolean shouldRetryMsg(Long offset) {
+ MessageRetryRecord record = this.records.get(offset);
+ return record != null &&
+ this.waiting.contains(record) &&
+ System.currentTimeMillis() >= record.retryTimeUTC;
+ }
+
+ /**
+ * A MessageRetryRecord holds the data of how many times a message has
+ * failed and been retried, and when the last failure occurred. It can
+ * determine whether it is ready to be retried by employing an exponential
+ * back-off calculation using config values stored in SpoutConfig:
+ * <ul>
+ * <li>retryInitialDelayMs - time to delay before the first retry</li>
+ * <li>retryDelayMultiplier - multiplier by which to increase the delay for each subsequent retry</li>
+ * <li>retryDelayMaxMs - maximum retry delay (once this delay time is reached, subsequent retries will
+ * delay for this amount of time every time)
+ * </li>
+ * </ul>
+ */
+ class MessageRetryRecord implements Comparable<MessageRetryRecord> {
+ private final long offset;
+ private final int retryNum;
+ private final long retryTimeUTC;
+
+ public MessageRetryRecord(long offset) {
+ this(offset, 1);
+ }
+
+ private MessageRetryRecord(long offset, int retryNum) {
+ this.offset = offset;
+ this.retryNum = retryNum;
+ this.retryTimeUTC = System.currentTimeMillis() + calculateRetryDelay();
+ }
+
+ /**
+ * Create a MessageRetryRecord for the next retry that should occur after this one.
+ * @return MessageRetryRecord with the next retry time, or null to indicate that another
+ * retry should not be performed. The latter case can happen if we are about to
+ * run into the backtype.storm.Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS in the Storm
+ * configuration.
+ */
+ public MessageRetryRecord createNextRetryRecord() {
+ return new MessageRetryRecord(this.offset, this.retryNum + 1);
+ }
+
+ private long calculateRetryDelay() {
+ double delayMultiplier = Math.pow(retryDelayMultiplier, this.retryNum - 1);
+ long delayThisRetryMs = (long) (retryInitialDelayMs * delayMultiplier);
+ return Math.min(delayThisRetryMs, retryDelayMaxMs);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return (other instanceof MessageRetryRecord
+ && this.offset == ((MessageRetryRecord) other).offset);
+ }
+
+ @Override
+ public int hashCode() {
+ return Long.valueOf(this.offset).hashCode();
+ }
+
+ @Override
+ public int compareTo(MessageRetryRecord other) {
+ return Long.compare(this.retryTimeUTC, other.retryTimeUTC);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/02bffc60/external/storm-kafka/src/jvm/storm/kafka/FailedMsgRetryManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/FailedMsgRetryManager.java b/external/storm-kafka/src/jvm/storm/kafka/FailedMsgRetryManager.java
new file mode 100644
index 0000000..3f0e117
--- /dev/null
+++ b/external/storm-kafka/src/jvm/storm/kafka/FailedMsgRetryManager.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package storm.kafka;
+
+public interface FailedMsgRetryManager {
+ public void failed(Long offset);
+ public void acked(Long offset);
+ public void retryStarted(Long offset);
+ public Long nextFailedMessageToRetry();
+ public boolean shouldRetryMsg(Long offset);
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/02bffc60/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
index cff6df0..17dd72d 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
@@ -23,7 +23,6 @@ import backtype.storm.metric.api.CountMetric;
import backtype.storm.metric.api.MeanReducer;
import backtype.storm.metric.api.ReducedMetric;
import backtype.storm.spout.SpoutOutputCollector;
-import backtype.storm.utils.Utils;
import com.google.common.collect.ImmutableMap;
import kafka.javaapi.consumer.SimpleConsumer;
import kafka.javaapi.message.ByteBufferMessageSet;
@@ -38,13 +37,6 @@ import java.util.*;
public class PartitionManager {
public static final Logger LOG = LoggerFactory.getLogger(PartitionManager.class);
- private static final String TIMES_UP_MSG =
- "Retry logic in your topology is taking longer to complete than is allowed by your"
- +" Storm Config setting TOPOLOGY_MESSAGE_TIMEOUT_SECS (%s seconds). (i.e., you have"
- +" called OutputCollector.fail() too many times for this message). KafkaSpout has"
- +" aborted next retry attempt (retry %s) for the Kafka message at offset %s since it"
- +" would occur after this timeout.";
- private static final long TIMEOUT_RESET_VALUE = -1L;
private final CombinedMetric _fetchAPILatencyMax;
private final ReducedMetric _fetchAPILatencyMean;
@@ -53,10 +45,9 @@ public class PartitionManager {
Long _emittedToOffset;
// _pending key = Kafka offset, value = time at which the message was first submitted to the topology
private SortedMap<Long,Long> _pending = new TreeMap<Long,Long>();
- private SortedSet<Long> failed = new TreeSet<Long>();
+ private final FailedMsgRetryManager _failedMsgRetryManager;
// retryRecords key = Kafka offset, value = retry info for the given message
- private Map<Long,MessageRetryRecord> retryRecords = new HashMap<Long,MessageRetryRecord>();
Long _committedTo;
LinkedList<MessageAndRealOffset> _waitingToEmit = new LinkedList<MessageAndRealOffset>();
Partition _partition;
@@ -77,6 +68,10 @@ public class PartitionManager {
_stormConf = stormConf;
numberAcked = numberFailed = 0;
+ _failedMsgRetryManager = new ExponentialBackoffMsgRetryManager(_spoutConfig.retryInitialDelayMs,
+ _spoutConfig.retryDelayMultiplier,
+ _spoutConfig.retryDelayMaxMs);
+
String jsonTopologyId = null;
Long jsonOffset = null;
String path = committedPath();
@@ -156,31 +151,15 @@ public class PartitionManager {
}
}
- /**
- * Fetch the failed messages ready for retry. If there are no failed messages, or none are ready for retry, then it
- * returns an empty List (i.e., not null).
- */
- private SortedSet<Long> failedMsgsReadyForRetry() {
- SortedSet<Long> ready = new TreeSet<Long>();
- for (Long offset : this.failed) {
- if (this.retryRecords.get(offset).isReadyForRetry()) {
- ready.add(offset);
- }
- }
- return ready;
- }
-
private void fill() {
long start = System.nanoTime();
- long offset;
- final SortedSet<Long> failedReady = failedMsgsReadyForRetry();
+ Long offset;
// Are there failed tuples? If so, fetch those first.
- final boolean had_failed = !failedReady.isEmpty();
- if (had_failed) {
- offset = failedReady.first();
- } else {
+ offset = this._failedMsgRetryManager.nextFailedMessageToRetry();
+ final boolean processingNewTuples = (offset == null);
+ if (processingNewTuples) {
offset = _emittedToOffset;
}
@@ -199,15 +178,15 @@ public class PartitionManager {
// Skip any old offsets.
continue;
}
- if (!had_failed || failedReady.contains(cur_offset)) {
+ if (processingNewTuples || this._failedMsgRetryManager.shouldRetryMsg(cur_offset)) {
numMessages += 1;
if (!_pending.containsKey(cur_offset)) {
_pending.put(cur_offset, System.currentTimeMillis());
}
_waitingToEmit.add(new MessageAndRealOffset(msg.message(), cur_offset));
_emittedToOffset = Math.max(msg.nextOffset(), _emittedToOffset);
- if (had_failed) {
- failed.remove(cur_offset);
+ if (_failedMsgRetryManager.shouldRetryMsg(cur_offset)) {
+ this._failedMsgRetryManager.retryStarted(cur_offset);
}
}
}
@@ -221,7 +200,7 @@ public class PartitionManager {
_pending.headMap(offset - _spoutConfig.maxOffsetBehind).clear();
}
_pending.remove(offset);
- retryRecords.remove(offset);
+ this._failedMsgRetryManager.acked(offset);
numberAcked++;
}
@@ -239,18 +218,7 @@ public class PartitionManager {
throw new RuntimeException("Too many tuple failures");
}
- try {
- MessageRetryRecord retryRecord = retryRecords.get(offset);
- retryRecord = retryRecord == null
- ? new MessageRetryRecord(offset)
- : retryRecord.createNextRetryRecord();
-
- retryRecords.put(offset, retryRecord);
- failed.add(offset);
-
- } catch (MessageRetryRecord.AvailableRetryTimeExceededException e) {
- LOG.error("cannot retry", e);
- }
+ this._failedMsgRetryManager.failed(offset);
}
}
@@ -304,90 +272,4 @@ public class PartitionManager {
this.offset = offset;
}
}
-
- /**
- * A MessageRetryRecord holds the data of how many times a message has
- * failed and been retried, and when the last failure occurred. It can
- * determine whether it is ready to be retried by employing an exponential
- * back-off calculation using config values stored in SpoutConfig:
- * <ul>
- * <li>retryInitialDelayMs - time to delay before the first retry</li>
- * <li>retryDelayMultiplier - multiplier by which to increase the delay for each subsequent retry</li>
- * <li>retryDelayMaxMs - maximum retry delay (once this delay time is reached, subsequent retries will
- * delay for this amount of time every time)
- * </li>
- * </ul>
- */
- class MessageRetryRecord {
- private final long offset;
- private final int retryNum;
- private final long retryTimeUTC;
-
- public MessageRetryRecord(long offset) throws AvailableRetryTimeExceededException {
- this(offset, 1);
- }
-
- private MessageRetryRecord(long offset, int retryNum) throws AvailableRetryTimeExceededException {
- this.offset = offset;
- this.retryNum = retryNum;
- this.retryTimeUTC = System.currentTimeMillis() + calculateRetryDelay();
- validateRetryTime();
- }
-
- /**
- * Create a MessageRetryRecord for the next retry that should occur after this one.
- * @return MessageRetryRecord with the next retry time, or null to indicate that another
- * retry should not be performed. The latter case can happen if we are about to
- * run into the backtype.storm.Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS in the Storm
- * configuration.
- */
- public MessageRetryRecord createNextRetryRecord() throws AvailableRetryTimeExceededException {
- return new MessageRetryRecord(this.offset, this.retryNum + 1);
- }
-
- private void validateRetryTime() throws AvailableRetryTimeExceededException {
- long stormStartTime = PartitionManager.this._pending.get(this.offset);
-
- if (stormStartTime == TIMEOUT_RESET_VALUE) {
- // This is a resubmission from the Storm framework after Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS
- // has elapsed. Restart my timer.
- PartitionManager.this._pending.put(this.offset, System.currentTimeMillis());
-
- } else {
- int timeoutSeconds = Utils.getInt(_stormConf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS));
- if (this.retryTimeUTC - stormStartTime > timeoutSeconds * 1000) {
-
- // Prepare for when the Storm framework calls fail()
- _pending.put(this.offset, TIMEOUT_RESET_VALUE);
-
- throw new AvailableRetryTimeExceededException(String.format(TIMES_UP_MSG,
- timeoutSeconds,
- this.retryNum,
- this.offset));
-
- } else {
- LOG.warn(String.format("allowing another retry: start=%s, retryTime=%s, timeoutSeconds=%s",
- (stormStartTime / 1000) % 1000,
- (this.retryTimeUTC / 1000) % 1000,
- timeoutSeconds));
- }
- }
- }
-
- private long calculateRetryDelay() {
- double delayMultiplier = Math.pow(_spoutConfig.retryDelayMultiplier, this.retryNum - 1);
- long delayThisRetryMs = (long) (_spoutConfig.retryInitialDelayMs * delayMultiplier);
- return Math.min(delayThisRetryMs, _spoutConfig.retryDelayMaxMs);
- }
-
- public boolean isReadyForRetry() {
- return System.currentTimeMillis() > this.retryTimeUTC;
- }
-
- class AvailableRetryTimeExceededException extends Exception {
- public AvailableRetryTimeExceededException(String msg) {
- super(msg);
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/02bffc60/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java b/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
index 1c40ddf..61d0b35 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
@@ -32,9 +32,6 @@ public class SpoutConfig extends KafkaConfig implements Serializable {
// Exponential back-off retry settings. These are used when retrying messages after a bolt
// calls OutputCollector.fail().
- //
- // Note: be sure to set backtype.storm.Config.MESSAGE_TIMEOUT_SECS appropriately to prevent
- // resubmitting the message while still retrying.
public long retryInitialDelayMs = 0;
public double retryDelayMultiplier = 1.0;
public long retryDelayMaxMs = 60 * 1000;
http://git-wip-us.apache.org/repos/asf/storm/blob/02bffc60/external/storm-kafka/src/test/storm/kafka/ExponentialBackoffMsgRetryManagerTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/test/storm/kafka/ExponentialBackoffMsgRetryManagerTest.java b/external/storm-kafka/src/test/storm/kafka/ExponentialBackoffMsgRetryManagerTest.java
new file mode 100644
index 0000000..ef30163
--- /dev/null
+++ b/external/storm-kafka/src/test/storm/kafka/ExponentialBackoffMsgRetryManagerTest.java
@@ -0,0 +1,194 @@
+package storm.kafka;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.junit.Test;
+
+public class ExponentialBackoffMsgRetryManagerTest {
+
+ private static final Long TEST_OFFSET = 101L;
+ private static final Long TEST_OFFSET2 = 102L;
+
+ @Test
+ public void testImmediateRetry() throws Exception {
+ ExponentialBackoffMsgRetryManager manager = new ExponentialBackoffMsgRetryManager(0, 0d, 0);
+ manager.failed(TEST_OFFSET);
+ Long next = manager.nextFailedMessageToRetry();
+ assertEquals("expect test offset next available for retry", TEST_OFFSET, next);
+ assertTrue("message should be ready for retry immediately", manager.shouldRetryMsg(TEST_OFFSET));
+
+ manager.retryStarted(TEST_OFFSET);
+
+ manager.failed(TEST_OFFSET);
+ next = manager.nextFailedMessageToRetry();
+ assertEquals("expect test offset next available for retry", TEST_OFFSET, next);
+ assertTrue("message should be ready for retry immediately", manager.shouldRetryMsg(TEST_OFFSET));
+ }
+
+ @Test
+ public void testSingleDelay() throws Exception {
+ ExponentialBackoffMsgRetryManager manager = new ExponentialBackoffMsgRetryManager(10, 1d, 100);
+ manager.failed(TEST_OFFSET);
+ Thread.sleep(5);
+ Long next = manager.nextFailedMessageToRetry();
+ assertNull("expect no message ready for retry yet", next);
+ assertFalse("message should not be ready for retry yet", manager.shouldRetryMsg(TEST_OFFSET));
+
+ Thread.sleep(10);
+ next = manager.nextFailedMessageToRetry();
+ assertEquals("expect test offset next available for retry", TEST_OFFSET, next);
+ assertTrue("message should be ready for retry", manager.shouldRetryMsg(TEST_OFFSET));
+ }
+
+ @Test
+ public void testExponentialBackoff() throws Exception {
+ final long initial = 10;
+ final double mult = 2d;
+ ExponentialBackoffMsgRetryManager manager = new ExponentialBackoffMsgRetryManager(initial, mult, initial * 10);
+
+ long expectedWaitTime = initial;
+ for (long i = 0L; i < 3L; ++i) {
+ manager.failed(TEST_OFFSET);
+
+ Thread.sleep((expectedWaitTime + 1L) / 2L);
+ assertFalse("message should not be ready for retry yet", manager.shouldRetryMsg(TEST_OFFSET));
+
+ Thread.sleep((expectedWaitTime + 1L) / 2L);
+ Long next = manager.nextFailedMessageToRetry();
+ assertEquals("expect test offset next available for retry", TEST_OFFSET, next);
+ assertTrue("message should be ready for retry", manager.shouldRetryMsg(TEST_OFFSET));
+
+ manager.retryStarted(TEST_OFFSET);
+ expectedWaitTime *= mult;
+ }
+ }
+
+ @Test
+ public void testRetryOrder() throws Exception {
+ final long initial = 10;
+ final double mult = 2d;
+ final long max = 20;
+ ExponentialBackoffMsgRetryManager manager = new ExponentialBackoffMsgRetryManager(initial, mult, max);
+
+ manager.failed(TEST_OFFSET);
+ Thread.sleep(initial);
+
+ manager.retryStarted(TEST_OFFSET);
+ manager.failed(TEST_OFFSET);
+ manager.failed(TEST_OFFSET2);
+
+ // although TEST_OFFSET failed first, it's retry delay time is longer b/c this is the second retry
+ // so TEST_OFFSET2 should come first
+
+ Thread.sleep(initial * 2);
+ assertTrue("message "+TEST_OFFSET+"should be ready for retry", manager.shouldRetryMsg(TEST_OFFSET));
+ assertTrue("message "+TEST_OFFSET2+"should be ready for retry", manager.shouldRetryMsg(TEST_OFFSET2));
+
+ Long next = manager.nextFailedMessageToRetry();
+ assertEquals("expect first message to retry is "+TEST_OFFSET2, TEST_OFFSET2, next);
+
+ Thread.sleep(initial);
+
+ // haven't retried yet, so first should still be TEST_OFFSET2
+ next = manager.nextFailedMessageToRetry();
+ assertEquals("expect first message to retry is "+TEST_OFFSET2, TEST_OFFSET2, next);
+ manager.retryStarted(next);
+
+ // now it should be TEST_OFFSET
+ next = manager.nextFailedMessageToRetry();
+ assertEquals("expect message to retry is now "+TEST_OFFSET, TEST_OFFSET, next);
+ manager.retryStarted(next);
+
+ // now none left
+ next = manager.nextFailedMessageToRetry();
+ assertNull("expect no message to retry now", next);
+ }
+
+ @Test
+ public void testQueriesAfterRetriedAlready() throws Exception {
+ ExponentialBackoffMsgRetryManager manager = new ExponentialBackoffMsgRetryManager(0, 0d, 0);
+ manager.failed(TEST_OFFSET);
+ Long next = manager.nextFailedMessageToRetry();
+ assertEquals("expect test offset next available for retry", TEST_OFFSET, next);
+ assertTrue("message should be ready for retry immediately", manager.shouldRetryMsg(TEST_OFFSET));
+
+ manager.retryStarted(TEST_OFFSET);
+ next = manager.nextFailedMessageToRetry();
+ assertNull("expect no message ready after retried", next);
+ assertFalse("message should not be ready after retried", manager.shouldRetryMsg(TEST_OFFSET));
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testRetryWithoutFail() throws Exception {
+ ExponentialBackoffMsgRetryManager manager = new ExponentialBackoffMsgRetryManager(0, 0d, 0);
+ manager.retryStarted(TEST_OFFSET);
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testFailRetryRetry() throws Exception {
+ ExponentialBackoffMsgRetryManager manager = new ExponentialBackoffMsgRetryManager(0, 0d, 0);
+ manager.failed(TEST_OFFSET);
+ try {
+ manager.retryStarted(TEST_OFFSET);
+ } catch (IllegalStateException ise) {
+ fail("IllegalStateException unexpected here: " + ise);
+ }
+
+ assertFalse("message should not be ready for retry", manager.shouldRetryMsg(TEST_OFFSET));
+ manager.retryStarted(TEST_OFFSET);
+ }
+
+ @Test
+ public void testMaxBackoff() throws Exception {
+ final long initial = 10;
+ final double mult = 2d;
+ final long max = 20;
+ ExponentialBackoffMsgRetryManager manager = new ExponentialBackoffMsgRetryManager(initial, mult, max);
+
+ long expectedWaitTime = initial;
+ for (long i = 0L; i < 4L; ++i) {
+ manager.failed(TEST_OFFSET);
+
+ Thread.sleep((expectedWaitTime + 1L) / 2L);
+ assertFalse("message should not be ready for retry yet", manager.shouldRetryMsg(TEST_OFFSET));
+
+ Thread.sleep((expectedWaitTime + 1L) / 2L);
+ Long next = manager.nextFailedMessageToRetry();
+ assertEquals("expect test offset next available for retry", TEST_OFFSET, next);
+ assertTrue("message should be ready for retry", manager.shouldRetryMsg(TEST_OFFSET));
+
+ manager.retryStarted(TEST_OFFSET);
+ expectedWaitTime = Math.min((long) (expectedWaitTime * mult), max);
+ }
+ }
+
+ @Test
+ public void testFailThenAck() throws Exception {
+ ExponentialBackoffMsgRetryManager manager = new ExponentialBackoffMsgRetryManager(0, 0d, 0);
+ manager.failed(TEST_OFFSET);
+ assertTrue("message should be ready for retry", manager.shouldRetryMsg(TEST_OFFSET));
+
+ manager.acked(TEST_OFFSET);
+
+ Long next = manager.nextFailedMessageToRetry();
+ assertNull("expect no message ready after acked", next);
+ assertFalse("message should not be ready after acked", manager.shouldRetryMsg(TEST_OFFSET));
+ }
+
+ @Test
+ public void testAckThenFail() throws Exception {
+ ExponentialBackoffMsgRetryManager manager = new ExponentialBackoffMsgRetryManager(0, 0d, 0);
+ manager.acked(TEST_OFFSET);
+ assertFalse("message should not be ready after acked", manager.shouldRetryMsg(TEST_OFFSET));
+
+ manager.failed(TEST_OFFSET);
+
+ Long next = manager.nextFailedMessageToRetry();
+ assertEquals("expect test offset next available for retry", TEST_OFFSET, next);
+ assertTrue("message should be ready for retry", manager.shouldRetryMsg(TEST_OFFSET));
+ }
+}
\ No newline at end of file