You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2014/05/14 17:40:00 UTC
[1/8] git commit: Change Twitter authentication
Repository: incubator-storm
Updated Branches:
refs/heads/master ac9038407 -> 79599237f
Change Twitter authentication
Change Twitter authentication stream using the new Twitter4J. An optional set of filters can be used to restricted tweets
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/75800059
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/75800059
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/75800059
Branch: refs/heads/master
Commit: 75800059a2d98a948939a81d47997590cb61ae00
Parents: b30de82
Author: Prabeesh K <pr...@gmail.com>
Authored: Sat Apr 5 10:02:36 2014 +0530
Committer: Prabeesh K <pr...@gmail.com>
Committed: Sat Apr 5 10:02:36 2014 +0530
----------------------------------------------------------------------
.../storm/starter/spout/TwitterSampleSpout.java | 44 +++++++++++++++-----
1 file changed, 33 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/75800059/examples/storm-starter/src/jvm/storm/starter/spout/TwitterSampleSpout.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/storm/starter/spout/TwitterSampleSpout.java b/examples/storm-starter/src/jvm/storm/starter/spout/TwitterSampleSpout.java
index 5cc12de..972b124 100644
--- a/examples/storm-starter/src/jvm/storm/starter/spout/TwitterSampleSpout.java
+++ b/examples/storm-starter/src/jvm/storm/starter/spout/TwitterSampleSpout.java
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/*
package storm.starter.spout;
@@ -35,18 +34,29 @@ import java.util.concurrent.LinkedBlockingQueue;
import twitter4j.Status;
import twitter4j.StatusDeletionNotice;
import twitter4j.StatusListener;
+import twitter4j.FilterQuery;
+import twitter4j.StallWarning;
+import twitter4j.auth.AccessToken;
public class TwitterSampleSpout extends BaseRichSpout {
SpoutOutputCollector _collector;
LinkedBlockingQueue<Status> queue = null;
TwitterStream _twitterStream;
- String _username;
- String _pwd;
+ String consumerKey;
+ String consumerSecret;
+ String accessToken;
+ String accessTokenSecret;
+ String[] keyWords;
- public TwitterSampleSpout(String username, String pwd) {
- _username = username;
- _pwd = pwd;
+ public TwitterSampleSpout(String consumerKey, String consumerSecret,
+ String accessToken, String accessTokenSecret, String[] keyWords) {
+
+ this.consumerKey = consumerKey;
+ this.consumerSecret = consumerSecret;
+ this.accessToken = accessToken;
+ this.accessTokenSecret = accessTokenSecret;
+ this.keyWords = keyWords;
}
@Override
@@ -77,10 +87,23 @@ public class TwitterSampleSpout extends BaseRichSpout {
}
};
- TwitterStreamFactory fact = new TwitterStreamFactory(new ConfigurationBuilder().setUser(_username).setPassword(_pwd).build());
- _twitterStream = fact.getInstance();
- _twitterStream.addListener(listener);
- _twitterStream.sample();
+
+ TwitterStream twitterStream = new TwitterStreamFactory(
+ new ConfigurationBuilder().setJSONStoreEnabled(true).build())
+ .getInstance();
+
+ twitterStream.addListener(listener);
+ twitterStream.setOAuthConsumer(consumerKey, consumerSecret);
+ AccessToken token = new AccessToken(accessToken, accessTokenSecret);
+ twitterStream.setOAuthAccessToken(token);
+
+ if (keyWords.length == 0) {
+ twitterStream.sample();
+ }
+ else {
+ FilterQuery query = new FilterQuery().track(keyWords);
+ twitterStream.filter(query);
+ }
}
@Override
@@ -119,4 +142,3 @@ public class TwitterSampleSpout extends BaseRichSpout {
}
}
-*/
\ No newline at end of file
[4/8] git commit: Update TwitterSampleSpout.java
Posted by bo...@apache.org.
Update TwitterSampleSpout.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/426468cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/426468cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/426468cc
Branch: refs/heads/master
Commit: 426468cc155ab1b80b44028812c7b3fc23bf2776
Parents: 8785f49
Author: Prabeesh K <pr...@gmail.com>
Authored: Thu May 8 10:01:43 2014 +0530
Committer: Prabeesh K <pr...@gmail.com>
Committed: Thu May 8 10:01:43 2014 +0530
----------------------------------------------------------------------
.../storm/starter/spout/TwitterSampleSpout.java | 240 ++++++++++---------
1 file changed, 130 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/426468cc/examples/storm-starter/src/jvm/storm/starter/spout/TwitterSampleSpout.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/storm/starter/spout/TwitterSampleSpout.java b/examples/storm-starter/src/jvm/storm/starter/spout/TwitterSampleSpout.java
index 972b124..40f8d72 100644
--- a/examples/storm-starter/src/jvm/storm/starter/spout/TwitterSampleSpout.java
+++ b/examples/storm-starter/src/jvm/storm/starter/spout/TwitterSampleSpout.java
@@ -18,10 +18,20 @@
package storm.starter.spout;
-import backtype.storm.Config;
-import twitter4j.conf.ConfigurationBuilder;
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import twitter4j.FilterQuery;
+import twitter4j.StallWarning;
+import twitter4j.Status;
+import twitter4j.StatusDeletionNotice;
+import twitter4j.StatusListener;
import twitter4j.TwitterStream;
import twitter4j.TwitterStreamFactory;
+import twitter4j.auth.AccessToken;
+import twitter4j.conf.ConfigurationBuilder;
+
+import backtype.storm.Config;
import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
@@ -29,116 +39,126 @@ import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
import backtype.storm.utils.Utils;
-import java.util.Map;
-import java.util.concurrent.LinkedBlockingQueue;
-import twitter4j.Status;
-import twitter4j.StatusDeletionNotice;
-import twitter4j.StatusListener;
-import twitter4j.FilterQuery;
-import twitter4j.StallWarning;
-import twitter4j.auth.AccessToken;
+@SuppressWarnings("serial")
public class TwitterSampleSpout extends BaseRichSpout {
- SpoutOutputCollector _collector;
- LinkedBlockingQueue<Status> queue = null;
- TwitterStream _twitterStream;
- String consumerKey;
- String consumerSecret;
- String accessToken;
- String accessTokenSecret;
- String[] keyWords;
-
-
- public TwitterSampleSpout(String consumerKey, String consumerSecret,
- String accessToken, String accessTokenSecret, String[] keyWords) {
-
- this.consumerKey = consumerKey;
- this.consumerSecret = consumerSecret;
- this.accessToken = accessToken;
- this.accessTokenSecret = accessTokenSecret;
- this.keyWords = keyWords;
- }
-
- @Override
- public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
- queue = new LinkedBlockingQueue<Status>(1000);
- _collector = collector;
- StatusListener listener = new StatusListener() {
-
- @Override
- public void onStatus(Status status) {
- queue.offer(status);
- }
-
- @Override
- public void onDeletionNotice(StatusDeletionNotice sdn) {
- }
-
- @Override
- public void onTrackLimitationNotice(int i) {
- }
-
- @Override
- public void onScrubGeo(long l, long l1) {
- }
-
- @Override
- public void onException(Exception e) {
- }
-
- };
-
- TwitterStream twitterStream = new TwitterStreamFactory(
+
+ SpoutOutputCollector _collector;
+ LinkedBlockingQueue<Status> queue = null;
+ TwitterStream _twitterStream;
+ String consumerKey;
+ String consumerSecret;
+ String accessToken;
+ String accessTokenSecret;
+ String[] keyWords;
+
+ public TwitterSampleSpout(String consumerKey, String consumerSecret,
+ String accessToken, String accessTokenSecret, String[] keyWords) {
+ this.consumerKey = consumerKey;
+ this.consumerSecret = consumerSecret;
+ this.accessToken = accessToken;
+ this.accessTokenSecret = accessTokenSecret;
+ this.keyWords = keyWords;
+ }
+
+ public TwitterSampleSpout() {
+ // TODO Auto-generated constructor stub
+ }
+
+ @Override
+ public void open(Map conf, TopologyContext context,
+ SpoutOutputCollector collector) {
+ queue = new LinkedBlockingQueue<Status>(1000);
+ _collector = collector;
+
+ StatusListener listener = new StatusListener() {
+
+ @Override
+ public void onStatus(Status status) {
+
+ queue.offer(status);
+ }
+
+ @Override
+ public void onDeletionNotice(StatusDeletionNotice sdn) {
+ }
+
+ @Override
+ public void onTrackLimitationNotice(int i) {
+ }
+
+ @Override
+ public void onScrubGeo(long l, long l1) {
+ }
+
+ @Override
+ public void onException(Exception ex) {
+ }
+
+ @Override
+ public void onStallWarning(StallWarning arg0) {
+ // TODO Auto-generated method stub
+
+ }
+
+ };
+
+ TwitterStream twitterStream = new TwitterStreamFactory(
new ConfigurationBuilder().setJSONStoreEnabled(true).build())
.getInstance();
-
- twitterStream.addListener(listener);
- twitterStream.setOAuthConsumer(consumerKey, consumerSecret);
- AccessToken token = new AccessToken(accessToken, accessTokenSecret);
- twitterStream.setOAuthAccessToken(token);
-
- if (keyWords.length == 0) {
- twitterStream.sample();
- }
- else {
- FilterQuery query = new FilterQuery().track(keyWords);
- twitterStream.filter(query);
- }
- }
-
- @Override
- public void nextTuple() {
- Status ret = queue.poll();
- if(ret==null) {
- Utils.sleep(50);
- } else {
- _collector.emit(new Values(ret));
- }
- }
-
- @Override
- public void close() {
- _twitterStream.shutdown();
- }
-
- @Override
- public Map<String, Object> getComponentConfiguration() {
- Config ret = new Config();
- ret.setMaxTaskParallelism(1);
- return ret;
- }
-
- @Override
- public void ack(Object id) {
- }
-
- @Override
- public void fail(Object id) {
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("tweet"));
- }
-
+
+ twitterStream.addListener(listener);
+ twitterStream.setOAuthConsumer(consumerKey, consumerSecret);
+ AccessToken token = new AccessToken(accessToken, accessTokenSecret);
+ twitterStream.setOAuthAccessToken(token);
+
+ if (keyWords.length == 0) {
+
+ twitterStream.sample();
+ }
+
+ else {
+
+ FilterQuery query = new FilterQuery().track(keyWords);
+ twitterStream.filter(query);
+ }
+
+ }
+
+ @Override
+ public void nextTuple() {
+ Status ret = queue.poll();
+ if (ret == null) {
+ Utils.sleep(50);
+ } else {
+ _collector.emit(new Values(ret));
+
+ }
+ }
+
+ @Override
+ public void close() {
+ _twitterStream.shutdown();
+ }
+
+ @Override
+ public Map<String, Object> getComponentConfiguration() {
+ Config ret = new Config();
+ ret.setMaxTaskParallelism(1);
+ return ret;
+ }
+
+ @Override
+ public void ack(Object id) {
+ }
+
+ @Override
+ public void fail(Object id) {
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("tweet"));
+ }
+
}
[8/8] git commit: Added STORM-310 to changelog
Posted by bo...@apache.org.
Added STORM-310 to changelog
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/79599237
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/79599237
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/79599237
Branch: refs/heads/master
Commit: 79599237f6b25e86f4723bbd86a78bf61e7495ff
Parents: f041e55
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Wed May 14 15:37:48 2014 +0000
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Wed May 14 15:37:48 2014 +0000
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/79599237/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 81bc834..2ed29cb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 0.9.2-incubating (unreleased)
+ * STORM-310: Change Twitter authentication
* STORM-305: Create developer documentation
* STORM-280: storm unit tests are failing on windows
* STORM-298: Logback file does not include full path for metrics appender fileNamePattern
[7/8] git commit: Merge branch 'master' of
https://github.com/prabeesh/incubator-storm into STORM-310
Posted by bo...@apache.org.
Merge branch 'master' of https://github.com/prabeesh/incubator-storm into STORM-310
STORM-310: Change Twitter authentication
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/f041e55f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/f041e55f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/f041e55f
Branch: refs/heads/master
Commit: f041e55fc7f8fcaf92d5b7d23c80dee2025c2921
Parents: ac90384 f0814d4
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Wed May 14 15:37:06 2014 +0000
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Wed May 14 15:37:06 2014 +0000
----------------------------------------------------------------------
examples/storm-starter/pom.xml | 5 +
.../jvm/storm/starter/PrintSampleStream.java | 24 +-
.../storm/starter/spout/TwitterSampleSpout.java | 222 +++++++++++--------
3 files changed, 151 insertions(+), 100 deletions(-)
----------------------------------------------------------------------
[6/8] git commit: Removed unnecessary comments
Posted by bo...@apache.org.
Removed unnecessary comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/f0814d4b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/f0814d4b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/f0814d4b
Branch: refs/heads/master
Commit: f0814d4b46af16ba4bb98458ba4f1d5339c7643a
Parents: aed4e12
Author: Prabeesh K <pr...@gmail.com>
Authored: Tue May 13 08:50:17 2014 +0530
Committer: Prabeesh K <pr...@gmail.com>
Committed: Tue May 13 08:50:17 2014 +0530
----------------------------------------------------------------------
.../storm-starter/src/jvm/storm/starter/PrintSampleStream.java | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/f0814d4b/examples/storm-starter/src/jvm/storm/starter/PrintSampleStream.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/storm/starter/PrintSampleStream.java b/examples/storm-starter/src/jvm/storm/starter/PrintSampleStream.java
index 81c1dee..2734fff 100644
--- a/examples/storm-starter/src/jvm/storm/starter/PrintSampleStream.java
+++ b/examples/storm-starter/src/jvm/storm/starter/PrintSampleStream.java
@@ -16,9 +16,6 @@
* limitations under the License.
*/
-// to use this example, uncomment the twitter4j dependency information in the project.clj,
-// uncomment storm.starter.spout.TwitterSampleSpout, and uncomment this class
-
package storm.starter;
import java.util.Arrays;
[5/8] git commit: Added twitter4j depedency
Posted by bo...@apache.org.
Added twitter4j depedency
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/aed4e124
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/aed4e124
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/aed4e124
Branch: refs/heads/master
Commit: aed4e12431b7437da8d181810c75ea78013c31a1
Parents: 426468c
Author: Prabeesh K <pr...@gmail.com>
Authored: Thu May 8 10:03:56 2014 +0530
Committer: Prabeesh K <pr...@gmail.com>
Committed: Thu May 8 10:03:56 2014 +0530
----------------------------------------------------------------------
examples/storm-starter/pom.xml | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/aed4e124/examples/storm-starter/pom.xml
----------------------------------------------------------------------
diff --git a/examples/storm-starter/pom.xml b/examples/storm-starter/pom.xml
index 3030a8f..11146b6 100644
--- a/examples/storm-starter/pom.xml
+++ b/examples/storm-starter/pom.xml
@@ -62,6 +62,11 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.twitter4j</groupId>
+ <artifactId>twitter4j-stream</artifactId>
+ <version>3.0.3</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>${project.version}</version>
[2/8] git commit: Update PrintSampleStream.java
Posted by bo...@apache.org.
Update PrintSampleStream.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/aa064727
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/aa064727
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/aa064727
Branch: refs/heads/master
Commit: aa064727d8b13d762c06509c185194b17edc40e7
Parents: 7580005
Author: Prabeesh K <pr...@gmail.com>
Authored: Sat Apr 5 10:04:37 2014 +0530
Committer: Prabeesh K <pr...@gmail.com>
Committed: Sat Apr 5 10:04:37 2014 +0530
----------------------------------------------------------------------
.../src/jvm/storm/starter/PrintSampleStream.java | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/aa064727/examples/storm-starter/src/jvm/storm/starter/PrintSampleStream.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/storm/starter/PrintSampleStream.java b/examples/storm-starter/src/jvm/storm/starter/PrintSampleStream.java
index 8bead91..de70850 100644
--- a/examples/storm-starter/src/jvm/storm/starter/PrintSampleStream.java
+++ b/examples/storm-starter/src/jvm/storm/starter/PrintSampleStream.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/*
+
// to use this example, uncomment the twitter4j dependency information in the project.clj,
// uncomment storm.starter.spout.TwitterSampleSpout, and uncomment this class
@@ -31,15 +31,20 @@ import storm.starter.bolt.PrinterBolt;
public class PrintSampleStream {
public static void main(String[] args) {
- String username = args[0];
- String pwd = args[1];
+ String consumerKey = args[0];
+ String consumerSecret = args[1];
+ String accessToken = args[2];
+ String accessTokenSecret = args[3];
+ String[] keyWords = Arrays.copyOfRange(arguments, 4, arguments.length);
+
TopologyBuilder builder = new TopologyBuilder();
- builder.setSpout("spout", new TwitterSampleSpout(username, pwd));
+ builder.setSpout("spoutId", new TwitterSampleSpout(consumerKey, consumerSecret,
+ accessToken, accessTokenSecret, keyWords));
builder.setBolt("print", new PrinterBolt())
.shuffleGrouping("spout");
-
+
Config conf = new Config();
@@ -51,4 +56,3 @@ public class PrintSampleStream {
cluster.shutdown();
}
}
-*/
\ No newline at end of file
[3/8] git commit: Update PrintSampleStream.java
Posted by bo...@apache.org.
Update PrintSampleStream.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/8785f491
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/8785f491
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/8785f491
Branch: refs/heads/master
Commit: 8785f491bab0c3a43d472e1e5250415aee6e287b
Parents: aa06472
Author: Prabeesh K <pr...@gmail.com>
Authored: Thu May 8 10:00:40 2014 +0530
Committer: Prabeesh K <pr...@gmail.com>
Committed: Thu May 8 10:00:40 2014 +0530
----------------------------------------------------------------------
.../src/jvm/storm/starter/PrintSampleStream.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/8785f491/examples/storm-starter/src/jvm/storm/starter/PrintSampleStream.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/storm/starter/PrintSampleStream.java b/examples/storm-starter/src/jvm/storm/starter/PrintSampleStream.java
index de70850..81c1dee 100644
--- a/examples/storm-starter/src/jvm/storm/starter/PrintSampleStream.java
+++ b/examples/storm-starter/src/jvm/storm/starter/PrintSampleStream.java
@@ -21,13 +21,15 @@
package storm.starter;
-import storm.starter.spout.TwitterSampleSpout;
+import java.util.Arrays;
+
import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.topology.TopologyBuilder;
import backtype.storm.utils.Utils;
-import storm.starter.bolt.PrinterBolt;
+import storm.starter.bolt.PrinterBolt;
+import storm.starter.spout.TwitterSampleSpout;
public class PrintSampleStream {
public static void main(String[] args) {
@@ -35,6 +37,7 @@ public class PrintSampleStream {
String consumerSecret = args[1];
String accessToken = args[2];
String accessTokenSecret = args[3];
+ String[] arguments = args.clone();
String[] keyWords = Arrays.copyOfRange(arguments, 4, arguments.length);
TopologyBuilder builder = new TopologyBuilder();