You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by ni...@apache.org on 2013/02/01 08:42:13 UTC
svn commit: r1441326 - in /camel/trunk/components/camel-twitter/src:
main/java/org/apache/camel/component/twitter/
main/java/org/apache/camel/component/twitter/consumer/search/
main/java/org/apache/camel/component/twitter/producer/
test/java/org/apache...
Author: ningjiang
Date: Fri Feb 1 07:42:13 2013
New Revision: 1441326
URL: http://svn.apache.org/viewvc?rev=1441326&view=rev
Log:
CAMEL-6021,CAMEL-6022 added the option of count and numberOfPages with thanks to Luca
Modified:
camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConfiguration.java
camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConstants.java
camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/consumer/search/SearchConsumer.java
camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/producer/SearchProducer.java
camel/trunk/components/camel-twitter/src/test/java/org/apache/camel/component/twitter/UriConfigurationTest.java
Modified: camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConfiguration.java
URL: http://svn.apache.org/viewvc/camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConfiguration.java?rev=1441326&r1=1441325&r2=1441326&view=diff
==============================================================================
--- camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConfiguration.java (original)
+++ camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConfiguration.java Fri Feb 1 07:42:13 2013
@@ -86,6 +86,11 @@ public class TwitterConfiguration {
* Used ot set the preferred language on which to search
*/
private String lang;
+
+ /**
+ * Used to set the maximum tweets per page (max = 100)
+ */
+ private Integer count;
private Date parsedDate;
private SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
@@ -101,6 +106,11 @@ public class TwitterConfiguration {
private TwitterStream twitterStream;
/**
+ * Number of page to iterate before stop (default is 1)
+ */
+ private Integer numberOfPages = new Integer(1);
+
+ /**
* Ensures required fields are available.
*/
public void checkComplete() {
@@ -271,6 +281,22 @@ public class TwitterConfiguration {
public void setLang(String lang) {
this.lang = lang;
}
+
+ public Integer getCount() {
+ return count;
+ }
+
+ public void setCount(int count) {
+ this.count = count;
+ }
+
+ public Integer getNumberOfPages() {
+ return numberOfPages;
+ }
+
+ public void setNumberOfPages(Integer numberOfPages) {
+ this.numberOfPages = numberOfPages;
+ }
}
Modified: camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConstants.java
URL: http://svn.apache.org/viewvc/camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConstants.java?rev=1441326&r1=1441325&r2=1441326&view=diff
==============================================================================
--- camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConstants.java (original)
+++ camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConstants.java Fri Feb 1 07:42:13 2013
@@ -24,6 +24,8 @@ public final class TwitterConstants {
public static final String TWITTER_KEYWORDS = "CamelTwitterKeywords";
public static final String TWITTER_SEARCH_LANGUAGE = "CamelTwitterSearchLanguage";
+ public static final String TWITTER_COUNT = "CamelTwitterCount";
+ public static final String TWITTER_NUMBER_OF_PAGES = "CamelTwitterNumberOfPages";
private TwitterConstants() {
// utility
Modified: camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/consumer/search/SearchConsumer.java
URL: http://svn.apache.org/viewvc/camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/consumer/search/SearchConsumer.java?rev=1441326&r1=1441325&r2=1441326&view=diff
==============================================================================
--- camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/consumer/search/SearchConsumer.java (original)
+++ camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/consumer/search/SearchConsumer.java Fri Feb 1 07:42:13 2013
@@ -27,6 +27,7 @@ import org.slf4j.LoggerFactory;
import twitter4j.Query;
import twitter4j.QueryResult;
import twitter4j.Status;
+import twitter4j.Twitter;
import twitter4j.TwitterException;
/**
@@ -41,35 +42,59 @@ public class SearchConsumer extends Twit
}
public List<Status> pollConsume() throws TwitterException {
+
String keywords = te.getProperties().getKeywords();
Query query = new Query(keywords);
if (te.getProperties().isFilterOld()) {
query.setSinceId(lastId);
}
- if (ObjectHelper.isNotEmpty(te.getProperties().getLang())) {
- query.setLang(te.getProperties().getLang());
- }
+
LOG.debug("Searching twitter with keywords: {}", keywords);
return search(query);
}
public List<Status> directConsume() throws TwitterException {
+
String keywords = te.getProperties().getKeywords();
if (keywords == null || keywords.trim().length() == 0) {
return Collections.emptyList();
}
Query query = new Query(keywords);
- if (ObjectHelper.isNotEmpty(te.getProperties().getLang())) {
- query.setLang(te.getProperties().getLang());
- }
+
LOG.debug("Searching twitter with keywords: {}", keywords);
return search(query);
}
private List<Status> search(Query query) throws TwitterException {
- QueryResult qr = te.getProperties().getTwitter().search(query);
+ Integer numberOfPages = 1;
+
+ if (ObjectHelper.isNotEmpty(te.getProperties().getLang())) {
+ query.setLang(te.getProperties().getLang());
+ }
+
+ if (ObjectHelper.isNotEmpty(te.getProperties().getCount())) {
+ query.setCount(te.getProperties().getCount());
+ }
+
+ if (ObjectHelper.isNotEmpty(te.getProperties().getNumberOfPages())) {
+ numberOfPages = te.getProperties().getNumberOfPages();
+ }
+
+ LOG.debug("Searching with " + numberOfPages + " pages.");
+
+ Twitter twitter = te.getProperties().getTwitter();
+ QueryResult qr = twitter.search(query);
List<Status> tweets = qr.getTweets();
+ for (int i = 1; i < numberOfPages; i++) {
+ if (!qr.hasNext()) {
+ break;
+ }
+
+ qr = twitter.search(qr.nextQuery());
+ tweets.addAll(qr.getTweets());
+ }
+
if (te.getProperties().isFilterOld()) {
for (Status t : tweets) {
checkLastId(t.getId());
Modified: camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/producer/SearchProducer.java
URL: http://svn.apache.org/viewvc/camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/producer/SearchProducer.java?rev=1441326&r1=1441325&r2=1441326&view=diff
==============================================================================
--- camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/producer/SearchProducer.java (original)
+++ camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/producer/SearchProducer.java Fri Feb 1 07:42:13 2013
@@ -16,8 +16,6 @@
*/
package org.apache.camel.component.twitter.producer;
-import java.util.List;
-
import org.apache.camel.CamelExchangeException;
import org.apache.camel.Exchange;
import org.apache.camel.component.twitter.TwitterConstants;
@@ -28,6 +26,8 @@ import twitter4j.QueryResult;
import twitter4j.Status;
import twitter4j.Twitter;
+import java.util.List;
+
public class SearchProducer extends Twitter4JProducer {
private volatile long lastId;
@@ -39,6 +39,7 @@ public class SearchProducer extends Twit
@Override
public void process(Exchange exchange) throws Exception {
long myLastId = lastId;
+ // KEYWORDS
// keywords from header take precedence
String keywords = exchange.getIn().getHeader(TwitterConstants.TWITTER_KEYWORDS, String.class);
if (keywords == null) {
@@ -48,12 +49,15 @@ public class SearchProducer extends Twit
if (keywords == null) {
throw new CamelExchangeException("No keywords to use for query", exchange);
}
-
+
Query query = new Query(keywords);
+
+ // filter of older tweets
if (te.getProperties().isFilterOld() && myLastId != 0) {
query.setSinceId(myLastId);
}
-
+
+ // language
String lang = exchange.getIn().getHeader(TwitterConstants.TWITTER_SEARCH_LANGUAGE, String.class);
if (lang == null) {
lang = te.getProperties().getLang();
@@ -63,11 +67,35 @@ public class SearchProducer extends Twit
query.setLang(lang);
}
+ // number of elemnt per page
+ Integer count = exchange.getIn().getHeader(TwitterConstants.TWITTER_COUNT, Integer.class);
+ if (count == null) {
+ count = te.getProperties().getCount();
+ }
+ if (ObjectHelper.isNotEmpty(count)) {
+ query.setCount(count);
+ }
+
+ // number of pages
+ Integer numberOfPages = exchange.getIn().getHeader(TwitterConstants.TWITTER_NUMBER_OF_PAGES, Integer.class);
+ if (numberOfPages == null) {
+ numberOfPages = te.getProperties().getNumberOfPages();
+ }
+
Twitter twitter = te.getProperties().getTwitter();
log.debug("Searching twitter with keywords: {}", keywords);
QueryResult results = twitter.search(query);
List<Status> list = results.getTweets();
+ for (int i = 1; i < numberOfPages; i++) {
+ if (results.hasNext() == false) {
+ break;
+ }
+ log.debug("Fetching page");
+ results = twitter.search(results.nextQuery());
+ list.addAll(results.getTweets());
+ }
+
if (te.getProperties().isFilterOld()) {
for (Status t : list) {
long newId = t.getId();
Modified: camel/trunk/components/camel-twitter/src/test/java/org/apache/camel/component/twitter/UriConfigurationTest.java
URL: http://svn.apache.org/viewvc/camel/trunk/components/camel-twitter/src/test/java/org/apache/camel/component/twitter/UriConfigurationTest.java?rev=1441326&r1=1441325&r2=1441326&view=diff
==============================================================================
--- camel/trunk/components/camel-twitter/src/test/java/org/apache/camel/component/twitter/UriConfigurationTest.java (original)
+++ camel/trunk/components/camel-twitter/src/test/java/org/apache/camel/component/twitter/UriConfigurationTest.java Fri Feb 1 07:42:13 2013
@@ -38,4 +38,14 @@ public class UriConfigurationTest extend
Assert.assertTrue(!twitterEndpoint.getProperties().getAccessToken().isEmpty());
Assert.assertTrue(!twitterEndpoint.getProperties().getAccessTokenSecret().isEmpty());
}
+
+ @Test
+ public void testPageSetting() throws Exception {
+ Endpoint endpoint = context.getEndpoint("twitter:todo/page?count=50&numberOfPages=2");
+ assertTrue("Endpoint not a TwitterEndpoint: " + endpoint, endpoint instanceof TwitterEndpoint);
+ TwitterEndpoint twitterEndpoint = (TwitterEndpoint) endpoint;
+
+ Assert.assertEquals(new Integer(50), twitterEndpoint.getProperties().getCount());
+ Assert.assertEquals(new Integer(2), twitterEndpoint.getProperties().getNumberOfPages());
+ }
}