You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by re...@apache.org on 2015/01/06 23:13:37 UTC
incubator-streams git commit: STREAMS-260 | Only calling the back off
strategy in the even that we get a rate limit or throttling error/exception
from the Facebook Graph API
Repository: incubator-streams
Updated Branches:
refs/heads/master 7f69bd450 -> 94c7bef22
STREAMS-260 | Only calling the back off strategy in the even that we get a rate limit or throttling error/exception from the Facebook Graph API
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/94c7bef2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/94c7bef2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/94c7bef2
Branch: refs/heads/master
Commit: 94c7bef228113c5ef3fa9313e13f011b1bceb70e
Parents: 7f69bd4
Author: Robert Douglas <rd...@w2ogroup.com>
Authored: Fri Jan 2 15:06:37 2015 -0600
Committer: Robert Douglas <rd...@w2ogroup.com>
Committed: Fri Jan 2 15:06:37 2015 -0600
----------------------------------------------------------------------
.../pagefeed/FacebookPageFeedDataCollector.java | 71 +++++++++++---------
1 file changed, 38 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/94c7bef2/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/pagefeed/FacebookPageFeedDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/pagefeed/FacebookPageFeedDataCollector.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/pagefeed/FacebookPageFeedDataCollector.java
index 2027c37..e460cbe 100644
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/pagefeed/FacebookPageFeedDataCollector.java
+++ b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/pagefeed/FacebookPageFeedDataCollector.java
@@ -47,39 +47,39 @@ public class FacebookPageFeedDataCollector extends FacebookDataCollector {
protected void getData(IdConfig id) throws Exception {
boolean exit = false;
- ResponseList<Post> facebookPosts = getPosts(id.getId());
- LOGGER.debug("Post received : {}", facebookPosts.size());
- backOff.reset();
- do {
- for(Post post : facebookPosts) {
- if(id.getBeforeDate() != null && id.getAfterDate() != null) {
- if(id.getBeforeDate().isAfter(post.getCreatedTime().getTime())
- && id.getAfterDate().isBefore(post.getCreatedTime().getTime())) {
- super.outputData(MAPPER.readValue(DataObjectFactory.getRawJSON(post), org.apache.streams.facebook.Post.class), post.getId());
+ ResponseList<Post> facebookPosts = getPosts(id.getId());
+ LOGGER.debug("Post received : {}", facebookPosts.size());
+ backOff.reset();
+ do {
+ for(Post post : facebookPosts) {
+ if(id.getBeforeDate() != null && id.getAfterDate() != null) {
+ if(id.getBeforeDate().isAfter(post.getCreatedTime().getTime())
+ && id.getAfterDate().isBefore(post.getCreatedTime().getTime())) {
+ super.outputData(MAPPER.readValue(DataObjectFactory.getRawJSON(post), org.apache.streams.facebook.Post.class), post.getId());
- }
- } else if(id.getBeforeDate() != null && id.getBeforeDate().isAfter(post.getCreatedTime().getTime())) {
- super.outputData(MAPPER.readValue(DataObjectFactory.getRawJSON(post), org.apache.streams.facebook.Post.class), post.getId());
- } else if(id.getAfterDate() != null && id.getAfterDate().isBefore(post.getCreatedTime().getTime())) {
- super.outputData(MAPPER.readValue(DataObjectFactory.getRawJSON(post), org.apache.streams.facebook.Post.class), post.getId());
- } else if(id.getBeforeDate() == null && id.getAfterDate() == null) {
- super.outputData(MAPPER.readValue(DataObjectFactory.getRawJSON(post), org.apache.streams.facebook.Post.class), post.getId());
- } else {
- exit = true;
- LOGGER.debug("Breaking on post, {}, with createdAtDate {}", post.getId(), post.getCreatedTime());
- break;
- }
- }
- if(facebookPosts.getPaging() != null && !exit) {
- LOGGER.debug("Paging. . .");
- facebookPosts = getPosts(facebookPosts.getPaging());
- backOff.reset();
- LOGGER.debug("Paging received {} posts*", facebookPosts.size());
- } else {
- LOGGER.debug("No more paging.");
- facebookPosts = null;
}
- } while(facebookPosts != null && facebookPosts.size() != 0);
+ } else if(id.getBeforeDate() != null && id.getBeforeDate().isAfter(post.getCreatedTime().getTime())) {
+ super.outputData(MAPPER.readValue(DataObjectFactory.getRawJSON(post), org.apache.streams.facebook.Post.class), post.getId());
+ } else if(id.getAfterDate() != null && id.getAfterDate().isBefore(post.getCreatedTime().getTime())) {
+ super.outputData(MAPPER.readValue(DataObjectFactory.getRawJSON(post), org.apache.streams.facebook.Post.class), post.getId());
+ } else if(id.getBeforeDate() == null && id.getAfterDate() == null) {
+ super.outputData(MAPPER.readValue(DataObjectFactory.getRawJSON(post), org.apache.streams.facebook.Post.class), post.getId());
+ } else {
+ exit = true;
+ LOGGER.debug("Breaking on post, {}, with createdAtDate {}", post.getId(), post.getCreatedTime());
+ break;
+ }
+ }
+ if(facebookPosts.getPaging() != null && !exit) {
+ LOGGER.debug("Paging. . .");
+ facebookPosts = getPosts(facebookPosts.getPaging());
+ backOff.reset();
+ LOGGER.debug("Paging received {} posts*", facebookPosts.size());
+ } else {
+ LOGGER.debug("No more paging.");
+ facebookPosts = null;
+ }
+ } while(facebookPosts != null && facebookPosts.size() != 0);
}
@@ -116,9 +116,14 @@ public class FacebookPageFeedDataCollector extends FacebookDataCollector {
LOGGER.error("Facebook returned an exception while trying to get feed for page, {} : {}", pageId, fe.getMessage());
//TODO Rate limit exceptions with facebook4j unclear http://facebook4j.org/oldjavadocs/1.1.12-2.0.0/2.0.0/index.html?facebook4j/internal/http/HttpResponseCode.html
// back off at all exceptions until figured out.
- super.backOff.backOff();
+ int errorCode = fe.getErrorCode();
+
+ //Some sort of rate limiting
+ if(errorCode == 17 || errorCode == 4 || errorCode == 341) {
+ super.backOff.backOff();
+ }
}
}
throw new Exception("Failed to get data from facebook after "+MAX_ATTEMPTS);
}
-}
+}
\ No newline at end of file