You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by re...@apache.org on 2014/11/07 22:04:30 UTC
[4/5] incubator-streams git commit: Added abstract class to reduce
repeative code
Added abstract class to reduce repeative code
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/4d6d02a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/4d6d02a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/4d6d02a0
Branch: refs/heads/master
Commit: 4d6d02a09d0cb708bb98fe17af44c82b71b0530c
Parents: ff6cb52
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Fri Nov 7 14:39:12 2014 -0600
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Fri Nov 7 14:39:12 2014 -0600
----------------------------------------------------------------------
.../gplus/provider/GPlusDataCollector.java | 50 ++++++++++++++++++++
.../provider/GPlusUserActivityCollector.java | 26 +---------
.../gplus/provider/GPlusUserDataCollector.java | 26 +---------
.../com/google/gplus/GPlusConfiguration.json | 2 +-
4 files changed, 55 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4d6d02a0/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusDataCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusDataCollector.java
new file mode 100644
index 0000000..1894dc4
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusDataCollector.java
@@ -0,0 +1,50 @@
+package com.google.gplus.provider;
+
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import org.apache.streams.util.api.requests.backoff.BackOffException;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public abstract class GPlusDataCollector implements Runnable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(GPlusDataCollector.class);
+
+
+ /**
+ * Looks at the status code of the expception. If the code indicates that the request should be retried,
+ * it executes the back off strategy and returns true.
+ * @param gjre
+ * @param backOff
+ * @return returns true if the error code of the exception indicates the request should be retried.
+ */
+ public boolean backoffAndIdentifyIfRetry(GoogleJsonResponseException gjre, BackOffStrategy backOff) throws BackOffException {
+ boolean tryAgain = false;
+ switch (gjre.getStatusCode()) {
+ case 400 :
+ LOGGER.warn("Bad Request : {}", gjre);
+ break;
+ case 401 :
+ LOGGER.warn("Invalid Credentials : {}", gjre);
+ case 403 :
+ LOGGER.warn("Possible rate limit exception. Retrying. : {}", gjre.getMessage());
+ backOff.backOff();
+ tryAgain = true;
+ break;
+ case 503 :
+ LOGGER.warn("Google Backend Service Error : {}", gjre);
+ break;
+ default:
+ LOGGER.warn("Google Service returned error : {}", gjre);
+ tryAgain = true;
+ backOff.backOff();
+ break;
+ }
+ return tryAgain;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4d6d02a0/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java
index d9a89d9..04f0aef 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java
@@ -22,7 +22,7 @@ import java.util.concurrent.BlockingQueue;
/**
* Collects the public activities of a GPlus user. Has ability to filter by date ranges.
*/
-public class GPlusUserActivityCollector implements Runnable {
+public class GPlusUserActivityCollector extends GPlusDataCollector {
/**
* Key for all public activities
@@ -90,29 +90,7 @@ public class GPlusUserActivityCollector implements Runnable {
}
}
} catch (GoogleJsonResponseException gjre) {
- switch (gjre.getStatusCode()) {
- case 400 :
- LOGGER.warn("Bad Request for user={} : {}", userInfo.getUserId(), gjre);
- tryAgain = false;
- break;
- case 401 :
- LOGGER.warn("Invalid Credentials : {}", gjre);
- tryAgain = false;
- case 403 :
- LOGGER.warn("Possible rate limit exception. Retrying. : {}", gjre.getMessage());
- this.backOff.backOff();
- tryAgain = true;
- break;
- case 503 :
- LOGGER.warn("Google Backend Service Error : {}", gjre);
- tryAgain = false;
- break;
- default:
- LOGGER.warn("Google Service returned error : {}", gjre);
- tryAgain = true;
- this.backOff.backOff();
- break;
- }
+ tryAgain = backoffAndIdentifyIfRetry(gjre, this.backOff);
++attempt;
}
} while((tryAgain || (feed != null && feed.getNextPageToken() != null)) && attempt < MAX_ATTEMPTS);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4d6d02a0/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java
index 5269e10..65ef9b1 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java
@@ -19,7 +19,7 @@ import java.util.concurrent.BlockingQueue;
/**
* Collects user profile information for a specific GPlus user
*/
-public class GPlusUserDataCollector implements Runnable{
+public class GPlusUserDataCollector extends GPlusDataCollector {
private static final Logger LOGGER = LoggerFactory.getLogger(GPlusUserDataCollector.class);
private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
@@ -56,29 +56,7 @@ public class GPlusUserDataCollector implements Runnable{
this.backOffStrategy.reset();
tryAgain = person == null;
} catch (GoogleJsonResponseException gjre) {
- switch (gjre.getStatusCode()) {
- case 400 :
- LOGGER.warn("Bad Request for user={} : {}", userInfo.getUserId(), gjre);
- tryAgain = false;
- break;
- case 401 :
- LOGGER.warn("Invalid Credentials : {}", gjre);
- tryAgain = false;
- case 403 :
- LOGGER.warn("Possible rate limit exception. Retrying. : {}", gjre.getMessage());
- this.backOffStrategy.backOff();
- tryAgain = true;
- break;
- case 503 :
- LOGGER.warn("Google Backend Service Error : {}", gjre);
- tryAgain = false;
- break;
- default:
- LOGGER.warn("Google Service returned error : {}", gjre);
- tryAgain = true;
- this.backOffStrategy.backOff();
- break;
- }
+ tryAgain = backoffAndIdentifyIfRetry(gjre, this.backOffStrategy);
}
++attempts;
} while(tryAgain && attempts < MAX_ATTEMPTS);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4d6d02a0/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/com/google/gplus/GPlusConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/com/google/gplus/GPlusConfiguration.json b/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/com/google/gplus/GPlusConfiguration.json
index 32f6a72..87ada8b 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/com/google/gplus/GPlusConfiguration.json
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/com/google/gplus/GPlusConfiguration.json
@@ -82,7 +82,7 @@
"properties": {
"userId": {
"type": "string",
- "description": "instagram user id"
+ "description": "Google+ user id"
},
"afterDate": {
"type": "string",