You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by mf...@apache.org on 2014/05/27 16:00:57 UTC
[1/5] git commit: STREAMS-87 | We now try to collect an ID's timeline
put to 5 times before giving up
Repository: incubator-streams
Updated Branches:
refs/heads/master ec5d3482e -> 4580d533d
STREAMS-87 | We now try to collect an ID's timeline put to 5 times before giving up
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/a84ec404
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/a84ec404
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/a84ec404
Branch: refs/heads/master
Commit: a84ec40452f9d257804ddffc28ed511e7391f785
Parents: aa376fd
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Mon May 19 14:51:43 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Mon May 19 14:51:43 2014 -0500
----------------------------------------------------------------------
.../apache/streams/twitter/provider/TwitterTimelineProvider.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/a84ec404/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
index e4b0460..4e9b58d 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
@@ -116,10 +116,8 @@ public class TwitterTimelineProvider implements StreamsProvider, Serializable {
int keepTrying = 0;
// keep trying to load, give it 5 attempts.
- //while (keepTrying < 10)
- while (keepTrying < 1)
+ while (keepTrying < 5)
{
-
try
{
statuses = client.getUserTimeline(currentId, paging);
[4/5] git commit: Merge branch 'master' into STREAMS-87
Posted by mf...@apache.org.
Merge branch 'master' into STREAMS-87
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/106eb008
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/106eb008
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/106eb008
Branch: refs/heads/master
Commit: 106eb008a4e1e282f383455ee56be829752277e5
Parents: ce531ba a2182b1
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Thu May 22 17:41:22 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Thu May 22 17:41:22 2014 -0500
----------------------------------------------------------------------
streams-contrib/pom.xml | 1 +
.../ElasticsearchPersistWriter.java | 121 +++++++++----
.../streams/hdfs/WebHdfsPersistReaderTask.java | 3 +-
streams-contrib/streams-processor-json/pom.xml | 86 ++++++++++
.../apache/streams/json/JsonPathExtractor.java | 140 +++++++++++++++
.../org/apache/streams/json/JsonPathFilter.java | 171 +++++++++++++++++++
.../json/test/JsonPathExtractorTest.java | 101 +++++++++++
.../src/test/resources/books.json | 21 +++
.../src/main/jsonschema/com/twitter/tweet.json | 21 +--
.../org/apache/streams/core/StreamsDatum.java | 4 +
.../local/builders/LocalStreamBuilder.java | 3 +-
.../local/tasks/StreamsProviderTask.java | 5 +-
.../apache/streams/local/tasks/StreamsTask.java | 10 +-
.../local/builders/LocalStreamBuilderTest.java | 37 +++-
.../test/providers/EmptyResultSetProvider.java | 63 +++++++
streams-runtimes/streams-runtime-pig/pom.xml | 45 +++--
.../streams/pig/StreamsComponentFactory.java | 22 ++-
.../apache/streams/pig/StreamsPigBuilder.java | 24 ++-
.../streams/pig/StreamsProcessDatumExec.java | 127 ++++++++++++--
.../streams/pig/StreamsProcessDocumentExec.java | 69 ++++----
.../streams/pig/StreamsSerializerExec.java | 36 ++--
.../org/apache/streams/pig/StreamsStorage.java | 22 ++-
.../src/test/java/PigProcessorTest.java | 32 ----
.../src/test/java/PigSerializerTest.java | 40 -----
.../streams/pig/test/CopyThriceProcessor.java | 56 ++++++
.../streams/pig/test/DoNothingProcessor.java | 54 ++++++
.../streams/pig/test/PigProcessDatumTest.java | 99 +++++++++++
.../pig/test/PigProcessDocumentTest.java | 90 ++++++++++
.../streams/pig/test/PigSerializerTest.java | 62 +++++++
.../test/resources/pigprocessdatumcopytest.pig | 4 +
.../src/test/resources/pigprocessdatumtest.pig | 4 +
.../test/resources/pigprocessdocumenttest.pig | 4 +
.../src/test/resources/pigprocessortest.pig | 7 -
.../src/test/resources/pigserializertest.pig | 12 +-
34 files changed, 1384 insertions(+), 212 deletions(-)
----------------------------------------------------------------------
[3/5] git commit: Merge pull request #1 from apache/master
Posted by mf...@apache.org.
Merge pull request #1 from apache/master
merge from apache
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/a2182b12
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/a2182b12
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/a2182b12
Branch: refs/heads/master
Commit: a2182b1237762493f1c9b7768bce7f4ff61add99
Parents: aa376fd 18b358f
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Thu May 22 17:39:43 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Thu May 22 17:39:43 2014 -0500
----------------------------------------------------------------------
streams-contrib/pom.xml | 1 +
.../ElasticsearchPersistWriter.java | 121 +++++++++----
.../streams/hdfs/WebHdfsPersistReaderTask.java | 3 +-
streams-contrib/streams-processor-json/pom.xml | 86 ++++++++++
.../apache/streams/json/JsonPathExtractor.java | 140 +++++++++++++++
.../org/apache/streams/json/JsonPathFilter.java | 171 +++++++++++++++++++
.../json/test/JsonPathExtractorTest.java | 101 +++++++++++
.../src/test/resources/books.json | 21 +++
.../src/main/jsonschema/com/twitter/tweet.json | 21 +--
.../org/apache/streams/core/StreamsDatum.java | 4 +
.../local/builders/LocalStreamBuilder.java | 3 +-
.../local/tasks/StreamsProviderTask.java | 5 +-
.../apache/streams/local/tasks/StreamsTask.java | 10 +-
.../local/builders/LocalStreamBuilderTest.java | 37 +++-
.../test/providers/EmptyResultSetProvider.java | 63 +++++++
streams-runtimes/streams-runtime-pig/pom.xml | 45 +++--
.../streams/pig/StreamsComponentFactory.java | 22 ++-
.../apache/streams/pig/StreamsPigBuilder.java | 24 ++-
.../streams/pig/StreamsProcessDatumExec.java | 127 ++++++++++++--
.../streams/pig/StreamsProcessDocumentExec.java | 69 ++++----
.../streams/pig/StreamsSerializerExec.java | 36 ++--
.../org/apache/streams/pig/StreamsStorage.java | 22 ++-
.../src/test/java/PigProcessorTest.java | 32 ----
.../src/test/java/PigSerializerTest.java | 40 -----
.../streams/pig/test/CopyThriceProcessor.java | 56 ++++++
.../streams/pig/test/DoNothingProcessor.java | 54 ++++++
.../streams/pig/test/PigProcessDatumTest.java | 99 +++++++++++
.../pig/test/PigProcessDocumentTest.java | 90 ++++++++++
.../streams/pig/test/PigSerializerTest.java | 62 +++++++
.../test/resources/pigprocessdatumcopytest.pig | 4 +
.../src/test/resources/pigprocessdatumtest.pig | 4 +
.../test/resources/pigprocessdocumenttest.pig | 4 +
.../src/test/resources/pigprocessortest.pig | 7 -
.../src/test/resources/pigserializertest.pig | 12 +-
34 files changed, 1384 insertions(+), 212 deletions(-)
----------------------------------------------------------------------
[2/5] git commit: STREAMS-87 | Added comment detailing thoughts about
why '5' was chosen for the number of retry attempts
Posted by mf...@apache.org.
STREAMS-87 | Added comment detailing thoughts about why '5' was chosen for the number of retry attempts
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/ce531ba7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/ce531ba7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/ce531ba7
Branch: refs/heads/master
Commit: ce531ba791fb55d107950526ad8efceda274693c
Parents: a84ec40
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Thu May 22 15:54:42 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Thu May 22 15:54:42 2014 -0500
----------------------------------------------------------------------
.../apache/streams/twitter/provider/TwitterTimelineProvider.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ce531ba7/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
index 4e9b58d..49a2d39 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
@@ -116,6 +116,9 @@ public class TwitterTimelineProvider implements StreamsProvider, Serializable {
int keepTrying = 0;
// keep trying to load, give it 5 attempts.
+ //This value was chosen because it seemed like a reasonable number of times
+ //to retry capturing a timeline given the sorts of errors that could potentially
+ //occur (network timeout/interruption, faulty client, etc.)
while (keepTrying < 5)
{
try
[5/5] git commit: Merged PR#19 from rdouglas/STREAMS-87
Posted by mf...@apache.org.
Merged PR#19 from rdouglas/STREAMS-87
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/4580d533
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/4580d533
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/4580d533
Branch: refs/heads/master
Commit: 4580d533dd94dec08900a17f9499d36467ad77fa
Parents: ec5d348 106eb00
Author: mfranklin <mf...@apache.org>
Authored: Tue May 27 09:49:02 2014 -0400
Committer: mfranklin <mf...@apache.org>
Committed: Tue May 27 09:49:02 2014 -0400
----------------------------------------------------------------------
.../provider/TwitterTimelineProvider.java | 23 ++++----------------
1 file changed, 4 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4580d533/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
----------------------------------------------------------------------
diff --cc streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
index b6e4332,49a2d39..72e58d7
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
@@@ -137,9 -116,11 +119,12 @@@ public class TwitterTimelineProvider im
int keepTrying = 0;
// keep trying to load, give it 5 attempts.
- while (keepTrying < 1)
+ //This value was chosen because it seemed like a reasonable number of times
+ //to retry capturing a timeline given the sorts of errors that could potentially
+ //occur (network timeout/interruption, faulty client, etc.)
+ while (keepTrying < 5)
{
+
try
{
statuses = client.getUserTimeline(currentId, paging);