You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by mf...@apache.org on 2014/06/23 20:22:09 UTC
[05/45] Fixed bugs in datasift provider and improved datasift
conversion to activity objects
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5516471e/streams-contrib/streams-provider-datasift/src/test/resources/youtube_datasift_json.txt
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/resources/youtube_datasift_json.txt b/streams-contrib/streams-provider-datasift/src/test/resources/youtube_datasift_json.txt
new file mode 100644
index 0000000..977ed70
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/test/resources/youtube_datasift_json.txt
@@ -0,0 +1,7 @@
+{"interaction":{"author":{"link":"http:\/\/youtube.com\/justmekkb","name":"justmekkb","username":"justmekkb"},"content":"Apache Snow II: Marines, Coalition Allies Battle Taliban in Helmand \nProvince!\uFEFF","contenttype":"html","created_at":"Sun, 23 Mar 2014 13:06:10 +0000","id":"1e3b28be72e0ad00e072d00a0815e3b6","link":"http:\/\/www.youtube.com\/all_comments?v=30FuTYnnXmM","received_at":4743642843909958769,"schema":{"version":3},"subtype":"comment","title":"Apache Snow II: ...","type":"youtube"},"language":{"confidence":98,"tag":"en","tag_extended":"en"},"salience":{"content":{"sentiment":0}},"youtube":{"author":{"link":"http:\/\/youtube.com\/justmekkb","name":"justmekkb","username":"justmekkb"},"commentslink":"http:\/\/www.youtube.com\/all_comments?v=30FuTYnnXmM","content":"Apache Snow II: Marines, Coalition Allies Battle Taliban in Helmand \nProvince!\uFEFF","contenttype":"html","crawled":"Sun, 30 Mar 2014 13:49:36 +0000","created_at":"Sun, 23 Mar 2014 13:06:10 +0000","domain":"
www.youtube.com","id":"1571681202","lang":"en","langid":22,"link":"http:\/\/www.youtube.com\/all_comments?v=30FuTYnnXmM","robots_disallowed":"no","site_commentid":"z12of13o5wixs3qde22ixfnxytfjgftpi04","siteid":"youtube","title":"Apache Snow II: ...","type":"comment","videoid":"5cuajavn6pjal18jnh1sp3","videolink":"http:\/\/www.youtube.com\/watch?v=30FuTYnnXmM"}}
+{"interaction":{"author":{"link":"http:\/\/youtube.com\/redman1808","name":"REDMAN1808","username":"redman1808"},"content":"Apache Knife Graduation 3-23-2014\nwww.Apache-Knife.com\uFEFF","contenttype":"html","created_at":"Tue, 25 Mar 2014 23:29:01 +0000","id":"1e3b4753edb9ac80e0722abeeb9c452c","link":"http:\/\/www.youtube.com\/all_comments?v=W0vcwufX9OM","schema":{"version":3},"subtype":"comment","title":"Apache Knife ...","type":"youtube"},"language":{"confidence":97,"tag":"en","tag_extended":"en"},"salience":{"content":{"sentiment":0}},"youtube":{"author":{"link":"http:\/\/youtube.com\/redman1808","name":"REDMAN1808","username":"redman1808"},"commentslink":"http:\/\/www.youtube.com\/all_comments?v=W0vcwufX9OM","content":"Apache Knife Graduation 3-23-2014\nwww.Apache-Knife.com\uFEFF","contenttype":"html","crawled":"Wed, 02 Apr 2014 00:23:13 +0000","created_at":"Tue, 25 Mar 2014 23:29:01 +0000","domain":"www.youtube.com","id":"1573452031","lang":"de","langid":31,"link":"http:\/\/www
.youtube.com\/all_comments?v=W0vcwufX9OM","robots_disallowed":"no","site_commentid":"z12ad53pimmefroo504ccbkitvjmzft4lwg0k","siteid":"youtube","title":"Apache Knife ...","type":"comment","videoid":"7cdig0ofjq8ls18jv5ftid","videolink":"http:\/\/www.youtube.com\/watch?v=W0vcwufX9OM"}}
+{"interaction":{"author":{"link":"http:\/\/youtube.com\/zja4ri3hkdviwos3sv_lsg","name":"zja4ri3hkdviwos3sv_lsg","username":"zja4ri3hkdviwos3sv_lsg"},"content":"Grande Apache, davvero un gran colpo ;)\uFEFF","contenttype":"html","created_at":"Thu, 27 Mar 2014 16:18:25 +0000","id":"1e3b5cb6c3a5ae80e072504981b43950","link":"http:\/\/www.youtube.com\/all_comments?v=bsCvwYmwwiQ","schema":{"version":3},"subtype":"comment","title":"Grande Apache, ...","type":"youtube"},"language":{"confidence":97,"tag":"it","tag_extended":"it"},"youtube":{"author":{"link":"http:\/\/youtube.com\/zja4ri3hkdviwos3sv_lsg","name":"zja4ri3hkdviwos3sv_lsg","username":"zja4ri3hkdviwos3sv_lsg"},"commentslink":"http:\/\/www.youtube.com\/all_comments?v=bsCvwYmwwiQ","content":"Grande Apache, davvero un gran colpo ;)\uFEFF","contenttype":"html","crawled":"Fri, 28 Mar 2014 13:44:20 +0000","created_at":"Thu, 27 Mar 2014 16:18:25 +0000","domain":"www.youtube.com","id":"1570252147","lang":"it","langid":42,"link":"http:\/\/
www.youtube.com\/all_comments?v=bsCvwYmwwiQ","robots_disallowed":"no","site_commentid":"z12kud2xxqy1vdb1a22twvzh0lmatvjet","siteid":"youtube","title":"Grande Apache, ...","type":"comment","videoid":"5n634p8c4591818k1ov5g5","videolink":"http:\/\/www.youtube.com\/watch?v=bsCvwYmwwiQ"}}
+{"interaction":{"author":{"link":"http:\/\/youtube.com\/cejowdum4lewo0p3dsv2ya","name":"cejowdum4lewo0p3dsv2ya","username":"cejowdum4lewo0p3dsv2ya"},"content":"Grande apache! Dove abiti?\uFEFF","contenttype":"html","created_at":"Thu, 27 Mar 2014 16:25:30 +0000","id":"1e3b5cc698c3a900e072da410d1cba80","link":"http:\/\/www.youtube.com\/all_comments?v=KJ2ovwkoFVw","received_at":4743644290448849319,"schema":{"version":3},"subtype":"comment","title":"Grande apache! Dove ...","type":"youtube"},"language":{"confidence":96,"tag":"en","tag_extended":"en"},"salience":{"content":{"sentiment":0},"title":{"sentiment":0}},"youtube":{"author":{"link":"http:\/\/youtube.com\/cejowdum4lewo0p3dsv2ya","name":"cejowdum4lewo0p3dsv2ya","username":"cejowdum4lewo0p3dsv2ya"},"commentslink":"http:\/\/www.youtube.com\/all_comments?v=KJ2ovwkoFVw","content":"Grande apache! Dove abiti?\uFEFF","contenttype":"html","crawled":"Thu, 03 Apr 2014 13:37:42 +0000","created_at":"Thu, 27 Mar 2014 16:25:30 +0000","domain":"
www.youtube.com","id":"1574460814","lang":"unknown","link":"http:\/\/www.youtube.com\/all_comments?v=KJ2ovwkoFVw","robots_disallowed":"no","site_commentid":"z133eviq3rquwfwp004ciptogzfwxzvy4w00k","siteid":"youtube","title":"Grande apache! Dove ...","type":"comment","videoid":"6ggq0sgl06i3318k44l1tr","videolink":"http:\/\/www.youtube.com\/watch?v=KJ2ovwkoFVw"}}
+{"interaction":{"author":{"link":"http:\/\/youtube.com\/tsunx4","name":"tsunx4","username":"tsunx4"},"content":"Sinnis Apache 125 - \u043A\u0438\u0442\u0430\u0439\u0441\u043A\u0430\u044F \u043A\u043E\u043F\u0438\u044F Suzuki DR125\uFEFF","contenttype":"html","created_at":"Sun, 30 Mar 2014 20:53:00 +0000","id":"1e3b84d47551ae00e07213c00f9208ca","link":"http:\/\/www.youtube.com\/all_comments?v=5mNtz3z-LwM","received_at":4743645206479298049,"schema":{"version":3},"subtype":"comment","title":"Sinnis Apache 125 - ...","type":"youtube"},"language":{"confidence":52,"tag":"ru","tag_extended":"ru"},"youtube":{"author":{"link":"http:\/\/youtube.com\/tsunx4","name":"tsunx4","username":"tsunx4"},"commentslink":"http:\/\/www.youtube.com\/all_comments?v=5mNtz3z-LwM","content":"Sinnis Apache 125 - \u043A\u0438\u0442\u0430\u0439\u0441\u043A\u0430\u044F \u043A\u043E\u043F\u0438\u044F Suzuki DR125\uFEFF","contenttype":"html","crawled":"Sun, 06 Apr 2014 02:17:33 +0000","created_at":"Sun, 30 Mar 2014 2
0:53:00 +0000","domain":"www.youtube.com","id":"1575948688","lang":"ru","langid":65,"link":"http:\/\/www.youtube.com\/all_comments?v=5mNtz3z-LwM","robots_disallowed":"no","site_commentid":"z13ztrsy2kmtxrxqp04cg5hqjrqjczdjbj0","siteid":"youtube","title":"Sinnis Apache 125 - ...","type":"comment","videoid":"6a4b6ht4qnqg518k8buu8c","videolink":"http:\/\/www.youtube.com\/watch?v=5mNtz3z-LwM"}}
+{"interaction":{"author":{"link":"http:\/\/youtube.com\/thepaulobraveheart","name":"ThePaulobraveheart","username":"thepaulobraveheart"},"content":"Boeing AH-64 Apache Longbow Attack Helicopter\uFEFF","contenttype":"html","created_at":"Thu, 03 Apr 2014 00:28:14 +0000","id":"1e3bac6d7ea6a300e0728131423302cc","link":"http:\/\/www.youtube.com\/all_comments?v=u4Z590EW3Jg","schema":{"version":3},"subtype":"comment","title":"Boeing AH-64 Apache ...","type":"youtube"},"language":{"confidence":97,"tag":"en","tag_extended":"en"},"salience":{"content":{"entities":[{"name":"Boeing","sentiment":0,"confident":1,"label":"Company","evidence":4,"type":"Company","about":1}],"sentiment":0},"title":{"entities":[{"name":"Boeing","sentiment":0,"confident":1,"label":"Company","evidence":4,"type":"Company","about":1}],"sentiment":0}},"youtube":{"author":{"link":"http:\/\/youtube.com\/thepaulobraveheart","name":"ThePaulobraveheart","username":"thepaulobraveheart"},"commentslink":"http:\/\/www.youtube.com\/
all_comments?v=u4Z590EW3Jg","content":"Boeing AH-64 Apache Longbow Attack Helicopter\uFEFF","contenttype":"html","crawled":"Thu, 10 Apr 2014 01:30:06 +0000","created_at":"Thu, 03 Apr 2014 00:28:14 +0000","domain":"www.youtube.com","id":"1578410306","lang":"unknown","link":"http:\/\/www.youtube.com\/all_comments?v=u4Z590EW3Jg","robots_disallowed":"no","site_commentid":"z13dsduxvsrzxpvrl04cddjiqzarhviz1vs0k","siteid":"youtube","title":"Boeing AH-64 Apache ...","type":"comment","videoid":"54geaa757jd6u18kijirss","videolink":"http:\/\/www.youtube.com\/watch?v=u4Z590EW3Jg"}}
+{"interaction":{"author":{"link":"http:\/\/youtube.com\/sqnaq_zotxi6axlutgp_ka","name":"sqnaq_zotxi6axlutgp_ka","username":"sqnaq_zotxi6axlutgp_ka"},"content":"Apache ft El Abusador de verso Lapiz calle\uFEFF","contenttype":"html","created_at":"Sat, 12 Apr 2014 14:32:27 +0000","id":"1e3c24f452c8af80e07269b9f7db9568","link":"http:\/\/www.youtube.com\/all_comments?v=ErKMJcM8H2U","received_at":4743648098787195355,"schema":{"version":3},"subtype":"comment","title":"Apache ft El ...","type":"youtube"},"language":{"confidence":97,"tag":"es","tag_extended":"es"},"salience":{"content":{"sentiment":0}},"youtube":{"author":{"link":"http:\/\/youtube.com\/sqnaq_zotxi6axlutgp_ka","name":"sqnaq_zotxi6axlutgp_ka","username":"sqnaq_zotxi6axlutgp_ka"},"commentslink":"http:\/\/www.youtube.com\/all_comments?v=ErKMJcM8H2U","content":"Apache ft El Abusador de verso Lapiz calle\uFEFF","contenttype":"html","crawled":"Mon, 14 Apr 2014 01:51:24 +0000","created_at":"Sat, 12 Apr 2014 14:32:27 +0000","domain":
"www.youtube.com","id":"1580763243","lang":"unknown","link":"http:\/\/www.youtube.com\/all_comments?v=ErKMJcM8H2U","robots_disallowed":"no","site_commentid":"z12qwfdjwte1gbjix04chfjpknihvbybt5s","siteid":"youtube","title":"Apache ft El ...","type":"comment","videoid":"75sq7rhi8thf118kvgcg6l","videolink":"http:\/\/www.youtube.com\/watch?v=ErKMJcM8H2U"}}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5516471e/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
index 7ea82b6..ff72e1b 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
@@ -130,8 +130,8 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
}
flushResults(resultSet);
// the way this works needs to change...
-// if(zeros > maxZeros)
-// this.keepRunning.set(false);
+ if(zeros > maxZeros)
+ this.keepRunning.set(false);
Thread.sleep(DEFAULT_SLEEP_TIME_MS);
} catch (InterruptedException e) {
this.keepRunning.set(false);