You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2014/05/26 21:43:44 UTC
[1/2] git commit: Additional tests Additional logging Fix to prepare
statement for compatibility with runtime-pig
Repository: incubator-streams
Updated Branches:
refs/heads/master 57f81623b -> cbb60a470
Additional tests
Additional logging
Fix to prepare statement for compatibility with runtime-pig
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/69244fad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/69244fad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/69244fad
Branch: refs/heads/master
Commit: 69244fad451d04149b325d6e8d93da22f864f039
Parents: a4573f2
Author: sblackmon <sb...@w2odigital.com>
Authored: Fri May 23 12:56:53 2014 -0500
Committer: sblackmon <sb...@w2odigital.com>
Committed: Fri May 23 12:56:53 2014 -0500
----------------------------------------------------------------------
.../apache/streams/json/JsonPathExtractor.java | 13 ++-
.../json/test/JsonPathExtractorTwitterTest.java | 92 ++++++++++++++++++++
.../src/test/resources/tweet.json | 1 +
3 files changed, 105 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/69244fad/streams-contrib/streams-processor-json/src/main/java/org/apache/streams/json/JsonPathExtractor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-json/src/main/java/org/apache/streams/json/JsonPathExtractor.java b/streams-contrib/streams-processor-json/src/main/java/org/apache/streams/json/JsonPathExtractor.java
index c4920d8..8164eb8 100644
--- a/streams-contrib/streams-processor-json/src/main/java/org/apache/streams/json/JsonPathExtractor.java
+++ b/streams-contrib/streams-processor-json/src/main/java/org/apache/streams/json/JsonPathExtractor.java
@@ -88,37 +88,46 @@ public class JsonPathExtractor implements StreamsProcessor {
if (readResult instanceof String) {
String match = (String) readResult;
+ LOGGER.info("Matched String: " + match);
StreamsDatum matchDatum = new StreamsDatum(match);
result.add(matchDatum);
} else if (readResult instanceof JSONObject) {
JSONObject match = (JSONObject) readResult;
+ LOGGER.info("Matched Object: " + match);
ObjectNode objectNode = mapper.readValue(mapper.writeValueAsString(match), ObjectNode.class);
StreamsDatum matchDatum = new StreamsDatum(objectNode);
result.add(matchDatum);
} else if (readResult instanceof JSONArray) {
+ LOGGER.info("Matched Array:");
JSONArray array = (JSONArray) readResult;
Iterator iterator = array.iterator();
while (iterator.hasNext()) {
Object item = iterator.next();
if( item instanceof String ) {
+ LOGGER.info("String Item:" + item);
String match = (String) item;
StreamsDatum matchDatum = new StreamsDatum(match);
result.add(matchDatum);
} else if ( item instanceof JSONObject ) {
+ LOGGER.info("Object Item:" + item);
JSONObject match = (JSONObject) item;
ObjectNode objectNode = mapper.readValue(mapper.writeValueAsString(match), ObjectNode.class);
StreamsDatum matchDatum = new StreamsDatum(objectNode);
result.add(matchDatum);
+ } else {
+ LOGGER.info("Other Item:" + item.toString());
}
}
} else {
-
+ LOGGER.info("Other Match:" + readResult.toString());
}
} catch( Exception e ) {
LOGGER.warn(e.getMessage());
}
+ } else {
+ LOGGER.warn("result empty");
}
return result;
@@ -129,6 +138,8 @@ public class JsonPathExtractor implements StreamsProcessor {
public void prepare(Object configurationObject) {
if( configurationObject instanceof String )
jsonPath = JsonPath.compile((String)(configurationObject));
+ else if( configurationObject instanceof String[] )
+ jsonPath = JsonPath.compile(((String[])(configurationObject))[0]);
mapper.registerModule(new JsonOrgModule());
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/69244fad/streams-contrib/streams-processor-json/src/test/java/org/apache/streams/json/test/JsonPathExtractorTwitterTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-json/src/test/java/org/apache/streams/json/test/JsonPathExtractorTwitterTest.java b/streams-contrib/streams-processor-json/src/test/java/org/apache/streams/json/test/JsonPathExtractorTwitterTest.java
new file mode 100644
index 0000000..b5d5766
--- /dev/null
+++ b/streams-contrib/streams-processor-json/src/test/java/org/apache/streams/json/test/JsonPathExtractorTwitterTest.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.json.test;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.commons.io.FileUtils;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.json.JsonPathExtractor;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test for extracting json fields and
+ * objects from datums using JsonPath syntax
+ */
+public class JsonPathExtractorTwitterTest {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(JsonPathExtractorTwitterTest.class);
+
+ private String testJson;
+
+ @Before
+ public void initialize() {
+ try {
+ testJson = FileUtils.readFileToString(new File("src/test/resources/tweet.json"));
+ } catch (IOException e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ @Test
+ public void test1()
+ {
+ JsonPathExtractor extractor = new JsonPathExtractor();
+ extractor.prepare("$.user.id_str");
+ List<StreamsDatum> result = extractor.process(new StreamsDatum(testJson));
+ assertThat(result.size(), is(1));
+ assertTrue(result.get(0).getDocument() instanceof String);
+ assertThat((String)result.get(0).getDocument(), is("3971941"));
+ }
+
+ @Test
+ public void test2()
+ {
+ JsonPathExtractor extractor = new JsonPathExtractor();
+ extractor.prepare("$.created_at");
+ List<StreamsDatum> result = extractor.process(new StreamsDatum(testJson));
+ assertThat(result.size(), is(1));
+ assertTrue(result.get(0).getDocument() instanceof String);
+ assertThat((String)result.get(0).getDocument(), is("Thu May 08 15:58:59 +0000 2014"));
+ }
+
+ @Test
+ public void test3()
+ {
+ JsonPathExtractor extractor = new JsonPathExtractor();
+ extractor.prepare("$.user");
+ List<StreamsDatum> result = extractor.process(new StreamsDatum(testJson));
+ assertThat(result.size(), is(1));
+ assertTrue(result.get(0).getDocument() instanceof ObjectNode);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/69244fad/streams-contrib/streams-processor-json/src/test/resources/tweet.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-json/src/test/resources/tweet.json b/streams-contrib/streams-processor-json/src/test/resources/tweet.json
new file mode 100644
index 0000000..8975829
--- /dev/null
+++ b/streams-contrib/streams-processor-json/src/test/resources/tweet.json
@@ -0,0 +1 @@
+{"contributors":null,"text":"I just read through an entire white paper and enjoyed it. Is this how mid-life begins?","geo":null,"retweeted":false,"in_reply_to_screen_name":null,"truncated":false,"lang":"en","entities":{"symbols":[],"urls":[],"hashtags":[],"user_mentions":[]},"in_reply_to_status_id_str":null,"id":464434297516986368,"source":"web","in_reply_to_user_id_str":null,"favorited":false,"in_reply_to_status_id":null,"retweet_count":1,"created_at":"Thu May 08 15:58:59 +0000 2014","in_reply_to_user_id":null,"favorite_count":0,"id_str":"464434297516986368","place":null,"user":{"location":"Austin, TX","default_profile":false,"profile_background_tile":false,"statuses_count":8135,"lang":"en","profile_link_color":"0000FF","profile_banner_url":"https://pbs.twimg.com/profile_banners/3971941/1397146508","id":3971941,"following":false,"protected":false,"favourites_count":19,"profile_text_color":"383838","description":"VP of the McBeth family, @alliemcbeth is CEO. Head of Media & Planning
@WCG, binge cyclist, west coast kid, self Google-er. Own most of my opinions.","verified":false,"contributors_enabled":false,"profile_sidebar_border_color":"000000","name":"Dean McBeth","profile_background_color":"101010","created_at":"Tue Apr 10 02:44:04 +0000 2007","is_translation_enabled":false,"default_profile_image":false,"followers_count":7453,"profile_image_url_https":"https://pbs.twimg.com/profile_images/2455687917/5nviqk54ga68222vxhhk_normal.jpeg","geo_enabled":true,"profile_background_image_url":"http://pbs.twimg.com/profile_background_images/449113958/JESS3_SXSW_dean.jpg","profile_background_image_url_https":"https://pbs.twimg.com/profile_background_images/449113958/JESS3_SXSW_dean.jpg","follow_request_sent":false,"entities":{"description":{"urls":[]},"url":{"urls":[{"expanded_url":"http://about.me/evilspinmeister","indices":[0,22],"display_url":"about.me/evilspinmeister","url":"http://t.co/1l1lAsEuO0"}]}},"url":"http://t.co/1l1lAsEuO0","utc_offset":-25200,"time_zone":"P
acific Time (US & Canada)","notifications":false,"profile_use_background_image":true,"friends_count":2768,"profile_sidebar_fill_color":"000000","screen_name":"evilspinmeister","id_str":"3971941","profile_image_url":"http://pbs.twimg.com/profile_images/2455687917/5nviqk54ga68222vxhhk_normal.jpeg","listed_count":383,"is_translator":false},"coordinates":null}
\ No newline at end of file
[2/2] git commit: Merge commit
'69244fad451d04149b325d6e8d93da22f864f039'
Posted by sb...@apache.org.
Merge commit '69244fad451d04149b325d6e8d93da22f864f039'
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/cbb60a47
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/cbb60a47
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/cbb60a47
Branch: refs/heads/master
Commit: cbb60a470713d9145915e8f3a7e3d4ca3dddfabd
Parents: 57f8162 69244fa
Author: Steve Blackmon <sb...@w2odigital.com>
Authored: Mon May 26 14:43:24 2014 -0500
Committer: Steve Blackmon <sb...@w2odigital.com>
Committed: Mon May 26 14:43:24 2014 -0500
----------------------------------------------------------------------
----------------------------------------------------------------------