You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2015/04/01 01:30:34 UTC
[1/2] incubator-streams git commit: resolves STREAMS-301 includes
tests demonstrating concept of use
Repository: incubator-streams
Updated Branches:
refs/heads/master 88bb08100 -> 96c140d1c
resolves STREAMS-301
includes tests demonstrating concept of use
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/e41b7b1e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/e41b7b1e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/e41b7b1e
Branch: refs/heads/master
Commit: e41b7b1e58e130eaa535a68a64e7a06e1529425c
Parents: b99b546
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Sat Mar 28 13:24:04 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Sat Mar 28 13:24:04 2015 -0500
----------------------------------------------------------------------
.../converter/HoconConverterProcessor.java | 76 ++++++++++++
.../streams/converter/HoconConverterUtil.java | 101 ++++++++++++++++
.../test/HoconConverterProcessorTest.java | 117 +++++++++++++++++++
.../converter/test/HoconConverterTest.java | 94 +++++++++++++++
.../src/test/resources/test1.conf | 2 +
.../src/test/resources/test2.conf | 4 +
.../src/test/resources/test3a.conf | 4 +
.../src/test/resources/test3b.conf | 2 +
8 files changed, 400 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e41b7b1e/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterProcessor.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterProcessor.java b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterProcessor.java
new file mode 100644
index 0000000..26b2efa
--- /dev/null
+++ b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterProcessor.java
@@ -0,0 +1,76 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+*/
+
+package org.apache.streams.converter;
+
+import com.google.common.collect.Lists;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.core.util.DatumUtils;
+import org.apache.streams.pojo.json.Activity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+/**
+ * HoconConverterProcessor is a utility processor for converting any datum document
+ * with translation rules expressed as HOCON in the classpath or at a URL.
+ *
+ * To use this capability without a dedicated stream processor, just use HoconConverterUtil.
+ */
+public class HoconConverterProcessor implements StreamsProcessor {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(HoconConverterProcessor.class);
+
+ protected Class outClass;
+ protected String hocon;
+ protected String outPath;
+
+ public HoconConverterProcessor(Class outClass, String hocon, String outPath) {
+ this.outClass = outClass;
+ this.hocon = hocon;
+ this.outPath = outPath;
+ }
+
+ @Override
+ public List<StreamsDatum> process(StreamsDatum entry) {
+
+ List<StreamsDatum> result = Lists.newLinkedList();
+ Object document = entry.getDocument();
+
+ Object outDoc = HoconConverterUtil.convert(document, outClass, hocon, outPath);
+
+ StreamsDatum datum = DatumUtils.cloneDatum(entry);
+ datum.setDocument(outDoc);
+ result.add(datum);
+
+ return result;
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+
+ }
+
+ @Override
+ public void cleanUp() {
+
+ }
+};
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e41b7b1e/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterUtil.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterUtil.java b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterUtil.java
new file mode 100644
index 0000000..7a40606
--- /dev/null
+++ b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterUtil.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.converter;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigRenderOptions;
+import com.typesafe.config.ConfigValue;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * HoconConverterUtil supports HoconConverterProcessor in converting types via application
+ * of hocon (https://github.com/typesafehub/config/blob/master/HOCON.md) scripts
+ */
+public class HoconConverterUtil {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(HoconConverterUtil.class);
+
+ private static ObjectMapper mapper = StreamsJacksonMapper.getInstance();
+
+ public static Object convert(Object object, Class outClass, String hoconResource) {
+ Config hocon = ConfigFactory.parseResources(hoconResource);
+ return convert(object, outClass, hocon);
+ }
+
+ public static Object convert(Object object, Class outClass, String hoconResource, String outPath) {
+ Config hocon = ConfigFactory.parseResources(hoconResource);
+ return convert(object, outClass, hocon, outPath);
+ }
+
+ public static Object convert(Object object, Class outClass, Config hocon) {
+ return convert(object, outClass, hocon, null);
+ }
+
+ public static Object convert(Object object, Class outClass, Config hocon, String outPath) {
+ String json = null;
+ Object outDoc = null;
+ if( object instanceof String ) {
+ json = (String) object;
+ } else {
+ try {
+ json = mapper.writeValueAsString(object);
+ } catch (JsonProcessingException e) {
+ LOGGER.warn("Failed to process input:", object);
+ return outDoc;
+ }
+ }
+
+ Config base = ConfigFactory.parseString(json);
+ Config converted = hocon.withFallback(base);
+
+ String outJson = null;
+ try {
+ if( outPath == null )
+ outJson = converted.resolve().root().render(ConfigRenderOptions.concise());
+ else {
+ Config resolved = converted.resolve();
+ ConfigObject outObject = resolved.withOnlyPath(outPath).root();
+ ConfigValue outValue = outObject.get(outPath);
+ outJson = outValue.render(ConfigRenderOptions.concise());
+ }
+ } catch (Exception e) {
+ LOGGER.warn("Failed to convert:", json);
+ LOGGER.warn(e.getMessage());
+ }
+ if( outClass == String.class )
+ return outJson;
+ else {
+ try {
+ outDoc = mapper.readValue( outJson, outClass );
+ } catch (IOException e) {
+ LOGGER.warn("Failed to convert:", object);
+ }
+ }
+ return outDoc;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e41b7b1e/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterProcessorTest.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterProcessorTest.java b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterProcessorTest.java
new file mode 100644
index 0000000..08b2332
--- /dev/null
+++ b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterProcessorTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.converter.test;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Lists;
+import org.apache.streams.converter.HoconConverterProcessor;
+import org.apache.streams.converter.HoconConverterUtil;
+import org.apache.streams.converter.TypeConverterProcessor;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.data.util.ActivityUtil;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+import static junit.framework.Assert.*;
+
+/**
+ * Tests for
+ * @see {@link HoconConverterProcessor}
+ */
+public class HoconConverterProcessorTest {
+
+ /**
+ * Tests in-place simple conversion from String to String
+ */
+ @Test
+ public void testHoconConverter1() {
+
+ final String TEST_JSON_1 = "{\"race\":\"klingon\",\"gender\":\"male\"}";
+
+ StreamsProcessor processor = new HoconConverterProcessor(String.class, "test1.conf", null);
+ processor.prepare(null);
+ StreamsDatum datum = new StreamsDatum(TEST_JSON_1, "1");
+ List<StreamsDatum> result = processor.process(datum);
+ assertNotNull(result);
+ assertEquals(1, result.size());
+ StreamsDatum resultDatum = result.get(0);
+ assertTrue(resultDatum.getDocument() instanceof String);
+ String result1 = (String) resultDatum.getDocument();
+ assertTrue(result1.contains("race"));
+ assertTrue(result1.contains("18"));
+ assertTrue(result1.contains("female"));
+ }
+
+ /**
+ * Tests derived object substitution conversion from String to ObjectNode
+ */
+ @Test
+ public void testHoconConverter2() {
+
+ final String TEST_ID_2 = "2";
+ final String TEST_JSON_2 = "{\"race\":\"klingon\",\"gender\":\"male\",\"age\":18}";
+
+ StreamsProcessor processor = new HoconConverterProcessor(ObjectNode.class, "test2.conf", "demographics");
+ processor.prepare(null);
+ StreamsDatum datum = new StreamsDatum(TEST_JSON_2, TEST_ID_2);
+ List<StreamsDatum> result2 = processor.process(datum);
+ assertNotNull(result2);
+ assertEquals(1, result2.size());
+ StreamsDatum resultDatum = result2.get(0);
+ assertTrue(resultDatum.getDocument() instanceof ObjectNode);
+ assertTrue(resultDatum.getId().equals(TEST_ID_2));
+ ObjectNode resultDoc = (ObjectNode) resultDatum.getDocument();
+ assertNotNull(resultDoc);
+ assertTrue(resultDoc.get("race") != null);
+ assertTrue(resultDoc.get("age").asDouble() == 18);
+ assertTrue(resultDoc.get("gender").asText().equals("female"));
+ }
+
+ /**
+ * Tests derived object import conversion from String to Activity
+ */
+ @Test
+ public void testHoconConverter3() {
+
+ final String TEST_JSON_3 = "{\"id\":\"123\",\"text\":\"buncha stuff\",\"user\":{\"name\":\"guy\"}}";
+
+ StreamsProcessor processor = new HoconConverterProcessor(Activity.class, "test3a.conf", "activity");
+ processor.prepare(null);
+ StreamsDatum datum = new StreamsDatum(TEST_JSON_3, "3");
+ List<StreamsDatum> result = processor.process(datum);
+ assertNotNull(result);
+ assertEquals(1, result.size());
+ StreamsDatum resultDatum = result.get(0);
+ assertTrue(resultDatum.getDocument() instanceof Activity);
+ Activity result3 = (Activity) resultDatum.getDocument();
+ assertNotNull(result3);
+ assertTrue(result3.getProvider() != null);
+ assertTrue(result3.getId().equals("id:123"));
+ assertTrue(result3.getContent().endsWith("stuff"));
+ assertTrue(result3.getActor().getDisplayName().equals("Jorge"));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e41b7b1e/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterTest.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterTest.java b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterTest.java
new file mode 100644
index 0000000..8628e98
--- /dev/null
+++ b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.converter.test;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Lists;
+import org.apache.streams.converter.HoconConverterUtil;
+import org.apache.streams.converter.TypeConverterProcessor;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.data.util.ActivityUtil;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+import static junit.framework.Assert.*;
+
+/**
+ * Tests for
+ * @see {@link HoconConverterUtil}
+ */
+public class HoconConverterTest {
+
+ /**
+ * Tests in-place simple conversion from String to String
+ */
+ @Test
+ public void testHoconConverter1() {
+
+ final String TEST_JSON_1 = "{\"race\":\"klingon\",\"gender\":\"male\"}";
+
+ String result1 = (String) HoconConverterUtil.convert(TEST_JSON_1, String.class, "test1.conf");
+
+ assertNotNull(result1);
+ assertTrue(result1.contains("race"));
+ assertTrue(result1.contains("18"));
+ assertTrue(result1.contains("female"));
+ }
+
+ /**
+ * Tests derived object substitution conversion from String to ObjectNode
+ */
+ @Test
+ public void testHoconConverter2() {
+
+ final String TEST_JSON_2 = "{\"race\":\"klingon\",\"gender\":\"male\",\"age\":18}";
+
+ ObjectNode result2 = (ObjectNode) HoconConverterUtil.convert(TEST_JSON_2, ObjectNode.class, "test2.conf", "demographics");
+
+ assertNotNull(result2);
+ assertTrue(result2.get("race") != null);
+ assertTrue(result2.get("age").asDouble() == 18);
+ assertTrue(result2.get("gender").asText().equals("female"));
+ }
+
+ /**
+ * Tests derived object import conversion from String to Activity
+ */
+ @Test
+ public void testHoconConverter3() {
+
+ final String TEST_JSON_3 = "{\"id\":\"123\",\"text\":\"buncha stuff\",\"user\":{\"name\":\"guy\"}}";
+
+ Activity result3 = (Activity) HoconConverterUtil.convert(TEST_JSON_3, Activity.class, "test3a.conf", "activity");
+
+ assertNotNull(result3);
+ assertTrue(result3.getProvider() != null);
+ assertTrue(result3.getId().equals("id:123"));
+ assertTrue(result3.getContent().endsWith("stuff"));
+ assertTrue(result3.getActor().getDisplayName().equals("Jorge"));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e41b7b1e/streams-components/streams-converters/src/test/resources/test1.conf
----------------------------------------------------------------------
diff --git a/streams-components/streams-converters/src/test/resources/test1.conf b/streams-components/streams-converters/src/test/resources/test1.conf
new file mode 100644
index 0000000..ae8056c
--- /dev/null
+++ b/streams-components/streams-converters/src/test/resources/test1.conf
@@ -0,0 +1,2 @@
+age = 18
+gender = female
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e41b7b1e/streams-components/streams-converters/src/test/resources/test2.conf
----------------------------------------------------------------------
diff --git a/streams-components/streams-converters/src/test/resources/test2.conf b/streams-components/streams-converters/src/test/resources/test2.conf
new file mode 100644
index 0000000..b3e2cb4
--- /dev/null
+++ b/streams-components/streams-converters/src/test/resources/test2.conf
@@ -0,0 +1,4 @@
+demographics.age = 17
+demographics.age = ${?age}
+demographics.race = ${race}
+demographics.gender = fe${gender}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e41b7b1e/streams-components/streams-converters/src/test/resources/test3a.conf
----------------------------------------------------------------------
diff --git a/streams-components/streams-converters/src/test/resources/test3a.conf b/streams-components/streams-converters/src/test/resources/test3a.conf
new file mode 100644
index 0000000..4690e37
--- /dev/null
+++ b/streams-components/streams-converters/src/test/resources/test3a.conf
@@ -0,0 +1,4 @@
+activity.provider.id = test
+activity.id = "id:"${id}
+activity.content = ${text}
+activity.actor = {include "test3b.conf"}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e41b7b1e/streams-components/streams-converters/src/test/resources/test3b.conf
----------------------------------------------------------------------
diff --git a/streams-components/streams-converters/src/test/resources/test3b.conf b/streams-components/streams-converters/src/test/resources/test3b.conf
new file mode 100644
index 0000000..b53d275
--- /dev/null
+++ b/streams-components/streams-converters/src/test/resources/test3b.conf
@@ -0,0 +1,2 @@
+id = "actorid"
+displayName = "Jorge"
[2/2] incubator-streams git commit: Merge branch 'STREAMS-301' into
asf-master
Posted by sb...@apache.org.
Merge branch 'STREAMS-301' into asf-master
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/96c140d1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/96c140d1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/96c140d1
Branch: refs/heads/master
Commit: 96c140d1c4e16a0a4db2c2b159fd9f75ac1682d1
Parents: 88bb081 e41b7b1
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Tue Mar 31 18:30:04 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Tue Mar 31 18:30:04 2015 -0500
----------------------------------------------------------------------
.../converter/HoconConverterProcessor.java | 76 ++++++++++++
.../streams/converter/HoconConverterUtil.java | 101 ++++++++++++++++
.../test/HoconConverterProcessorTest.java | 117 +++++++++++++++++++
.../converter/test/HoconConverterTest.java | 94 +++++++++++++++
.../src/test/resources/test1.conf | 2 +
.../src/test/resources/test2.conf | 4 +
.../src/test/resources/test3a.conf | 4 +
.../src/test/resources/test3b.conf | 2 +
8 files changed, 400 insertions(+)
----------------------------------------------------------------------