You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2014/08/21 23:50:03 UTC
git commit: moved json cleaner into streams-processor-jackson better
javadoc comment
Repository: incubator-streams
Updated Branches:
refs/heads/STREAMS-142 614f40344 -> 0536c98a8
moved json cleaner into streams-processor-jackson
better javadoc comment
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/0536c98a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/0536c98a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/0536c98a
Branch: refs/heads/STREAMS-142
Commit: 0536c98a845def2e09c7fa14ca8c0c4df0666505
Parents: 614f403
Author: sblackmon <sb...@apache.org>
Authored: Thu Aug 21 16:49:35 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Thu Aug 21 16:49:35 2014 -0500
----------------------------------------------------------------------
streams-contrib/pom.xml | 1 +
.../streams-processor-jackson/pom.xml | 86 ++++++++++++++++++++
.../CleanAdditionalPropertiesProcessor.java | 62 ++++++++++++++
.../streams-provider-datasift/pom.xml | 5 ++
.../CleanAdditionalPropertiesProcessor.java | 62 --------------
.../DatasiftTypeConverterProcessor.java | 1 +
6 files changed, 155 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0536c98a/streams-contrib/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/pom.xml b/streams-contrib/pom.xml
index 699274e..e290466 100644
--- a/streams-contrib/pom.xml
+++ b/streams-contrib/pom.xml
@@ -48,6 +48,7 @@
<!--<module>streams-processor-lucene</module>-->
<!--<module>streams-processor-tika</module>-->
<module>streams-provider-instagram</module>
+ <module>streams-processor-jackson</module>
<module>streams-processor-json</module>
<module>streams-processor-urls</module>
<module>streams-provider-datasift</module>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0536c98a/streams-contrib/streams-processor-jackson/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-jackson/pom.xml b/streams-contrib/streams-processor-jackson/pom.xml
new file mode 100644
index 0000000..8bb44fd
--- /dev/null
+++ b/streams-contrib/streams-processor-jackson/pom.xml
@@ -0,0 +1,86 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>streams-processor-jackson</artifactId>
+ <version>0.1-SNAPSHOT</version>
+
+ <parent>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-contrib</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </parent>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.jayway.jsonpath</groupId>
+ <artifactId>json-path</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.jayway.jsonpath</groupId>
+ <artifactId>json-path-assert</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0536c98a/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/CleanAdditionalPropertiesProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/CleanAdditionalPropertiesProcessor.java b/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/CleanAdditionalPropertiesProcessor.java
new file mode 100644
index 0000000..56b0c5c
--- /dev/null
+++ b/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/CleanAdditionalPropertiesProcessor.java
@@ -0,0 +1,62 @@
+package org.apache.streams.jackson;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.datatype.jsonorg.JsonOrgModule;
+import com.google.common.collect.Lists;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This processor walks an input objectnode and corrects any artifacts
+ * that may have occured from improper serialization of jsonschema2pojo beans.
+ *
+ * The logic is also available for inclusion in other module via static import.
+ */
+public class CleanAdditionalPropertiesProcessor implements StreamsProcessor {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(CleanAdditionalPropertiesProcessor.class);
+
+ private ObjectMapper mapper;
+
+ @Override
+ public List<StreamsDatum> process(StreamsDatum datum) {
+ List<StreamsDatum> result = Lists.newLinkedList();
+ ObjectNode activity = this.mapper.convertValue(datum.getDocument(), ObjectNode.class);
+ cleanAdditionalProperties(activity);
+ datum.setDocument(activity);
+ result.add(datum);
+ return result;
+ }
+
+ @Override
+ public void prepare(Object o) {
+ this.mapper = StreamsJacksonMapper.getInstance();
+ this.mapper.registerModule(new JsonOrgModule());
+ }
+
+ @Override
+ public void cleanUp() {
+
+ }
+
+ public static void cleanAdditionalProperties(ObjectNode node) {
+ if( node.get("additionalProperties") != null ) {
+ ObjectNode additionalProperties = (ObjectNode) node.get("additionalProperties");
+ cleanAdditionalProperties(additionalProperties);
+ Iterator<Map.Entry<String, JsonNode>> jsonNodeIterator = additionalProperties.fields();
+ while( jsonNodeIterator.hasNext() ) {
+ Map.Entry<String, JsonNode> entry = jsonNodeIterator.next();
+ node.put(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0536c98a/streams-contrib/streams-provider-datasift/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/pom.xml b/streams-contrib/streams-provider-datasift/pom.xml
index f2aab08..782db07 100644
--- a/streams-contrib/streams-provider-datasift/pom.xml
+++ b/streams-contrib/streams-provider-datasift/pom.xml
@@ -37,6 +37,11 @@
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
+ <artifactId>streams-processor-jackson</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
<artifactId>streams-provider-twitter</artifactId>
<version>${project.version}</version>
<optional>true</optional>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0536c98a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java
deleted file mode 100644
index 00f08e7..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java
+++ /dev/null
@@ -1,62 +0,0 @@
-package org.apache.streams.datasift.processor;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.fasterxml.jackson.datatype.jsonorg.JsonOrgModule;
-import com.google.common.collect.Lists;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-/**
- * HACK PROCESSOR. Changes need to be made in apache streams to fix this issue long term.
- */
-public class CleanAdditionalPropertiesProcessor implements StreamsProcessor {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(CleanAdditionalPropertiesProcessor.class);
-
- private static final String ADDITIONAL_PROPERTIES = "additionalProperties";
- private static final String EXTENSIONS = "extensions";
-
- private ObjectMapper mapper;
-
- @Override
- public List<StreamsDatum> process(StreamsDatum datum) {
- List<StreamsDatum> result = Lists.newLinkedList();
- ObjectNode activity = this.mapper.convertValue(datum.getDocument(), ObjectNode.class);
- cleanAdditionalProperties(activity);
- datum.setDocument(activity);
- result.add(datum);
- return result;
- }
-
- @Override
- public void prepare(Object o) {
- this.mapper = StreamsJacksonMapper.getInstance();
- this.mapper.registerModule(new JsonOrgModule());
- }
-
- @Override
- public void cleanUp() {
-
- }
-
- public static void cleanAdditionalProperties(ObjectNode node) {
- if( node.get("additionalProperties") != null ) {
- ObjectNode additionalProperties = (ObjectNode) node.get("additionalProperties");
- cleanAdditionalProperties(additionalProperties);
- Iterator<Map.Entry<String, JsonNode>> jsonNodeIterator = additionalProperties.fields();
- while( jsonNodeIterator.hasNext() ) {
- Map.Entry<String, JsonNode> entry = jsonNodeIterator.next();
- node.put(entry.getKey(), entry.getValue());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0536c98a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
index e85e8b1..a00cf23 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
@@ -28,6 +28,7 @@ import org.apache.streams.datasift.provider.DatasiftConverter;
import org.apache.streams.datasift.serializer.DatasiftActivitySerializer;
import org.apache.streams.datasift.util.StreamsDatasiftMapper;
import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.jackson.CleanAdditionalPropertiesProcessor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;