You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2014/03/24 16:50:38 UTC
[01/13] adding some uncommitted modules reorganizing pom improvements
Repository: incubator-streams
Updated Branches:
refs/heads/springcleaning [created] adb43b295
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestExpectedDatumsPersitWriter.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestExpectedDatumsPersitWriter.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestExpectedDatumsPersitWriter.java
new file mode 100644
index 0000000..d12b250
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestExpectedDatumsPersitWriter.java
@@ -0,0 +1,38 @@
+package org.apache.streams.test.component.tests;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.test.component.ExpectedDatumsPersistWriter;
+import org.apache.streams.test.component.StringToDocumentConverter;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Created by rebanks on 2/28/14.
+ */
+public class TestExpectedDatumsPersitWriter {
+
+ private static final StreamsDatum[] INPUT_DATUMS = new StreamsDatum[] {
+ new StreamsDatum("Document1"),
+ new StreamsDatum("Document2"),
+ new StreamsDatum("Document3"),
+ new StreamsDatum("Document4")
+// Uncomment to prove failures occur, or comment out a datum above
+// ,new StreamsDatum("Document5")
+ };
+
+ @Test
+ public void testExpectedDatumsPersistWriterFileName() {
+ testDatums(new ExpectedDatumsPersistWriter(new StringToDocumentConverter(), "/TestFile.txt"));
+ }
+
+
+
+ private void testDatums(ExpectedDatumsPersistWriter writer) {
+ writer.prepare(null);
+ for(StreamsDatum datum : INPUT_DATUMS) {
+ writer.write(datum);
+ }
+ writer.cleanUp();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestFileReaderProvider.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestFileReaderProvider.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestFileReaderProvider.java
new file mode 100644
index 0000000..c562c20
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestFileReaderProvider.java
@@ -0,0 +1,39 @@
+package org.apache.streams.test.component.tests;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsResultSet;
+import org.apache.streams.test.component.FileReaderProvider;
+import org.apache.streams.test.component.StringToDocumentConverter;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.InputStream;
+
+import static org.junit.Assert.*;
+
+/**
+ * Created by rebanks on 2/28/14.
+ */
+public class TestFileReaderProvider {
+
+
+ @Test
+ public void testFileReaderProviderFileName() {
+ String fileName = "/TestFile.txt";
+ FileReaderProvider provider = new FileReaderProvider(fileName, new StringToDocumentConverter());
+ provider.prepare(null);
+ StreamsResultSet resultSet = provider.readCurrent();
+ int count = 0;
+ for(StreamsDatum datum : resultSet) {
+ ++count;
+ }
+ assertEquals(4, count);
+ provider.cleanUp();
+ }
+
+
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-storm/pom.xml
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-storm/pom.xml b/streams-runtimes/streams-runtime-storm/pom.xml
new file mode 100644
index 0000000..819eb82
--- /dev/null
+++ b/streams-runtimes/streams-runtime-storm/pom.xml
@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>streams-project</artifactId>
+ <groupId>org.apache.streams</groupId>
+ <version>0.1-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>streams-runtime-storm</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config-graph</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-util</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-collections4</artifactId>
+ <version>4.0</version>
+ </dependency>
+ <dependency>
+ <groupId>storm</groupId>
+ <artifactId>storm-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>storm</groupId>
+ <artifactId>storm-netty</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ <version>2.9.2</version>
+ <scope>compile</scope>
+ <type>jar</type>
+ </dependency>
+ <dependency>
+ <groupId>com.101tec</groupId>
+ <artifactId>zkclient</artifactId>
+ <version>0.2</version>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-storm/src/main/java/org/apache/streams/storm/PipelineGraphLauncher.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-storm/src/main/java/org/apache/streams/storm/PipelineGraphLauncher.java b/streams-runtimes/streams-runtime-storm/src/main/java/org/apache/streams/storm/PipelineGraphLauncher.java
new file mode 100644
index 0000000..a0ad2ea
--- /dev/null
+++ b/streams-runtimes/streams-runtime-storm/src/main/java/org/apache/streams/storm/PipelineGraphLauncher.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.storm;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.typesafe.config.Config;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.util.RegexUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: sblackmon
+ * Date: 9/20/13
+ * Time: 11:17 AM
+ * To change this template use File | Settings | File Templates.
+ */
+public class PipelineGraphLauncher {
+
+ private static final Logger log = LoggerFactory.getLogger(PipelineGraphLauncher.class);
+
+ private static Config streamsConfiguration;
+
+ private static List<Pair<String,Class>> topologies;
+
+ private static List<Pair<String,Class>> resolveClasses(List<Pair<String,String>> topologyPairs) throws IOException, ClassNotFoundException {
+
+ List<Pair<String,Class>> topologies = new ArrayList<Pair<String,Class>>();
+
+ for( Pair<String,String> pair : topologyPairs ) {
+ String topologyId = pair.getLeft();
+ Class topologyClass = Class.forName(pair.getRight());
+ topologies.add(new ImmutablePair(topologyId, topologyClass));
+ }
+
+ return topologies;
+ }
+
+ private static List<Pair<String,Class>> loadTopologiesFromPipelineTopologyListFile(File file) throws IOException, ClassNotFoundException {
+
+ List<String> lines = IOUtils.readLines(FileUtils.openInputStream(file));
+ String pattern = "^([\\w-]*)[\\s]*?([\\w.]*)$";
+
+ List<Pair<String,String>> topologyPairs = RegexUtils.getTwoMatchedGroupsList(lines, pattern);
+
+ topologies = resolveClasses(topologyPairs);
+
+ for( Pair<String,String> pair : topologyPairs ) {
+ String topologyId = pair.getLeft();
+ Class topologyClass = Class.forName(pair.getRight());
+ topologies.add(new ImmutablePair(topologyId, topologyClass));
+ }
+
+ return topologies;
+ }
+
+ private static List<Pair<String,Class>> loadTopologiesFromPipelineGraphFile(File file) throws IOException, ClassNotFoundException {
+
+ List<String> lines = IOUtils.readLines(FileUtils.openInputStream(file));
+ String pattern = "$([\\w-]*)\\s([\\w.)";
+
+ List<Pair<String,String>> topologyPairs = RegexUtils.getTwoMatchedGroupsList(lines, pattern);
+
+ topologies = resolveClasses(topologyPairs);
+
+ return topologies;
+ }
+
+ public static boolean isLocal(String[] args) {
+ if(args.length >= 1 && args[1].equals("deploy"))
+ return false;
+ else return true;
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ if(args.length < 3) {
+ log.error("Not enough arguments");
+ log.error(" storm {local,deploy} <pipelinedef>");
+ return;
+ }
+ if(!args[1].equals("deploy")) {
+ log.error("Not a deploy");
+ log.error(" storm {local,deploy} <pipelinedef>");
+ return;
+ }
+
+ ObjectMapper mapper = new ObjectMapper();
+
+ URL configFileUrl = PipelineGraphLauncher.class.getResource(args[0]);
+
+ File configFile;
+ try {
+ configFile = new File(configFileUrl.toURI());
+ } catch(URISyntaxException e) {
+ configFile = new File(configFileUrl.getPath());
+ }
+
+ streamsConfiguration = StreamsConfigurator.config;
+
+// String pipelineIdentifier = streamsConfiguration.getPipeline();
+//
+// for( Map.Entry<String, Object> moduleConfig : streamsConfiguration.getAdditionalProperties().entrySet()) {
+//
+// }
+
+// each defined graph becomes a topology
+
+//
+// URL pipelineFileUrl = PipelineGraphLauncher.class.getResource(args[1]);
+//
+// File pipelineFile;
+// try {
+// pipelineFile = new File(pipelineFileUrl.toURI());
+// } catch(URISyntaxException e) {
+// pipelineFile = new File(pipelineFileUrl.getPath());
+// }
+//
+// topologies = loadTopologiesFromPipelineTopologyListFile(pipelineFile);
+//
+// for( Pair<String,Class> topology : topologies ) {
+// Class topologyClass = topology.getRight();
+//
+// try {
+// Constructor ctor = topologyClass.getDeclaredConstructor(
+// String.class,
+// StreamsConfiguration.class);
+// ctor.setAccessible(true);
+// Object topologyObject = ctor.newInstance(pipelineIdentifier, streamsConfiguration);
+// Runnable runnable = (Runnable) topologyObject;
+// runnable.run();
+// } catch (InstantiationException x) {
+// log.warn(x.getMessage());
+// x.printStackTrace();
+// } catch (IllegalAccessException x) {
+// log.warn(x.getMessage());
+// x.printStackTrace();
+// } catch (InvocationTargetException x) {
+// log.warn(x.getMessage());
+// x.printStackTrace();
+// } catch (NoSuchMethodException x) {
+// log.warn(x.getMessage());
+// x.printStackTrace();
+//
+// try {
+// Constructor ctor = topologyClass.getDeclaredConstructor(
+// String[].class);
+// ctor.setAccessible(true);
+// Object topologyObject = ctor.newInstance(args);
+// Method main = topologyClass.getMethod("main", String[].class);
+// main.invoke(topologyObject, args);
+// } catch (InstantiationException x2) {
+// log.warn(x2.getMessage());
+// x.printStackTrace();
+// } catch (IllegalAccessException x2) {
+// log.warn(x2.getMessage());
+// x.printStackTrace();
+// } catch (InvocationTargetException x2) {
+// log.warn(x2.getMessage());
+// x.printStackTrace();
+// } catch (NoSuchMethodException x2) {
+// log.error(x2.getMessage());
+// x.printStackTrace();
+// }
+// }
+// }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-storm/src/main/java/org/apache/streams/storm/trident/StreamsPersistWriterState.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-storm/src/main/java/org/apache/streams/storm/trident/StreamsPersistWriterState.java b/streams-runtimes/streams-runtime-storm/src/main/java/org/apache/streams/storm/trident/StreamsPersistWriterState.java
new file mode 100644
index 0000000..999099c
--- /dev/null
+++ b/streams-runtimes/streams-runtime-storm/src/main/java/org/apache/streams/storm/trident/StreamsPersistWriterState.java
@@ -0,0 +1,106 @@
+package org.apache.streams.storm.trident;
+
+import backtype.storm.task.IMetricsContext;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsPersistWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import storm.trident.operation.TridentCollector;
+import storm.trident.state.BaseStateUpdater;
+import storm.trident.state.State;
+import storm.trident.state.StateFactory;
+import storm.trident.tuple.TridentTuple;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Created by sblackmon on 1/16/14.
+ */
+public class StreamsPersistWriterState implements State {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(StreamsPersistWriterState.class);
+
+ StreamsPersistWriter writer;
+ StreamsPersistStateController controller;
+
+ public StreamsPersistWriterState(StreamsPersistStateController controller) {
+ this.controller = new StreamsPersistStateController();
+ writer.prepare(null);
+ }
+
+ public void bulkMessages(List<TridentTuple> tuples) {
+ for (TridentTuple tuple : tuples) {
+ StreamsDatum entry = this.controller.fromTuple(tuple);
+ try {
+ writer.write(entry);
+ } catch (Exception e) {
+ LOGGER.error("Exception writing entry : {}", e, entry);
+ }
+ }
+ LOGGER.debug("******** Ending commit");
+ }
+
+ @Override
+ public void beginCommit(Long aLong) {
+
+ }
+
+ @Override
+ public void commit(Long aLong) {
+
+ }
+
+ public static class Factory implements StateFactory {
+
+ private Logger logger;
+ private StreamsPersistStateController controller;
+
+ public Factory(StreamsPersistWriter writer, StreamsPersistStateController controller) {
+ this.controller = controller;
+ this.logger = LoggerFactory.getLogger(Factory.class);
+ }
+
+ @Override
+ public State makeState(Map map, IMetricsContext iMetricsContext, int i, int i2) {
+ this.logger.debug("Called makeState. . . ");
+ // convert map to config object
+ return new StreamsPersistWriterState(controller);
+ }
+
+ }
+
+ public static class StreamsPersistStateController implements Serializable {
+
+ private String fieldName;
+ private ObjectMapper mapper = new ObjectMapper();
+
+ public StreamsPersistStateController() {
+ this.fieldName = "datum";
+ }
+
+ public StreamsPersistStateController(String fieldName) {
+ this.fieldName = fieldName;
+ }
+
+ public StreamsDatum fromTuple(TridentTuple tuple) {
+ return mapper.convertValue(tuple.getValueByField(this.fieldName), StreamsDatum.class);
+ }
+
+ }
+
+
+
+ public static class StreamsPersistWriterSendMessage extends BaseStateUpdater<StreamsPersistWriterState> {
+
+ private Logger logger = LoggerFactory.getLogger(StreamsPersistWriterSendMessage.class);
+
+ @Override
+ public void updateState(StreamsPersistWriterState writerState, List<TridentTuple> tridentTuples, TridentCollector tridentCollector) {
+ this.logger.debug("**** calling send message. . .");
+ writerState.bulkMessages(tridentTuples);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-storm/src/main/java/org/apache/streams/storm/trident/StreamsProviderSpout.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-storm/src/main/java/org/apache/streams/storm/trident/StreamsProviderSpout.java b/streams-runtimes/streams-runtime-storm/src/main/java/org/apache/streams/storm/trident/StreamsProviderSpout.java
new file mode 100644
index 0000000..8aa04d8
--- /dev/null
+++ b/streams-runtimes/streams-runtime-storm/src/main/java/org/apache/streams/storm/trident/StreamsProviderSpout.java
@@ -0,0 +1,67 @@
+package org.apache.streams.storm.trident;
+
+import backtype.storm.task.TopologyContext;
+import backtype.storm.tuple.Fields;
+import com.google.common.collect.Lists;
+import org.apache.commons.collections4.IteratorUtils;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import storm.trident.operation.TridentCollector;
+import storm.trident.spout.IBatchSpout;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Created by sblackmon on 1/16/14.
+ */
+public class StreamsProviderSpout implements IBatchSpout {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(StreamsProviderSpout.class);
+
+ StreamsProvider provider;
+
+ public StreamsProviderSpout(StreamsProvider provider) {
+ this.provider = provider;
+ }
+
+ @Override
+ public void open(Map map, TopologyContext topologyContext) {
+ provider.prepare(topologyContext);
+ }
+
+ @Override
+ public synchronized void emitBatch(long l, TridentCollector tridentCollector) {
+ List<StreamsDatum> batch;
+ batch = IteratorUtils.toList(provider.readCurrent().iterator());
+ for( StreamsDatum datum : batch ) {
+ tridentCollector.emit( Lists.newArrayList(
+ datum.getTimestamp(),
+ datum.getSequenceid(),
+ datum.getDocument()
+ ));
+ }
+ }
+
+ @Override
+ public void ack(long l) {
+
+ }
+
+ @Override
+ public void close() {
+ provider.cleanUp();
+ }
+
+ @Override
+ public Map getComponentConfiguration() {
+ return null;
+ }
+
+ @Override
+ public Fields getOutputFields() {
+ return new Fields("timestamp", "sequenceid", "datum");
+ }
+};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-storm/src/main/java/org/apache/streams/storm/trident/StreamsTopology.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-storm/src/main/java/org/apache/streams/storm/trident/StreamsTopology.java b/streams-runtimes/streams-runtime-storm/src/main/java/org/apache/streams/storm/trident/StreamsTopology.java
new file mode 100644
index 0000000..63c64a5
--- /dev/null
+++ b/streams-runtimes/streams-runtime-storm/src/main/java/org/apache/streams/storm/trident/StreamsTopology.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.storm.trident;
+
+import backtype.storm.Config;
+import org.apache.streams.StreamsConfiguration;
+import org.apache.streams.config.StreamsConfigurator;
+import storm.trident.TridentTopology;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: sblackmon
+ * Date: 9/20/13
+ * Time: 5:48 PM
+ * To change this template use File | Settings | File Templates.
+ */
+public abstract class StreamsTopology extends TridentTopology {
+
+ StreamsConfiguration configuration;
+ Config stormConfig;
+ String runmode;
+
+ protected StreamsTopology() {
+
+ runmode = StreamsConfigurator.config.getConfig("storm").getString("runmode");
+ stormConfig = new Config();
+
+ }
+
+ protected StreamsTopology(StreamsConfiguration configuration) {
+ this.configuration = configuration;
+ }
+
+ public StreamsConfiguration getConfiguration() {
+ return configuration;
+ }
+
+ public void setConfiguration(StreamsConfiguration configuration) {
+ this.configuration = configuration;
+ }
+
+ public String getRunmode() {
+ return runmode;
+ }
+
+ public void setRunmode(String runmode) {
+ this.runmode = runmode;
+ }
+
+ public Config getStormConfig() {
+ return stormConfig;
+ }
+
+ public void setStormConfig(Config stormConfig) {
+ this.stormConfig = stormConfig;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-storm/pom.xml
----------------------------------------------------------------------
diff --git a/streams-storm/pom.xml b/streams-storm/pom.xml
deleted file mode 100644
index fb8f8a2..0000000
--- a/streams-storm/pom.xml
+++ /dev/null
@@ -1,108 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>streams-project</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
- <artifactId>streams-storm</artifactId>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>0.1-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>0.1-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config-graph</artifactId>
- <version>0.1-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>0.1-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-collections4</artifactId>
- <version>4.0</version>
- </dependency>
- <dependency>
- <groupId>storm</groupId>
- <artifactId>storm-core</artifactId>
- </dependency>
- <dependency>
- <groupId>storm</groupId>
- <artifactId>storm-netty</artifactId>
- </dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <version>2.9.2</version>
- <scope>compile</scope>
- <type>jar</type>
- </dependency>
- <dependency>
- <groupId>com.101tec</groupId>
- <artifactId>zkclient</artifactId>
- <version>0.2</version>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-storm/src/main/java/org/apache/streams/storm/PipelineGraphLauncher.java
----------------------------------------------------------------------
diff --git a/streams-storm/src/main/java/org/apache/streams/storm/PipelineGraphLauncher.java b/streams-storm/src/main/java/org/apache/streams/storm/PipelineGraphLauncher.java
deleted file mode 100644
index a0ad2ea..0000000
--- a/streams-storm/src/main/java/org/apache/streams/storm/PipelineGraphLauncher.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.storm;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.util.RegexUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Created with IntelliJ IDEA.
- * User: sblackmon
- * Date: 9/20/13
- * Time: 11:17 AM
- * To change this template use File | Settings | File Templates.
- */
-public class PipelineGraphLauncher {
-
- private static final Logger log = LoggerFactory.getLogger(PipelineGraphLauncher.class);
-
- private static Config streamsConfiguration;
-
- private static List<Pair<String,Class>> topologies;
-
- private static List<Pair<String,Class>> resolveClasses(List<Pair<String,String>> topologyPairs) throws IOException, ClassNotFoundException {
-
- List<Pair<String,Class>> topologies = new ArrayList<Pair<String,Class>>();
-
- for( Pair<String,String> pair : topologyPairs ) {
- String topologyId = pair.getLeft();
- Class topologyClass = Class.forName(pair.getRight());
- topologies.add(new ImmutablePair(topologyId, topologyClass));
- }
-
- return topologies;
- }
-
- private static List<Pair<String,Class>> loadTopologiesFromPipelineTopologyListFile(File file) throws IOException, ClassNotFoundException {
-
- List<String> lines = IOUtils.readLines(FileUtils.openInputStream(file));
- String pattern = "^([\\w-]*)[\\s]*?([\\w.]*)$";
-
- List<Pair<String,String>> topologyPairs = RegexUtils.getTwoMatchedGroupsList(lines, pattern);
-
- topologies = resolveClasses(topologyPairs);
-
- for( Pair<String,String> pair : topologyPairs ) {
- String topologyId = pair.getLeft();
- Class topologyClass = Class.forName(pair.getRight());
- topologies.add(new ImmutablePair(topologyId, topologyClass));
- }
-
- return topologies;
- }
-
- private static List<Pair<String,Class>> loadTopologiesFromPipelineGraphFile(File file) throws IOException, ClassNotFoundException {
-
- List<String> lines = IOUtils.readLines(FileUtils.openInputStream(file));
- String pattern = "$([\\w-]*)\\s([\\w.)";
-
- List<Pair<String,String>> topologyPairs = RegexUtils.getTwoMatchedGroupsList(lines, pattern);
-
- topologies = resolveClasses(topologyPairs);
-
- return topologies;
- }
-
- public static boolean isLocal(String[] args) {
- if(args.length >= 1 && args[1].equals("deploy"))
- return false;
- else return true;
- }
-
- public static void main(String[] args) throws Exception {
-
- if(args.length < 3) {
- log.error("Not enough arguments");
- log.error(" storm {local,deploy} <pipelinedef>");
- return;
- }
- if(!args[1].equals("deploy")) {
- log.error("Not a deploy");
- log.error(" storm {local,deploy} <pipelinedef>");
- return;
- }
-
- ObjectMapper mapper = new ObjectMapper();
-
- URL configFileUrl = PipelineGraphLauncher.class.getResource(args[0]);
-
- File configFile;
- try {
- configFile = new File(configFileUrl.toURI());
- } catch(URISyntaxException e) {
- configFile = new File(configFileUrl.getPath());
- }
-
- streamsConfiguration = StreamsConfigurator.config;
-
-// String pipelineIdentifier = streamsConfiguration.getPipeline();
-//
-// for( Map.Entry<String, Object> moduleConfig : streamsConfiguration.getAdditionalProperties().entrySet()) {
-//
-// }
-
-// each defined graph becomes a topology
-
-//
-// URL pipelineFileUrl = PipelineGraphLauncher.class.getResource(args[1]);
-//
-// File pipelineFile;
-// try {
-// pipelineFile = new File(pipelineFileUrl.toURI());
-// } catch(URISyntaxException e) {
-// pipelineFile = new File(pipelineFileUrl.getPath());
-// }
-//
-// topologies = loadTopologiesFromPipelineTopologyListFile(pipelineFile);
-//
-// for( Pair<String,Class> topology : topologies ) {
-// Class topologyClass = topology.getRight();
-//
-// try {
-// Constructor ctor = topologyClass.getDeclaredConstructor(
-// String.class,
-// StreamsConfiguration.class);
-// ctor.setAccessible(true);
-// Object topologyObject = ctor.newInstance(pipelineIdentifier, streamsConfiguration);
-// Runnable runnable = (Runnable) topologyObject;
-// runnable.run();
-// } catch (InstantiationException x) {
-// log.warn(x.getMessage());
-// x.printStackTrace();
-// } catch (IllegalAccessException x) {
-// log.warn(x.getMessage());
-// x.printStackTrace();
-// } catch (InvocationTargetException x) {
-// log.warn(x.getMessage());
-// x.printStackTrace();
-// } catch (NoSuchMethodException x) {
-// log.warn(x.getMessage());
-// x.printStackTrace();
-//
-// try {
-// Constructor ctor = topologyClass.getDeclaredConstructor(
-// String[].class);
-// ctor.setAccessible(true);
-// Object topologyObject = ctor.newInstance(args);
-// Method main = topologyClass.getMethod("main", String[].class);
-// main.invoke(topologyObject, args);
-// } catch (InstantiationException x2) {
-// log.warn(x2.getMessage());
-// x.printStackTrace();
-// } catch (IllegalAccessException x2) {
-// log.warn(x2.getMessage());
-// x.printStackTrace();
-// } catch (InvocationTargetException x2) {
-// log.warn(x2.getMessage());
-// x.printStackTrace();
-// } catch (NoSuchMethodException x2) {
-// log.error(x2.getMessage());
-// x.printStackTrace();
-// }
-// }
-// }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-storm/src/main/java/org/apache/streams/storm/trident/StreamsPersistWriterState.java
----------------------------------------------------------------------
diff --git a/streams-storm/src/main/java/org/apache/streams/storm/trident/StreamsPersistWriterState.java b/streams-storm/src/main/java/org/apache/streams/storm/trident/StreamsPersistWriterState.java
deleted file mode 100644
index dd360d6..0000000
--- a/streams-storm/src/main/java/org/apache/streams/storm/trident/StreamsPersistWriterState.java
+++ /dev/null
@@ -1,107 +0,0 @@
-package org.apache.streams.storm.trident;
-
-import backtype.storm.task.IMetricsContext;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import storm.trident.operation.TridentCollector;
-import storm.trident.state.BaseStateUpdater;
-import storm.trident.state.State;
-import storm.trident.state.StateFactory;
-import storm.trident.tuple.TridentTuple;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-
-/**
- * Created by sblackmon on 1/16/14.
- */
-public class StreamsPersistWriterState implements State {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(StreamsPersistWriterState.class);
-
- StreamsPersistWriter writer;
- StreamsPersistStateController controller;
-
- public StreamsPersistWriterState(StreamsPersistStateController controller) {
- this.controller = new StreamsPersistStateController();
- writer.start();
- }
-
- public void bulkMessages(List<TridentTuple> tuples) {
- for (TridentTuple tuple : tuples) {
- StreamsDatum entry = this.controller.fromTuple(tuple);
- try {
- writer.write(entry);
- } catch (Exception e) {
- LOGGER.error("Exception writing entry : {}", e, entry);
- }
- }
- LOGGER.debug("******** Ending commit");
- }
-
- @Override
- public void beginCommit(Long aLong) {
-
- }
-
- @Override
- public void commit(Long aLong) {
-
- }
-
- public static class Factory implements StateFactory {
-
- private Logger logger;
- private StreamsPersistStateController controller;
-
- public Factory(StreamsPersistWriter writer, StreamsPersistStateController controller) {
- this.controller = controller;
- this.logger = LoggerFactory.getLogger(Factory.class);
- }
-
- @Override
- public State makeState(Map map, IMetricsContext iMetricsContext, int i, int i2) {
- this.logger.debug("Called makeState. . . ");
- // convert map to config object
- return new StreamsPersistWriterState(controller);
- }
-
- }
-
- public static class StreamsPersistStateController implements Serializable {
-
- private String fieldName;
- private ObjectMapper mapper = new ObjectMapper();
-
- public StreamsPersistStateController() {
- this.fieldName = "datum";
- }
-
- public StreamsPersistStateController(String fieldName) {
- this.fieldName = fieldName;
- }
-
- public StreamsDatum fromTuple(TridentTuple tuple) {
- return mapper.convertValue(tuple.getValueByField(this.fieldName), StreamsDatum.class);
- }
-
- }
-
-
-
- public static class StreamsPersistWriterSendMessage extends BaseStateUpdater<StreamsPersistWriterState> {
-
- private Logger logger = LoggerFactory.getLogger(StreamsPersistWriterSendMessage.class);
-
- @Override
- public void updateState(StreamsPersistWriterState writerState, List<TridentTuple> tridentTuples, TridentCollector tridentCollector) {
- this.logger.debug("**** calling send message. . .");
- writerState.bulkMessages(tridentTuples);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-storm/src/main/java/org/apache/streams/storm/trident/StreamsProviderSpout.java
----------------------------------------------------------------------
diff --git a/streams-storm/src/main/java/org/apache/streams/storm/trident/StreamsProviderSpout.java b/streams-storm/src/main/java/org/apache/streams/storm/trident/StreamsProviderSpout.java
deleted file mode 100644
index ad7adc4..0000000
--- a/streams-storm/src/main/java/org/apache/streams/storm/trident/StreamsProviderSpout.java
+++ /dev/null
@@ -1,69 +0,0 @@
-package org.apache.streams.storm.trident;
-
-import backtype.storm.task.TopologyContext;
-import backtype.storm.tuple.Fields;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Lists;
-import org.apache.commons.collections4.IteratorUtils;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import storm.trident.operation.TridentCollector;
-import storm.trident.spout.IBatchSpout;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Created by sblackmon on 1/16/14.
- */
-public class StreamsProviderSpout implements IBatchSpout {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(StreamsProviderSpout.class);
-
- StreamsProvider provider;
-
- public StreamsProviderSpout(StreamsProvider provider) {
- this.provider = provider;
- }
-
- @Override
- public void open(Map map, TopologyContext topologyContext) {
- provider.start();
- }
-
- @Override
- public synchronized void emitBatch(long l, TridentCollector tridentCollector) {
- List<StreamsDatum> batch;
- batch = IteratorUtils.toList(provider.getProviderQueue().iterator());
- for( StreamsDatum datum : batch ) {
- tridentCollector.emit( Lists.newArrayList(
- datum.getTimestamp(),
- datum.getSequenceid(),
- datum.getDocument()
- ));
- }
- }
-
- @Override
- public void ack(long l) {
-
- }
-
- @Override
- public void close() {
- provider.stop();
- }
-
- @Override
- public Map getComponentConfiguration() {
- return null;
- }
-
- @Override
- public Fields getOutputFields() {
- return new Fields("timestamp", "sequenceid", "datum");
- }
-};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-storm/src/main/java/org/apache/streams/storm/trident/StreamsTopology.java
----------------------------------------------------------------------
diff --git a/streams-storm/src/main/java/org/apache/streams/storm/trident/StreamsTopology.java b/streams-storm/src/main/java/org/apache/streams/storm/trident/StreamsTopology.java
deleted file mode 100644
index 63c64a5..0000000
--- a/streams-storm/src/main/java/org/apache/streams/storm/trident/StreamsTopology.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.storm.trident;
-
-import backtype.storm.Config;
-import org.apache.streams.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import storm.trident.TridentTopology;
-
-/**
- * Created with IntelliJ IDEA.
- * User: sblackmon
- * Date: 9/20/13
- * Time: 5:48 PM
- * To change this template use File | Settings | File Templates.
- */
-public abstract class StreamsTopology extends TridentTopology {
-
- StreamsConfiguration configuration;
- Config stormConfig;
- String runmode;
-
- protected StreamsTopology() {
-
- runmode = StreamsConfigurator.config.getConfig("storm").getString("runmode");
- stormConfig = new Config();
-
- }
-
- protected StreamsTopology(StreamsConfiguration configuration) {
- this.configuration = configuration;
- }
-
- public StreamsConfiguration getConfiguration() {
- return configuration;
- }
-
- public void setConfiguration(StreamsConfiguration configuration) {
- this.configuration = configuration;
- }
-
- public String getRunmode() {
- return runmode;
- }
-
- public void setRunmode(String runmode) {
- this.runmode = runmode;
- }
-
- public Config getStormConfig() {
- return stormConfig;
- }
-
- public void setStormConfig(Config stormConfig) {
- this.stormConfig = stormConfig;
- }
-}
[06/13] adding uncommitted modules
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/89ff615e/streams-core/src/main/java/org/apache/streams/core/DatumStatus.java
----------------------------------------------------------------------
diff --git a/streams-core/src/main/java/org/apache/streams/core/DatumStatus.java b/streams-core/src/main/java/org/apache/streams/core/DatumStatus.java
new file mode 100644
index 0000000..be18707
--- /dev/null
+++ b/streams-core/src/main/java/org/apache/streams/core/DatumStatus.java
@@ -0,0 +1,7 @@
+package org.apache.streams.core;
+
+public enum DatumStatus {
+ SUCCESS,
+ PARTIAL,
+ FAIL
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/89ff615e/streams-core/src/main/java/org/apache/streams/core/DatumStatusCounter.java
----------------------------------------------------------------------
diff --git a/streams-core/src/main/java/org/apache/streams/core/DatumStatusCounter.java b/streams-core/src/main/java/org/apache/streams/core/DatumStatusCounter.java
new file mode 100644
index 0000000..7798fcd
--- /dev/null
+++ b/streams-core/src/main/java/org/apache/streams/core/DatumStatusCounter.java
@@ -0,0 +1,41 @@
+package org.apache.streams.core;
+
+public class DatumStatusCounter
+{
+ private volatile int success = 0;
+ private volatile int fail = 0;
+ private volatile int partial = 0;
+ private volatile int recordsEmitted = 0;
+
+ public int getSuccess() { return this.success; }
+ public int getFail() { return this.fail; }
+ public int getPartial() { return this.partial; }
+ public int getEmitted() { return this.recordsEmitted; }
+
+ public void add(DatumStatusCounter datumStatusCounter) {
+ this.success += datumStatusCounter.getSuccess();
+ this.partial = datumStatusCounter.getPartial();
+ this.fail += datumStatusCounter.getFail();
+ this.recordsEmitted += datumStatusCounter.getEmitted();
+ }
+
+ public void add(DatumStatus workStatus) {
+ // add this to the record counter
+ switch(workStatus) {
+ case SUCCESS: this.success++; break;
+ case PARTIAL: this.partial++; break;
+ case FAIL: this.fail++; break;
+ }
+ this.recordsEmitted += 1;
+ }
+
+ @Override
+ public String toString() {
+ return "DatumStatusCounter{" +
+ "success=" + success +
+ ", fail=" + fail +
+ ", partial=" + partial +
+ ", recordsEmitted=" + recordsEmitted +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/89ff615e/streams-runtimes/pom.xml
----------------------------------------------------------------------
diff --git a/streams-runtimes/pom.xml b/streams-runtimes/pom.xml
new file mode 100644
index 0000000..cf0d57f
--- /dev/null
+++ b/streams-runtimes/pom.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-project</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>streams-runtimes</artifactId>
+
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>streams-runtime-local</module>
+ <module>streams-runtime-storm</module>
+ </modules>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/89ff615e/streams-runtimes/streams-runtime-local/pom.xml
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/pom.xml b/streams-runtimes/streams-runtime-local/pom.xml
new file mode 100644
index 0000000..fa64225
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/pom.xml
@@ -0,0 +1,103 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-runtimes</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>streams-runtime-local</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-util</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-sources/jsonschema2pojo</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-maven-plugin</artifactId>
+ <configuration>
+ <addCompileSourceRoot>true</addCompileSourceRoot>
+ <generateBuilders>true</generateBuilders>
+ <sourcePaths>
+ <sourcePath>src/main/jsonschema/LocalRuntimeConfiguration.json</sourcePath>
+ </sourcePaths>
+ <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
+ <targetPackage>org.apache.streams.local</targetPackage>
+ <useLongIntegers>true</useLongIntegers>
+ <useJodaDates>false</useJodaDates>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/89ff615e/streams-runtimes/streams-runtime-local/src/main/jsonschema/LocalRuntimeConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/jsonschema/LocalRuntimeConfiguration.json b/streams-runtimes/streams-runtime-local/src/main/jsonschema/LocalRuntimeConfiguration.json
new file mode 100644
index 0000000..3432318
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/main/jsonschema/LocalRuntimeConfiguration.json
@@ -0,0 +1,12 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType" : "org.apache.streams.local.LocalRuntimeConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "providerTimeoutMs": {
+ "type" : "integer"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/89ff615e/streams-runtimes/streams-runtime-local/src/test/resources/TestFile.txt
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/resources/TestFile.txt b/streams-runtimes/streams-runtime-local/src/test/resources/TestFile.txt
new file mode 100644
index 0000000..9384ee5
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/test/resources/TestFile.txt
@@ -0,0 +1,4 @@
+Document1
+Document2
+Document3
+Document4
\ No newline at end of file
[05/13] git commit: adding some uncommitted modules reorganizing pom
improvements
Posted by sb...@apache.org.
adding some uncommitted modules
reorganizing
pom improvements
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/1993313b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/1993313b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/1993313b
Branch: refs/heads/springcleaning
Commit: 1993313bf6b2f449fc67613109e9084884ea5931
Parents: d72d6b4
Author: Steve Blackmon <sb...@w2odigital.com>
Authored: Sun Mar 23 18:26:31 2014 -0500
Committer: Steve Blackmon <sb...@w2odigital.com>
Committed: Sun Mar 23 18:26:31 2014 -0500
----------------------------------------------------------------------
pom.xml | 6 +-
streams-contrib/pom.xml | 2 +-
streams-contrib/streams-components-test/pom.xml | 46 ---
.../component/ExpectedDatumsPersistWriter.java | 54 ----
.../test/component/FileReaderProvider.java | 98 -------
.../test/component/StreamsDatumConverter.java | 13 -
.../component/StringToDocumentConverter.java | 15 -
.../tests/TestCompoentsLocalStream.java | 23 --
.../tests/TestExpectedDatumsPersitWriter.java | 38 ---
.../component/tests/TestFileReaderProvider.java | 39 ---
.../src/test/resources/TestFile.txt | 4 -
.../streams/console/ConsolePersistWriter.java | 1 -
.../ElasticsearchPersistWriter.java | 9 +-
streams-contrib/streams-persist-hdfs/pom.xml | 15 +-
.../streams/hdfs/WebHdfsPersistReader.java | 2 -
.../streams/hdfs/WebHdfsPersistReaderTask.java | 9 +-
.../streams-provider-datasift/pom.xml | 34 +++
.../main/jsonschema/com/datasift/Datasift.json | 192 ++++++-------
.../com/datasift/DatasiftConfiguration.json | 4 +-
.../com/datasift/test/DatasiftSerDeTest.java | 5 +-
.../src/test/resources/datasift_jsons.txt | 101 -------
.../streams/rss/test/Top100FeedsTest.java | 1 -
.../org/apache/streams/core/StreamBuilder.java | 104 +++++++
.../apache/streams/core/StreamsResultSet.java | 25 +-
.../core/builders/InvalidStreamException.java | 23 --
.../core/builders/LocalStreamBuilder.java | 265 -----------------
.../streams/core/builders/StreamBuilder.java | 106 -------
.../streams/core/builders/StreamComponent.java | 229 ---------------
.../streams/core/tasks/BaseStreamsTask.java | 148 ----------
.../core/tasks/LocalStreamMonitorThread.java | 69 -----
.../streams/core/tasks/StreamsMergeTask.java | 58 ----
.../core/tasks/StreamsPersistWriterTask.java | 103 -------
.../core/tasks/StreamsProcessorTask.java | 102 -------
.../streams/core/tasks/StreamsProviderTask.java | 153 ----------
.../apache/streams/core/tasks/StreamsTask.java | 58 ----
.../core/builders/LocalStreamBuilderTest.java | 161 -----------
.../core/builders/ToyLocalBuilderExample.java | 28 --
.../streams/core/tasks/BasicTasksTest.java | 285 -------------------
.../test/processors/DoNothingProcessor.java | 32 ---
.../PassthroughDatumCounterProcessor.java | 49 ----
.../test/providers/NumericMessageProvider.java | 104 -------
.../core/test/writer/DatumCounterWriter.java | 31 --
.../core/test/writer/DoNothingWriter.java | 24 --
.../core/test/writer/SystemOutWriter.java | 24 --
.../local/builders/InvalidStreamException.java | 23 ++
.../local/builders/LocalStreamBuilder.java | 276 ++++++++++++++++++
.../streams/local/builders/StreamComponent.java | 229 +++++++++++++++
.../streams/local/tasks/BaseStreamsTask.java | 148 ++++++++++
.../tasks/LocalStreamProcessMonitorThread.java | 69 +++++
.../streams/local/tasks/StreamsMergeTask.java | 58 ++++
.../local/tasks/StreamsPersistWriterTask.java | 103 +++++++
.../local/tasks/StreamsProcessorTask.java | 101 +++++++
.../local/tasks/StreamsProviderTask.java | 175 ++++++++++++
.../apache/streams/local/tasks/StreamsTask.java | 58 ++++
.../local/builders/LocalStreamBuilderTest.java | 162 +++++++++++
.../local/builders/ToyLocalBuilderExample.java | 28 ++
.../streams/local/tasks/BasicTasksTest.java | 285 +++++++++++++++++++
.../test/processors/DoNothingProcessor.java | 32 +++
.../PassthroughDatumCounterProcessor.java | 49 ++++
.../test/providers/NumericMessageProvider.java | 102 +++++++
.../local/test/writer/DatumCounterWriter.java | 31 ++
.../local/test/writer/DoNothingWriter.java | 24 ++
.../local/test/writer/SystemOutWriter.java | 24 ++
.../component/ExpectedDatumsPersistWriter.java | 54 ++++
.../test/component/FileReaderProvider.java | 98 +++++++
.../test/component/StreamsDatumConverter.java | 13 +
.../component/StringToDocumentConverter.java | 15 +
.../tests/TestComponentsLocalStream.java | 23 ++
.../tests/TestExpectedDatumsPersitWriter.java | 38 +++
.../component/tests/TestFileReaderProvider.java | 39 +++
streams-runtimes/streams-runtime-storm/pom.xml | 108 +++++++
.../streams/storm/PipelineGraphLauncher.java | 196 +++++++++++++
.../trident/StreamsPersistWriterState.java | 106 +++++++
.../storm/trident/StreamsProviderSpout.java | 67 +++++
.../streams/storm/trident/StreamsTopology.java | 73 +++++
streams-storm/pom.xml | 108 -------
.../streams/storm/PipelineGraphLauncher.java | 196 -------------
.../trident/StreamsPersistWriterState.java | 107 -------
.../storm/trident/StreamsProviderSpout.java | 69 -----
.../streams/storm/trident/StreamsTopology.java | 73 -----
80 files changed, 3093 insertions(+), 3159 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index fd39720..c552cf1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -94,11 +94,11 @@
<module>streams-config</module>
<module>streams-config-graph</module>
<module>streams-pojo</module>
- <!--<module>streams-storm</module>-->
<module>streams-util</module>
<module>streams-pojo-extensions</module>
<module>streams-contrib</module>
<module>streams-core</module>
+ <module>streams-runtimes</module>
</modules>
<packaging>pom</packaging>
@@ -154,10 +154,6 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
- <dependency>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-classic</artifactId>
- </dependency>
</dependencies>
<dependencyManagement>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/pom.xml b/streams-contrib/pom.xml
index fc08b4e..a5ed004 100644
--- a/streams-contrib/pom.xml
+++ b/streams-contrib/pom.xml
@@ -38,6 +38,7 @@
<modules>
<module>streams-persist-console</module>
<module>streams-persist-elasticsearch</module>
+ <module>streams-persist-hbase</module>
<module>streams-persist-hdfs</module>
<module>streams-persist-kafka</module>
<module>streams-persist-mongo</module>
@@ -49,7 +50,6 @@
<module>streams-provider-twitter</module>
<module>streams-provider-sysomos</module>
<module>streams-provider-rss</module>
- <module>streams-components-test</module>
</modules>
<dependencyManagement>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-components-test/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-components-test/pom.xml b/streams-contrib/streams-components-test/pom.xml
deleted file mode 100644
index 88973ba..0000000
--- a/streams-contrib/streams-components-test/pom.xml
+++ /dev/null
@@ -1,46 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>streams-contrib</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.1-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-components-test</artifactId>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>0.1-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- </dependencies>
-
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <version>2.4</version>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
-
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/ExpectedDatumsPersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/ExpectedDatumsPersistWriter.java b/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/ExpectedDatumsPersistWriter.java
deleted file mode 100644
index 675edd7..0000000
--- a/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/ExpectedDatumsPersistWriter.java
+++ /dev/null
@@ -1,54 +0,0 @@
-package org.apache.streams.test.component;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import static org.junit.Assert.*;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.InputStream;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Scanner;
-
-/**
- * Created by rebanks on 2/27/14.
- */
-public class ExpectedDatumsPersistWriter implements StreamsPersistWriter{
-
- private StreamsDatumConverter converter;
- private String fileName;
- private List<StreamsDatum> expectedDatums;
- private int counted = 0;
- private int expectedSize = 0;
-
- public ExpectedDatumsPersistWriter(StreamsDatumConverter converter, String filePathInResources) {
- this.converter = converter;
- this.fileName = filePathInResources;
- }
-
-
-
- @Override
- public void write(StreamsDatum entry) {
- int index = this.expectedDatums.indexOf(entry);
- assertNotEquals("Datum not expected. "+entry.toString(), -1, index);
- this.expectedDatums.remove(index);
- ++this.counted;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- Scanner scanner = new Scanner(ExpectedDatumsPersistWriter.class.getResourceAsStream(this.fileName));
- this.expectedDatums = new LinkedList<StreamsDatum>();
- while(scanner.hasNextLine()) {
- this.expectedDatums.add(this.converter.convert(scanner.nextLine()));
- }
- this.expectedSize = this.expectedDatums.size();
- }
-
- @Override
- public void cleanUp() {
- assertEquals("Did not received the expected number of StreamsDatums", this.expectedSize, this.counted);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/FileReaderProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/FileReaderProvider.java b/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/FileReaderProvider.java
deleted file mode 100644
index a7c98c9..0000000
--- a/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/FileReaderProvider.java
+++ /dev/null
@@ -1,98 +0,0 @@
-package org.apache.streams.test.component;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.joda.time.DateTime;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.InputStream;
-import java.math.BigInteger;
-import java.sql.ResultSet;
-import java.util.Iterator;
-import java.util.Queue;
-import java.util.Scanner;
-
-/**
- * FOR TESTING PURPOSES ONLY.
- *
- * The StreamProvider reads from a File or InputStream. Each line of the file will be emitted as the document of a
- * streams datum.
- *
- */
-public class FileReaderProvider implements StreamsProvider {
-
- private String fileName;
- private InputStream inStream;
- private Scanner scanner;
- private StreamsDatumConverter converter;
-
- public FileReaderProvider(String filePathInResources, StreamsDatumConverter converter) {
- this.fileName = filePathInResources;
- this.converter = converter;
- }
-
- @Override
- public void startStream() {
-
- }
-
- @Override
- public StreamsResultSet readCurrent() {
- return new ResultSet();
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void prepare(Object configurationObject) {
- this.scanner = new Scanner(FileReaderProvider.class.getResourceAsStream(this.fileName));
- }
-
- @Override
- public void cleanUp() {
- this.scanner.close();
- }
-
- private class ResultSet extends StreamsResultSet {
-
- public ResultSet() {
- super(null);
- }
-
-
- @Override
- public Iterator<StreamsDatum> iterator() {
- return new FileProviderIterator();
- }
-
- private class FileProviderIterator implements Iterator<StreamsDatum> {
-
-
-
- @Override
- public boolean hasNext() {
- return scanner.hasNextLine();
- }
-
- @Override
- public StreamsDatum next() {
- return converter.convert(scanner.nextLine());
- }
-
- @Override
- public void remove() {
-
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/StreamsDatumConverter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/StreamsDatumConverter.java b/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/StreamsDatumConverter.java
deleted file mode 100644
index d6ec2f4..0000000
--- a/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/StreamsDatumConverter.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package org.apache.streams.test.component;
-
-import org.apache.streams.core.StreamsDatum;
-
-import java.io.Serializable;
-
-/**
- * Created by rebanks on 2/27/14.
- */
-public interface StreamsDatumConverter extends Serializable {
-
- public StreamsDatum convert(String s);
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/StringToDocumentConverter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/StringToDocumentConverter.java b/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/StringToDocumentConverter.java
deleted file mode 100644
index d97de91..0000000
--- a/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/StringToDocumentConverter.java
+++ /dev/null
@@ -1,15 +0,0 @@
-package org.apache.streams.test.component;
-
-import org.apache.streams.core.StreamsDatum;
-
-/**
- * Created by rebanks on 2/28/14.
- */
-public class StringToDocumentConverter implements StreamsDatumConverter {
-
- @Override
- public StreamsDatum convert(String s) {
- return new StreamsDatum(s);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/tests/TestCompoentsLocalStream.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/tests/TestCompoentsLocalStream.java b/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/tests/TestCompoentsLocalStream.java
deleted file mode 100644
index 23a804c..0000000
--- a/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/tests/TestCompoentsLocalStream.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package org.apache.streams.test.component.tests;
-
-import org.apache.streams.core.builders.LocalStreamBuilder;
-import org.apache.streams.test.component.ExpectedDatumsPersistWriter;
-import org.apache.streams.test.component.FileReaderProvider;
-import org.apache.streams.test.component.StringToDocumentConverter;
-import org.junit.Test;
-
-/**
- * Created by rebanks on 2/28/14.
- */
-public class TestCompoentsLocalStream {
-
- @Test
- public void testLocalStreamWithComponent() {
- LocalStreamBuilder builder = new LocalStreamBuilder();
- builder.newReadCurrentStream("provider", new FileReaderProvider("/TestFile.txt",
- new StringToDocumentConverter()));
- builder.addStreamsPersistWriter("writer", new ExpectedDatumsPersistWriter(new StringToDocumentConverter(),
- "/TestFile.txt"), 1, "provider")
- .start();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/tests/TestExpectedDatumsPersitWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/tests/TestExpectedDatumsPersitWriter.java b/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/tests/TestExpectedDatumsPersitWriter.java
deleted file mode 100644
index d12b250..0000000
--- a/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/tests/TestExpectedDatumsPersitWriter.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package org.apache.streams.test.component.tests;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.test.component.ExpectedDatumsPersistWriter;
-import org.apache.streams.test.component.StringToDocumentConverter;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * Created by rebanks on 2/28/14.
- */
-public class TestExpectedDatumsPersitWriter {
-
- private static final StreamsDatum[] INPUT_DATUMS = new StreamsDatum[] {
- new StreamsDatum("Document1"),
- new StreamsDatum("Document2"),
- new StreamsDatum("Document3"),
- new StreamsDatum("Document4")
-// Uncomment to prove failures occur, or comment out a datum above
-// ,new StreamsDatum("Document5")
- };
-
- @Test
- public void testExpectedDatumsPersistWriterFileName() {
- testDatums(new ExpectedDatumsPersistWriter(new StringToDocumentConverter(), "/TestFile.txt"));
- }
-
-
-
- private void testDatums(ExpectedDatumsPersistWriter writer) {
- writer.prepare(null);
- for(StreamsDatum datum : INPUT_DATUMS) {
- writer.write(datum);
- }
- writer.cleanUp();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/tests/TestFileReaderProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/tests/TestFileReaderProvider.java b/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/tests/TestFileReaderProvider.java
deleted file mode 100644
index c562c20..0000000
--- a/streams-contrib/streams-components-test/src/test/java/org/apache/streams/test/component/tests/TestFileReaderProvider.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package org.apache.streams.test.component.tests;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.test.component.FileReaderProvider;
-import org.apache.streams.test.component.StringToDocumentConverter;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.io.InputStream;
-
-import static org.junit.Assert.*;
-
-/**
- * Created by rebanks on 2/28/14.
- */
-public class TestFileReaderProvider {
-
-
- @Test
- public void testFileReaderProviderFileName() {
- String fileName = "/TestFile.txt";
- FileReaderProvider provider = new FileReaderProvider(fileName, new StringToDocumentConverter());
- provider.prepare(null);
- StreamsResultSet resultSet = provider.readCurrent();
- int count = 0;
- for(StreamsDatum datum : resultSet) {
- ++count;
- }
- assertEquals(4, count);
- provider.cleanUp();
- }
-
-
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-components-test/src/test/resources/TestFile.txt
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-components-test/src/test/resources/TestFile.txt b/streams-contrib/streams-components-test/src/test/resources/TestFile.txt
deleted file mode 100644
index 9384ee5..0000000
--- a/streams-contrib/streams-components-test/src/test/resources/TestFile.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-Document1
-Document2
-Document3
-Document4
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java b/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java
index b76cc0b..e8efc6d 100644
--- a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java
+++ b/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java
@@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.core.tasks.StreamsPersistWriterTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
index 595011b..250e15a 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
@@ -1,6 +1,5 @@
package org.apache.streams.elasticsearch;
-import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Objects;
import com.google.common.base.Optional;
@@ -9,8 +8,6 @@ import com.typesafe.config.Config;
import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.core.tasks.StreamsPersistWriterTask;
-import org.apache.streams.pojo.json.Activity;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
@@ -32,11 +29,13 @@ import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.*;
+import java.io.Closeable;
+import java.io.Flushable;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.*;
-import java.util.concurrent.ConcurrentLinkedQueue;
public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushable, Closeable
{
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-persist-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-hdfs/pom.xml b/streams-contrib/streams-persist-hdfs/pom.xml
index 40e73a5..e1d2d36 100644
--- a/streams-contrib/streams-persist-hdfs/pom.xml
+++ b/streams-contrib/streams-persist-hdfs/pom.xml
@@ -11,26 +11,33 @@
<artifactId>streams-persist-hdfs</artifactId>
+ <repositories>
+ <repository>
+ <id>cloudera</id>
+ <url>https://repository.cloudera.com/artifactory/repo</url>
+ </repository>
+ </repositories>
+
<dependencies>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-config</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-core</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-pojo</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-util</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReader.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReader.java b/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReader.java
index e01d9d4..b0d9904 100644
--- a/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReader.java
+++ b/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReader.java
@@ -154,8 +154,6 @@ public class WebHdfsPersistReader implements StreamsPersistReader {
@Override
public StreamsResultSet readCurrent() {
- LOGGER.debug("readCurrent: {}", persistQueue.size());
-
Collection<StreamsDatum> currentIterator = Lists.newArrayList();
Iterators.addAll(currentIterator, persistQueue.iterator());
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReaderTask.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReaderTask.java b/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReaderTask.java
index dc6ea16..95a8ef6 100644
--- a/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReaderTask.java
+++ b/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReaderTask.java
@@ -26,7 +26,7 @@ public class WebHdfsPersistReaderTask implements Runnable {
BufferedReader bufferedReader;
LOGGER.info("Found " + fileStatus.getPath().getName());
if( fileStatus.isFile() && !fileStatus.getPath().getName().startsWith("_")) {
- LOGGER.info("Processing " + fileStatus.getPath().getName());
+ LOGGER.info("Started Processing " + fileStatus.getPath().getName());
try {
bufferedReader = new BufferedReader(new InputStreamReader(reader.client.open(fileStatus.getPath())));
} catch (Exception e) {
@@ -56,7 +56,12 @@ public class WebHdfsPersistReaderTask implements Runnable {
}
} while( !Strings.isNullOrEmpty(line) );
LOGGER.info("Finished Processing " + fileStatus.getPath().getName());
-
+ try {
+ bufferedReader.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.error(e.getMessage());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-provider-datasift/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/pom.xml b/streams-contrib/streams-provider-datasift/pom.xml
index b49db93..3e84adc 100644
--- a/streams-contrib/streams-provider-datasift/pom.xml
+++ b/streams-contrib/streams-provider-datasift/pom.xml
@@ -11,8 +11,41 @@
<artifactId>streams-provider-datasift</artifactId>
+ <repositories>
+ <repository>
+ <id>boundary-site</id>
+ <url>http://maven.boundary.com/artifactory/repo</url>
+ </repository>
+ </repositories>
+
<dependencies>
<dependency>
+ <groupId>com.typesafe</groupId>
+ <artifactId>config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-pojo</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-provider-twitter</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.datasift.client</groupId>
+ <artifactId>datasift-java</artifactId>
+ <version>3.0.0-Beta2</version>
+ </dependency>
+ <dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</dependency>
@@ -32,6 +65,7 @@
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
+
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
index d6c973d..87b1ac3 100644
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
+++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
@@ -1,16 +1,16 @@
{
"type": "object",
"$schema": "http://json-schema.org/draft-03/schema",
- "javaType": "com.datasift.Datasift",
+ "javaType": "org.apache.streams.datasift.Datasift",
"properties": {
"blog": {
- "javaType": "com.datasift.blog.Blog",
+ "javaType": "org.apache.streams.datasift.blog.Blog",
"type": "object",
"dynamic": "true",
"properties": {
"author": {
"type": "object",
- "javaType": "com.datasift.blog.BlogAuthor",
+ "javaType": "org.apache.streams.datasift.blog.BlogAuthor",
"dynamic": "true",
"properties": {
"link": {
@@ -56,7 +56,7 @@
}
},
"facebook": {
- "javaType": "com.datasift.config.Facebook",
+ "javaType": "org.apache.streams.datasift.config.Facebook",
"type": "object",
"dynamic": "true",
"properties": {
@@ -64,7 +64,7 @@
"type": "string"
},
"author": {
- "javaType": "com.datasift.config.Author",
+ "javaType": "org.apache.streams.datasift.config.Author",
"type": "object",
"dynamic": "true",
"properties": {
@@ -89,12 +89,12 @@
"type": "string"
},
"likes": {
- "javaType": "com.datasift.config.Likes",
+ "javaType": "org.apache.streams.datasift.config.Likes",
"type": "object",
"dynamic": "true",
"properties": {
"count": {
- "type": "long"
+ "type": "integer"
},
"ids": {
"type": "string"
@@ -111,7 +111,7 @@
"type": "string"
},
"to": {
- "javaType": "com.datasift.config.To",
+ "javaType": "org.apache.streams.datasift.config.To",
"type": "object",
"dynamic": "true",
"properties": {
@@ -139,12 +139,12 @@
}
},
"interaction": {
- "javaType": "com.datasift.interaction.Interaction",
+ "javaType": "org.apache.streams.datasift.interaction.Interaction",
"type": "object",
"dynamic": "true",
"properties": {
"author": {
- "javaType": "com.datasift.interaction.Author",
+ "javaType": "org.apache.streams.datasift.interaction.Author",
"type": "object",
"dynamic": "true",
"properties": {
@@ -152,7 +152,7 @@
"type": "string"
},
"id": {
- "type": "long"
+ "type": "integer"
},
"link": {
"type": "string"
@@ -175,15 +175,15 @@
"type": "string"
},
"geo": {
- "javaType": "com.datasift.interaction.Geo",
+ "javaType": "org.apache.streams.datasift.interaction.Geo",
"type": "object",
"dynamic": "true",
"properties": {
"latitude": {
- "type": "double"
+ "type": "number"
},
"longitude": {
- "type": "double"
+ "type": "number"
}
}
},
@@ -197,7 +197,7 @@
"dynamic": "true",
"properties": {
"version": {
- "type": "long"
+ "type": "integer"
}
}
},
@@ -221,22 +221,22 @@
}
},
"klout": {
- "javaType": "com.datasift.klout.Klout",
+ "javaType": "org.apache.streams.datasift.klout.Klout",
"type": "object",
"dynamic": "true",
"properties": {
"score": {
- "type": "long"
+ "type": "integer"
}
}
},
"language": {
- "javaType": "com.datasift.interaction.Language",
+ "javaType": "org.apache.streams.datasift.interaction.Language",
"type": "object",
"dynamic": "true",
"properties": {
"confidence": {
- "type": "long"
+ "type": "integer"
},
"tag": {
"type": "string"
@@ -244,12 +244,12 @@
}
},
"links": {
- "javaType": "com.datasift.interaction.Links",
+ "javaType": "org.apache.streams.datasift.interaction.Links",
"type": "object",
"dynamic": "true",
"properties": {
"code": {
- "type": "long"
+ "type": "integer"
},
"created_at": {
"type": "array",
@@ -269,7 +269,7 @@
},
"meta": {
"dynamic": "true",
- "javaType": "com.datasift.interaction.Meta",
+ "javaType": "org.apache.streams.datasift.interaction.Meta",
"type": "object",
"properties": {
"charset": {
@@ -316,7 +316,7 @@
"type": "array",
"items": [
{
- "javaType": "com.datasift.interaction.meta.OpenGraph",
+ "javaType": "org.apache.streams.datasift.interaction.meta.OpenGraph",
"type": "object",
"dynamic": "true",
"properties": {
@@ -346,7 +346,7 @@
"type": "array",
"items": [
{
- "javaType": "com.datasift.interaction.meta.Twitter",
+ "javaType": "org.apache.streams.datasift.interaction.meta.Twitter",
"type": "object",
"dynamic": "true",
"properties": {
@@ -395,7 +395,7 @@
]
},
"retweet_count": {
- "type": "long"
+ "type": "integer"
},
"title": {
"type": "array",
@@ -416,12 +416,12 @@
}
},
"salience": {
- "javaType": "com.datasift.salience.Salience",
+ "javaType": "org.apache.streams.datasift.salience.Salience",
"type": "object",
"dynamic": "true",
"properties": {
"content": {
- "javaType": "com.datasift.salience.content.Content",
+ "javaType": "org.apache.streams.datasift.salience.content.Content",
"type": "object",
"dynamic": "true",
"properties": {
@@ -429,18 +429,18 @@
"type": "array",
"items": [
{
- "javaType": "com.datasift.salience.content.Entities",
+ "javaType": "org.apache.streams.datasift.salience.content.Entities",
"type": "object",
"dynamic": "true",
"properties": {
"about": {
- "type": "long"
+ "type": "integer"
},
"confident": {
- "type": "long"
+ "type": "integer"
},
"evidence": {
- "type": "long"
+ "type": "integer"
},
"label": {
"type": "string"
@@ -449,7 +449,7 @@
"type": "string"
},
"sentiment": {
- "type": "long"
+ "type": "integer"
},
"themes": {
"type": "string"
@@ -462,12 +462,12 @@
]
},
"sentiment": {
- "type": "long"
+ "type": "integer"
}
}
},
"title": {
- "javaType": "com.datasift.salience.title.Title",
+ "javaType": "org.apache.streams.datasift.salience.title.Title",
"type": "object",
"dynamic": "true",
"properties": {
@@ -475,18 +475,18 @@
"type": "array",
"items": [
{
- "javaType": "com.datasift.salience.title.Entities",
+ "javaType": "org.apache.streams.datasift.salience.title.Entities",
"type": "object",
"dynamic": "true",
"properties": {
"about": {
- "type": "long"
+ "type": "integer"
},
"confident": {
- "type": "long"
+ "type": "integer"
},
"evidence": {
- "type": "long"
+ "type": "integer"
},
"label": {
"type": "string"
@@ -495,7 +495,7 @@
"type": "string"
},
"sentiment": {
- "type": "long"
+ "type": "integer"
},
"themes": {
"type": "string"
@@ -508,14 +508,14 @@
]
},
"sentiment": {
- "type": "long"
+ "type": "integer"
}
}
}
}
},
"trends": {
- "javaType": "com.datasift.trends.Trends",
+ "javaType": "org.apache.streams.datasift.trends.Trends",
"type": "object",
"dynamic": "true",
"properties": {
@@ -546,7 +546,7 @@
}
},
"twitter": {
- "javaType": "com.datasift.twitter.Twitter",
+ "javaType": "org.apache.streams.datasift.twitter.Twitter",
"type": "object",
"dynamic": "true",
"properties": {
@@ -565,15 +565,15 @@
"type": "string"
},
"geo": {
- "javaType": "com.datasift.twitter.Geo",
+ "javaType": "org.apache.streams.datasift.twitter.Geo",
"type": "object",
"dynamic": "true",
"properties": {
"latitude": {
- "type": "double"
+ "type": "number"
},
"longitude": {
- "type": "double"
+ "type": "number"
}
}
},
@@ -612,7 +612,7 @@
"type": "array",
"items": [
{
- "javaType": "com.datasift.twitter.Media",
+ "javaType": "org.apache.streams.datasift.twitter.Media",
"type": "object",
"dynamic": "true",
"properties": {
@@ -623,7 +623,7 @@
"type": "string"
},
"id": {
- "type": "long"
+ "type": "integer"
},
"id_str": {
"type": "string"
@@ -641,13 +641,13 @@
"dynamic": "true",
"properties": {
"h": {
- "type": "long"
+ "type": "integer"
},
"resize": {
"type": "string"
},
"w": {
- "type": "long"
+ "type": "integer"
}
}
},
@@ -655,13 +655,13 @@
"dynamic": "true",
"properties": {
"h": {
- "type": "long"
+ "type": "integer"
},
"resize": {
"type": "string"
},
"w": {
- "type": "long"
+ "type": "integer"
}
}
},
@@ -669,13 +669,13 @@
"dynamic": "true",
"properties": {
"h": {
- "type": "long"
+ "type": "integer"
},
"resize": {
"type": "string"
},
"w": {
- "type": "long"
+ "type": "integer"
}
}
},
@@ -683,20 +683,20 @@
"dynamic": "true",
"properties": {
"h": {
- "type": "long"
+ "type": "integer"
},
"resize": {
"type": "string"
},
"w": {
- "type": "long"
+ "type": "integer"
}
}
}
}
},
"source_status_id": {
- "type": "long"
+ "type": "integer"
},
"source_status_id_str": {
"type": "string"
@@ -712,7 +712,7 @@
]
},
"mention_ids": {
- "type": "long"
+ "type": "integer"
},
"mentions": {
"type": "array",
@@ -723,7 +723,7 @@
]
},
"place": {
- "javaType": "com.datasift.twitter.Place",
+ "javaType": "org.apache.streams.datasift.twitter.Place",
"type": "object",
"dynamic": "true",
"properties": {
@@ -759,12 +759,12 @@
}
},
"retweet": {
- "javaType": "com.datasift.twitter.Retweet",
+ "javaType": "org.apache.streams.datasift.twitter.Retweet",
"type": "object",
"dynamic": "true",
"properties": {
"count": {
- "type": "long"
+ "type": "integer"
},
"created_at": {
"type": "string"
@@ -809,7 +809,7 @@
"type": "string"
},
"id": {
- "type": "long"
+ "type": "integer"
},
"id_str": {
"type": "string"
@@ -827,13 +827,13 @@
"dynamic": "true",
"properties": {
"h": {
- "type": "long"
+ "type": "integer"
},
"resize": {
"type": "string"
},
"w": {
- "type": "long"
+ "type": "integer"
}
}
},
@@ -841,13 +841,13 @@
"dynamic": "true",
"properties": {
"h": {
- "type": "long"
+ "type": "integer"
},
"resize": {
"type": "string"
},
"w": {
- "type": "long"
+ "type": "integer"
}
}
},
@@ -855,13 +855,13 @@
"dynamic": "true",
"properties": {
"h": {
- "type": "long"
+ "type": "integer"
},
"resize": {
"type": "string"
},
"w": {
- "type": "long"
+ "type": "integer"
}
}
},
@@ -869,20 +869,20 @@
"dynamic": "true",
"properties": {
"h": {
- "type": "long"
+ "type": "integer"
},
"resize": {
"type": "string"
},
"w": {
- "type": "long"
+ "type": "integer"
}
}
}
}
},
"source_status_id": {
- "type": "long"
+ "type": "integer"
},
"source_status_id_str": {
"type": "string"
@@ -896,7 +896,7 @@
}
},
"mention_ids": {
- "type": "long"
+ "type": "integer"
},
"mentions": {
"type": "array",
@@ -913,7 +913,7 @@
"type": "string"
},
"user": {
- "javaType": "com.datasift.twitter.User",
+ "javaType": "org.apache.streams.datasift.twitter.User",
"type": "object",
"dynamic": "true",
"properties": {
@@ -924,16 +924,16 @@
"type": "string"
},
"followers_count": {
- "type": "long"
+ "type": "integer"
},
"friends_count": {
- "type": "long"
+ "type": "integer"
},
"geo_enabled": {
"type": "boolean"
},
"id": {
- "type": "long"
+ "type": "integer"
},
"id_str": {
"type": "string"
@@ -942,7 +942,7 @@
"type": "string"
},
"listed_count": {
- "type": "long"
+ "type": "integer"
},
"location": {
"type": "string"
@@ -957,7 +957,7 @@
"type": "string"
},
"statuses_count": {
- "type": "long"
+ "type": "integer"
},
"time_zone": {
"type": "string"
@@ -966,14 +966,14 @@
"type": "string"
},
"utc_offset": {
- "type": "long"
+ "type": "integer"
}
}
}
}
},
"retweeted": {
- "javaType": "com.datasift.twitter.Retweeted",
+ "javaType": "org.apache.streams.datasift.twitter.Retweeted",
"type": "object",
"dynamic": "true",
"properties": {
@@ -984,10 +984,10 @@
"dynamic": "true",
"properties": {
"latitude": {
- "type": "double"
+ "type": "number"
},
"longitude": {
- "type": "double"
+ "type": "number"
}
}
},
@@ -1037,16 +1037,16 @@
"type": "string"
},
"followers_count": {
- "type": "long"
+ "type": "integer"
},
"friends_count": {
- "type": "long"
+ "type": "integer"
},
"geo_enabled": {
"type": "boolean"
},
"id": {
- "type": "long"
+ "type": "integer"
},
"id_str": {
"type": "string"
@@ -1055,7 +1055,7 @@
"type": "string"
},
"listed_count": {
- "type": "long"
+ "type": "integer"
},
"location": {
"type": "string"
@@ -1070,7 +1070,7 @@
"type": "string"
},
"statuses_count": {
- "type": "long"
+ "type": "integer"
},
"time_zone": {
"type": "string"
@@ -1079,7 +1079,7 @@
"type": "string"
},
"utc_offset": {
- "type": "long"
+ "type": "integer"
},
"verified": {
"type": "boolean"
@@ -1095,7 +1095,7 @@
"type": "string"
},
"user": {
- "javaType": "com.datasift.interaction.User",
+ "javaType": "org.apache.streams.datasift.interaction.User",
"type": "object",
"dynamic": "true",
"properties": {
@@ -1106,16 +1106,16 @@
"type": "string"
},
"followers_count": {
- "type": "long"
+ "type": "integer"
},
"friends_count": {
- "type": "long"
+ "type": "integer"
},
"geo_enabled": {
"type": "boolean"
},
"id": {
- "type": "long"
+ "type": "integer"
},
"id_str": {
"type": "string"
@@ -1124,7 +1124,7 @@
"type": "string"
},
"listed_count": {
- "type": "long"
+ "type": "integer"
},
"location": {
"type": "string"
@@ -1139,7 +1139,7 @@
"type": "string"
},
"statuses_count": {
- "type": "long"
+ "type": "integer"
},
"time_zone": {
"type": "string"
@@ -1148,7 +1148,7 @@
"type": "string"
},
"utc_offset": {
- "type": "long"
+ "type": "integer"
},
"verified": {
"type": "boolean"
@@ -1158,12 +1158,12 @@
}
},
"youtube": {
- "javaType": "com.datasift.youtube.YouTube",
+ "javaType": "org.apache.streams.datasift.youtube.YouTube",
"type": "object",
"dynamic": "true",
"properties": {
"author": {
- "javaType": "com.datasift.youtube.Author",
+ "javaType": "org.apache.streams.datasift.youtube.Author",
"type": "object",
"dynamic": "true",
"properties": {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
index 531f30b..a61e5f5 100644
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
+++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
@@ -2,13 +2,13 @@
"type": "object",
"$schema": "http://json-schema.org/draft-03/schema",
"id": "#",
- "javaType" : "com.datasift.DatasiftConfiguration",
+ "javaType" : "org.apache.streams.datasift.DatasiftConfiguration",
"properties": {
"apiKey": {
"type": "string"
},
"userName": {
- "type": "long"
+ "type": "string"
},
"streamHash": {
"type": "array",
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-provider-datasift/src/test/java/com/datasift/test/DatasiftSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java/com/datasift/test/DatasiftSerDeTest.java b/streams-contrib/streams-provider-datasift/src/test/java/com/datasift/test/DatasiftSerDeTest.java
index 6a586d0..382a40b 100644
--- a/streams-contrib/streams-provider-datasift/src/test/java/com/datasift/test/DatasiftSerDeTest.java
+++ b/streams-contrib/streams-provider-datasift/src/test/java/com/datasift/test/DatasiftSerDeTest.java
@@ -1,9 +1,8 @@
package com.datasift.test;
-import com.datasift.Datasift;
-
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.streams.datasift.Datasift;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
@@ -35,7 +34,7 @@ public class DatasiftSerDeTest {
mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
- InputStream is = DatasiftSerDeTest.class.getResourceAsStream("/datasift_jsons.txt");
+ InputStream is = DatasiftSerDeTest.class.getResourceAsStream("/part-r-00000.json");
InputStreamReader isr = new InputStreamReader(is);
BufferedReader br = new BufferedReader(isr);
[09/13] git commit: adding overlays to .gitignore
Posted by sb...@apache.org.
adding overlays to .gitignore
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/381d7583
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/381d7583
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/381d7583
Branch: refs/heads/springcleaning
Commit: 381d7583ebaa7eabd326103a9e511687ab23480a
Parents: 89ff615
Author: Steve Blackmon <sb...@w2odigital.com>
Authored: Sun Mar 23 18:27:59 2014 -0500
Committer: Steve Blackmon <sb...@w2odigital.com>
Committed: Sun Mar 23 18:27:59 2014 -0500
----------------------------------------------------------------------
.gitignore | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/381d7583/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 247f694..23d6cc9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@ streams-master-pom/
# Compile directores and files #
target/
+overlays/
dependency-reduced-pom.xml
# IDE directories/files #
[07/13] adding uncommitted modules
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/89ff615e/streams-contrib/streams-provider-datasift/src/test/resources/part-r-00000.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/resources/part-r-00000.json b/streams-contrib/streams-provider-datasift/src/test/resources/part-r-00000.json
new file mode 100644
index 0000000..b9dc309
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/test/resources/part-r-00000.json
@@ -0,0 +1,862 @@
+{"demographic":{"gender":"male"},"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/1384599202\/picture","id":"1384599202","link":"http:\/\/www.facebook.com\/profile.php?id=1384599202","name":"Harry Medina"},"caption":"www.youtube.com","created_at":"Wed, 02 Oct 2013 23:23:48 +0000","id":"1384599202_10202229370960760","link":"http:\/\/www.youtube.com\/watch?v=QoDltFECFgI","message":"http:\/\/www.youtube.com\/watch?v=QoDltFECFgI","name":"Apache P\u00F3nmela en el aire","source":"http:\/\/www.youtube.com\/v\/QoDltFECFgI?version=3&autohide=1&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/1384599202\/picture","id":"1384599202","link":"http:\/\/www.facebook.com\/profile.php?id=1384599202","name":"Harry Medina"},"content":"http:\/\/www.youtube.com\/watch?v=QoDltFECFgI","created_at":"Wed, 02 Oct 2013 23:24:22 +0000","id":"1e32bb9b06b1aa00e06699f748eabed2","link":"http:\/\/www.facebook.com\/1384599202_10202229370960760","schema":{"vers
ion":3},"source":"http:\/\/www.youtube.com\/v\/QoDltFECFgI?version=3&autohide=1&autoplay=1","subtype":"video","title":"Apache P\u00F3nmela en el aire","type":"facebook"},"language":{"confidence":100,"tag":"en"},"links":{"code":[200],"created_at":["Wed, 02 Oct 2013 23:24:20 +0000"],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"lang":["en"],"opengraph":[{"site_name":"YouTube","type":"video","description":"","url":"http:\/\/www.youtube.com\/watch?v=QoDltFECFgI","title":"Apache P\u00F3nmela en el aire","image":"http:\/\/i1.ytimg.com\/vi\/QoDltFECFgI\/hqdefault.jpg?feature=og"}],"twitter":[{"player_width":"480","description":"","player_height":"360","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/QoDltFECFgI","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/QoDltFECFgI","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=QoDltFECFgI","id":"com.google.android.youtube"}},"site":"@yout
ube","url":"http:\/\/www.youtube.com\/watch?v=QoDltFECFgI","title":"Apache P\u00F3nmela en el aire","image":"http:\/\/i1.ytimg.com\/vi\/QoDltFECFgI\/hqdefault.jpg","player":"https:\/\/www.youtube.com\/embed\/QoDltFECFgI"}]},"normalized_url":["http:\/\/youtube.com\/watch?v=QoDltFECFgI"],"retweet_count":[0],"title":["Apache P\u00F3nmela en el aire - YouTube"],"url":["http:\/\/www.youtube.com\/watch?v=QoDltFECFgI"]},"salience":{"content":{"sentiment":0},"title":{"sentiment":0}}}
+{"demographic":{"gender":"female"},"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/100001186762687\/picture","id":"100001186762687","link":"http:\/\/www.facebook.com\/profile.php?id=100001186762687","name":"Magdalena Piela"},"caption":"www.youtube.com","created_at":"Wed, 02 Oct 2013 23:32:27 +0000","description":"US Army Video - Apache Helicopter (ACDC - Thunderstruck)","id":"100001186762687_567905416592345","link":"http:\/\/www.youtube.com\/watch?v=mYMiEU0vKIM","message":"there are things that you should not do ... simply not...\nhttp:\/\/www.youtube.com\/watch?v=mYMiEU0vKIM","name":"US Army Video - Apache Helicopter (ACDC - Thunderstruck)","source":"http:\/\/www.youtube.com\/v\/mYMiEU0vKIM?autohide=1&version=3&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/100001186762687\/picture","id":"100001186762687","link":"http:\/\/www.facebook.com\/profile.php?id=100001186762687","name":"Magdalena Piela"},"content":"there are thing
s that you should not do ... simply not...\nhttp:\/\/www.youtube.com\/watch?v=mYMiEU0vKIM","created_at":"Wed, 02 Oct 2013 23:32:38 +0000","id":"1e32bbae5c43a780e06676c9b3bc93d0","link":"http:\/\/www.facebook.com\/100001186762687_567905416592345","received_at":4743578121590502169,"schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/mYMiEU0vKIM?autohide=1&version=3&autoplay=1","subtype":"video","title":"US Army Video - Apache Helicopter (ACDC - Thunderstruck)","type":"facebook"},"language":{"confidence":100,"tag":"en"},"links":{"code":[200],"created_at":["Wed, 02 Oct 2013 00:11:00 +0000"],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["US Army Video - Apache Helicopter (ACDC - Thunderstruck)"],"keywords":[["US","Army"]],"lang":["en"],"opengraph":[{"site_name":"YouTube","type":"video","description":"US Army Video - Apache Helicopter (ACDC - Thunderstruck)","url":"http:\/\/www.youtube.com\/watch?v=mYMiEU0vKIM","title":"US Army Video - Apache Helicopter
(ACDC - Thunderstruck)","image":"http:\/\/i1.ytimg.com\/vi\/mYMiEU0vKIM\/hqdefault.jpg?feature=og"}],"twitter":[{"player_width":"480","description":"US Army Video - Apache Helicopter (ACDC - Thunderstruck)","player_height":"360","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/mYMiEU0vKIM","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/mYMiEU0vKIM","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=mYMiEU0vKIM","id":"com.google.android.youtube"}},"site":"@youtube","url":"http:\/\/www.youtube.com\/watch?v=mYMiEU0vKIM","title":"US Army Video - Apache Helicopter (ACDC - Thunderstruck)","image":"http:\/\/i1.ytimg.com\/vi\/mYMiEU0vKIM\/hqdefault.jpg","player":"https:\/\/www.youtube.com\/embed\/mYMiEU0vKIM"}]},"normalized_url":["http:\/\/youtube.com\/watch?v=mYMiEU0vKIM"],"retweet_count":[0],"title":["US Army Video - Apache Helicopter (ACDC - Thunderstruck) - YouTube"],"url":["http:\/\/www.you
tube.com\/watch?v=mYMiEU0vKIM"]},"salience":{"content":{"sentiment":0},"title":{"sentiment":0,"topics":[{"name":"War","score":0.64078038930893,"hits":0},{"name":"Aviation","score":0.48720294237137,"hits":0},{"name":"Video Games","score":0.45472154021263,"hits":0}]}}}
+{"demographic":{"gender":"male"},"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/100004185664005\/picture","id":"100004185664005","link":"http:\/\/www.facebook.com\/profile.php?id=100004185664005","name":"Jeff Moses"},"caption":"www.moderntimesmagazine.com","created_at":"Wed, 02 Oct 2013 23:32:49 +0000","description":"The Third Annual, Wilderness Surrounded, Local Music Festival Will Bring Some Of The Best Musical Talent In Arizona To Apache Lake For A Beachside Party For The Ages","id":"100004185664005_239409246208596","link":"http:\/\/www.moderntimesmagazine.com\/page19\/Music_Apache_Lake_131001\/Music_Apache_Lake_131001.php","message":"For Us Presents aren't the only people putting on great events! Check out Apache Lake Music Festival 2013 shit's going to be live!","name":"Apache Lake Music Festival Ramps Up For Third Try","source":"web","to":{"ids":["106816409488565","618209761535313"],"names":["For Us Presents","Apache Lake Music Festival 2013"]},"type":"link"},"i
nteraction":{"author":{"avatar":"https:\/\/graph.facebook.com\/100004185664005\/picture","id":"100004185664005","link":"http:\/\/www.facebook.com\/profile.php?id=100004185664005","name":"Jeff Moses"},"content":"For Us Presents aren't the only people putting on great events! Check out Apache Lake Music Festival 2013 shit's going to be live!","created_at":"Wed, 02 Oct 2013 23:33:11 +0000","id":"1e32bbaf2e12a680e06698742eeb58a8","link":"http:\/\/www.facebook.com\/100004185664005_239409246208596","mention_ids":["106816409488565","618209761535313"],"mentions":["For Us Presents","Apache Lake Music Festival 2013"],"schema":{"version":3},"source":"web","subtype":"link","title":"Apache Lake Music Festival Ramps Up For Third Try","type":"facebook"},"language":{"confidence":100,"tag":"en"},"links":{"code":[200],"created_at":["Wed, 02 Oct 2013 22:10:40 +0000"],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"keywords":[["Apache Lake Music Festival","ALMF","Kongo\u2019s","Future Loves
Past","The Sugar Thieves","Apache Lake Resort and Marina","229.5 mile marker","Roosevelt","Ariz.","McDowell Mountain Music Festival","The Wiley Ones","Banana Gun","Mergence","Dry River Yacht Club","Jared and the Mill","Decker","Phoenix Metro","The Hourglass Cats","The Haymarket Squares","Sister Lip","Japhy\u2019s Descent","Kush County","Steff and the Articles","The Cordials","Elizabeth Rose","Mr. Mudd and Mr. Gold","Greyhound Soul","The Prowling Kind","Banshee Bones","Sun Ghost","Sasquanaut","Sara Robinson and the Midnight Special","The Black Moods","Princess","Walt Richardson","DJ Stowner","TKLB?","Kevin Redlich\/C-Posse\/Corey Golden Explosion","Ghetto Cowgirl","Huckleberry","The Senators","Northern Hustle"]],"lang":["unknown"],"opengraph":[{"description":"The Third Annual, Wilderness Surrounded, Local Music Festival Will Bring Some Of The Best Musical Talent In Arizona To Apache Lake For A Beachside Party For The Ages","type":"article","url":"http:\/\/www.moderntimesmagazine.com\
/page19\/Music_Apache_Lake_131001\/Music_Apache_Lake_131001.php","image":"http:\/\/www.moderntimesmagazine.com\/moderntimesimages2013\/MT_Images_1310\/ALMF_560.jpg","title":"Apache Lake Music Festival Ramps Up For Third Try"}]},"normalized_url":["http:\/\/moderntimesmagazine.com\/page19\/Music_Apache_Lake_131001\/Music_Apache_Lake_131001.php"],"retweet_count":[0],"title":["Apache Lake Music Festival Ramps Up For Third Try"],"url":["http:\/\/www.moderntimesmagazine.com\/page19\/Music_Apache_Lake_131001\/Music_Apache_Lake_131001.php"]},"salience":{"content":{"sentiment":0},"title":{"sentiment":0}}}
+{"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/106816409488565\/picture","id":"106816409488565","link":"http:\/\/www.facebook.com\/profile.php?id=106816409488565","name":"For Us Presents"},"caption":"www.moderntimesmagazine.com","created_at":"Wed, 02 Oct 2013 23:33:47 +0000","description":"The Third Annual, Wilderness Surrounded, Local Music Festival Will Bring Some Of The Best Musical Talent In Arizona To Apache Lake For A Beachside Party For The Ages","id":"106816409488565_125339574302915","link":"http:\/\/www.moderntimesmagazine.com\/page19\/Music_Apache_Lake_131001\/Music_Apache_Lake_131001.php","message":"We aint the only locals putting on awesome events! Check out Apache Lake Music Festival coming up this month! It should be rather epic!","name":"Apache Lake Music Festival Ramps Up For Third Try","source":"web","to":{"ids":["122055604509070"],"names":["Apache Lake Music Festival"]},"type":"link"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\
/106816409488565\/picture","id":"106816409488565","link":"http:\/\/www.facebook.com\/profile.php?id=106816409488565","name":"For Us Presents"},"content":"We aint the only locals putting on awesome events! Check out Apache Lake Music Festival coming up this month! It should be rather epic!","created_at":"Wed, 02 Oct 2013 23:34:16 +0000","id":"1e32bbb15733af80e0669248f7499464","link":"http:\/\/www.facebook.com\/106816409488565_125339574302915","mention_ids":["122055604509070"],"mentions":["Apache Lake Music Festival"],"received_at":4743578121999296233,"schema":{"version":3},"source":"web","subtype":"link","title":"Apache Lake Music Festival Ramps Up For Third Try","type":"facebook"},"language":{"confidence":97,"tag":"en"},"links":{"code":[200],"created_at":["Wed, 02 Oct 2013 22:10:40 +0000"],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"keywords":[["Apache Lake Music Festival","ALMF","Kongo\u2019s","Future Loves Past","The Sugar Thieves","Apache Lake Resort and Marina","
229.5 mile marker","Roosevelt","Ariz.","McDowell Mountain Music Festival","The Wiley Ones","Banana Gun","Mergence","Dry River Yacht Club","Jared and the Mill","Decker","Phoenix Metro","The Hourglass Cats","The Haymarket Squares","Sister Lip","Japhy\u2019s Descent","Kush County","Steff and the Articles","The Cordials","Elizabeth Rose","Mr. Mudd and Mr. Gold","Greyhound Soul","The Prowling Kind","Banshee Bones","Sun Ghost","Sasquanaut","Sara Robinson and the Midnight Special","The Black Moods","Princess","Walt Richardson","DJ Stowner","TKLB?","Kevin Redlich\/C-Posse\/Corey Golden Explosion","Ghetto Cowgirl","Huckleberry","The Senators","Northern Hustle"]],"lang":["unknown"],"opengraph":[{"description":"The Third Annual, Wilderness Surrounded, Local Music Festival Will Bring Some Of The Best Musical Talent In Arizona To Apache Lake For A Beachside Party For The Ages","type":"article","url":"http:\/\/www.moderntimesmagazine.com\/page19\/Music_Apache_Lake_131001\/Music_Apache_Lake_131001
.php","image":"http:\/\/www.moderntimesmagazine.com\/moderntimesimages2013\/MT_Images_1310\/ALMF_560.jpg","title":"Apache Lake Music Festival Ramps Up For Third Try"}]},"normalized_url":["http:\/\/moderntimesmagazine.com\/page19\/Music_Apache_Lake_131001\/Music_Apache_Lake_131001.php"],"retweet_count":[0],"title":["Apache Lake Music Festival Ramps Up For Third Try"],"url":["http:\/\/www.moderntimesmagazine.com\/page19\/Music_Apache_Lake_131001\/Music_Apache_Lake_131001.php"]},"salience":{"content":{"sentiment":9},"title":{"sentiment":0}}}
+{"demographic":{"gender":"male"},"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/100001967485067\/picture","id":"100001967485067","link":"http:\/\/www.facebook.com\/profile.php?id=100001967485067","name":"Felipe Cardoso"},"caption":"www.youtube.com","created_at":"Wed, 02 Oct 2013 23:41:46 +0000","description":"Apache (Indian) song - Tatanka \/\/ Indi\u00E1nsk\u00E9 p\u00EDsn\u011B - Tatanka","id":"100001967485067_516580328417545","link":"https:\/\/www.youtube.com\/watch?v=PoO3yga6Ln8","message":"https:\/\/www.youtube.com\/watch?v=PoO3yga6Ln8","name":"Apache song - Tatanka","source":"http:\/\/www.youtube.com\/v\/PoO3yga6Ln8?autohide=1&version=3&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/100001967485067\/picture","id":"100001967485067","link":"http:\/\/www.facebook.com\/profile.php?id=100001967485067","name":"Felipe Cardoso"},"content":"https:\/\/www.youtube.com\/watch?v=PoO3yga6Ln8","created_at":"Wed, 02 Oct 2013 23:42:1
0 +0000","id":"1e32bbc32f4da900e0666cecb54a8f3a","link":"http:\/\/www.facebook.com\/100001967485067_516580328417545","received_at":"A\u0513+\u0504,=","schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/PoO3yga6Ln8?autohide=1&version=3&autoplay=1","subtype":"video","title":"Apache song - Tatanka","type":"facebook"},"language":{"confidence":100,"tag":"en"},"links":{"code":[200],"created_at":["Wed, 02 Oct 2013 23:42:08 +0000"],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["Apache (Indian) song - Tatanka \/\/ Indi\u00E1nsk\u00E9 p\u00EDsn\u011B - Tatanka"],"keywords":[["Apache Song","Indi\u00E1nsk\u00E1 p\u00EDse\u0148","Indian song","Relax song"]],"lang":["en"],"opengraph":[{"site_name":"YouTube","type":"video","description":"Apache (Indian) song - Tatanka \/\/ Indi\u00E1nsk\u00E9 p\u00EDsn\u011B - Tatanka","url":"http:\/\/www.youtube.com\/watch?v=PoO3yga6Ln8","title":"Apache song - Tatanka","image":"https:\/\/i1.ytimg.com\/vi\/PoO3yga6Ln8\/hqdefaul
t.jpg?feature=og"}],"twitter":[{"player_width":"480","description":"Apache (Indian) song - Tatanka \/\/ Indi\u00E1nsk\u00E9 p\u00EDsn\u011B - Tatanka","player_height":"360","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/PoO3yga6Ln8","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/PoO3yga6Ln8","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=PoO3yga6Ln8","id":"com.google.android.youtube"}},"site":"@youtube","url":"http:\/\/www.youtube.com\/watch?v=PoO3yga6Ln8","title":"Apache song - Tatanka","image":"https:\/\/i1.ytimg.com\/vi\/PoO3yga6Ln8\/hqdefault.jpg","player":"https:\/\/www.youtube.com\/embed\/PoO3yga6Ln8"}]},"normalized_url":["https:\/\/youtube.com\/watch?v=PoO3yga6Ln8"],"retweet_count":[0],"title":["Apache song - Tatanka - YouTube"],"url":["https:\/\/www.youtube.com\/watch?v=PoO3yga6Ln8"]},"salience":{"content":{"sentiment":0},"title":{"entities":[{"name":"Apache","sentiment":0,"
confident":1,"label":"Company","evidence":1,"type":"Company","about":1}],"sentiment":0}}}
+{"demographic":{"gender":"male"},"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/100001967485067\/picture","id":"100001967485067","link":"http:\/\/www.facebook.com\/profile.php?id=100001967485067","name":"Felipe Cardoso"},"caption":"www.youtube.com","created_at":"Wed, 02 Oct 2013 23:42:44 +0000","description":"Apache (Indian) song - Tatanka \/\/ Indi\u00E1nsk\u00E9 p\u00EDsn\u011B - Tatanka","id":"100001967485067_516580495084195","link":"https:\/\/www.youtube.com\/watch?v=PoO3yga6Ln8","message":"Trabalhar ent\u00E3o :D \nhttps:\/\/www.youtube.com\/watch?v=PoO3yga6Ln8","name":"Apache song - Tatanka","source":"http:\/\/www.youtube.com\/v\/PoO3yga6Ln8?autohide=1&version=3&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/100001967485067\/picture","id":"100001967485067","link":"http:\/\/www.facebook.com\/profile.php?id=100001967485067","name":"Felipe Cardoso"},"content":"Trabalhar ent\u00E3o :D \nhttps:\/\/www.youtube.com\/watch?v
=PoO3yga6Ln8","created_at":"Wed, 02 Oct 2013 23:43:16 +0000","id":"1e32bbc5586fa200e06602d8ed5fbfda","link":"http:\/\/www.facebook.com\/100001967485067_516580495084195","received_at":4743578124263229699,"schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/PoO3yga6Ln8?autohide=1&version=3&autoplay=1","subtype":"video","title":"Apache song - Tatanka","type":"facebook"},"language":{"confidence":62,"tag":"pt"},"links":{"code":[200],"created_at":["Wed, 02 Oct 2013 23:42:08 +0000"],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["Apache (Indian) song - Tatanka \/\/ Indi\u00E1nsk\u00E9 p\u00EDsn\u011B - Tatanka"],"keywords":[["Apache Song","Indi\u00E1nsk\u00E1 p\u00EDse\u0148","Indian song","Relax song"]],"lang":["en"],"opengraph":[{"site_name":"YouTube","type":"video","description":"Apache (Indian) song - Tatanka \/\/ Indi\u00E1nsk\u00E9 p\u00EDsn\u011B - Tatanka","url":"http:\/\/www.youtube.com\/watch?v=PoO3yga6Ln8","title":"Apache song - Tatanka","image
":"https:\/\/i1.ytimg.com\/vi\/PoO3yga6Ln8\/hqdefault.jpg?feature=og"}],"twitter":[{"player_width":"480","description":"Apache (Indian) song - Tatanka \/\/ Indi\u00E1nsk\u00E9 p\u00EDsn\u011B - Tatanka","player_height":"360","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/PoO3yga6Ln8","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/PoO3yga6Ln8","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=PoO3yga6Ln8","id":"com.google.android.youtube"}},"site":"@youtube","url":"http:\/\/www.youtube.com\/watch?v=PoO3yga6Ln8","title":"Apache song - Tatanka","image":"https:\/\/i1.ytimg.com\/vi\/PoO3yga6Ln8\/hqdefault.jpg","player":"https:\/\/www.youtube.com\/embed\/PoO3yga6Ln8"}]},"normalized_url":["https:\/\/youtube.com\/watch?v=PoO3yga6Ln8"],"retweet_count":[0],"title":["Apache song - Tatanka - YouTube"],"url":["https:\/\/www.youtube.com\/watch?v=PoO3yga6Ln8"]},"salience":{"content":{"sentiment":0},"
title":{"sentiment":0}}}
+{"demographic":{"gender":"female"},"facebook":{"application":"Links","author":{"avatar":"https:\/\/graph.facebook.com\/100005172182096\/picture","id":"100005172182096","link":"http:\/\/www.facebook.com\/profile.php?id=100005172182096","name":"Melissa Wright"},"caption":"www.youtube.com","created_at":"Wed, 02 Oct 2013 23:44:11 +0000","description":"US Army Video - Apache Helicopter (ACDC - Thunderstruck)","id":"100005172182096_202896713226067","link":"http:\/\/www.youtube.com\/attribution_link?a=2ITrTNk-GH0pXMd9WQQZkw&u=%2Fwatch%3Fv%3DmYMiEU0vKIM%26feature%3Dshare","message":"SOMEONE ANYONE PLEASE RE-SHARE THIS AS I HAVE BLOCKED THESE PEOPLE AND I NEED THEM TO KNOW IM ON TO YOU...............ALL FAKERS... IF YOU HATE A SCAM ARTIST THAT TAKES ADVANTAGE OF CAUSES FOR CHILDREN YOU WILL SHARE THIS POST.........\n\nTO THE IMMEDIATE ATTENTION OF THE PERSONS THAT TARGETED MY POST ON ADVOCATING CHILD ABUSE AND NEGLECT AND HUMAN TRAFFICKING AND CLIPED THEM WITHOUT RE-SHARING THEM.\n EVERY SIN
GEL ONE OF YOU SHOULD BE ASHAMED OF YOUR SELFS FOR CLIPING POST AND PASSING THEM ON AS YOUR WORK AND WORDS IN THE NAME OF ADVOCATING THESE CAUSES. MY SOLE INTENT IN POSTING THE POST WAS TO RAISE AWARENESS TO THESE CAUSES BY RE-SAHRING THEM FOR COMMENTS WHICH IN TURN RAISES AWARENESS TO THE CAUSES THAT I NOT ONLY ADVOCATE FOR BUT EAT SLEEP AND BREATH TO DO SO. \n\nWITH THAT BEING SAID I FURTHER WISH FOR YOU ALL TO KNOW I REFUSE TO LET YOU CLIP MY POST TO DESCISE AS YOUR ON \u201CTO PULL ON THE HEART STRING OF OTHERS, IN THE NAME OF YOUR GREED AND ILL GOTTEN INTENT. I HAVE BEEN REMOVING ALL OF MY POST AND HAVE SEVERED TIES WITH YOU THAT HAVE DONE THIS. I HAVE FURTHER LOCKED DOWN THE GROUP SOLDIERS FOR CHILDREN AND YOU HAVE BEEN PERMITLY BANNED BECAUSE OF YOUR SHADY NATUER. \n I FURTHER WISH FOR YOU TO KNOW IM ON TO YOU \u201CFAKERS THAT GO AROUND PULLING ON THE HEART STRINGS OF THE PUBLIC FOR PERSONAL GAIN AND FAME AND TRUST ME WHEN I TELL YOU THIS: I WILL PUBLICLY EXPOSE YOU FOR WHAT
YOU TRULY ARE. THEREFORE, SONG WRITERS AND MEMORIAL BUILDERS BEWARE BECAUSE I KNOW THINGS ABOUT YOU \u201CDONE DIRT CHEEP DEEDS THAT THE WHOLE WORLD WILL WANT TO KNOW ABOUT. SO WHEN PEOPLE QUESTION ME ABOUT WHO YOU ARE I WILL SIMPLY SAY IT CAN BE TOLD ON YOUR OWN TIMELINES TO BE FOUND THAT YOUR WORKS WAS NEVER YOUR OWN. \n\n IN CLOSING, I WOULD LIKE TO SAY THAT GOD SEES ALL THINGS AND TRUTH WILL ALWAYS BE SO JUST AS IT IS ALSO TRUE IN THE LAW OF HARVEST IT WILL BE SO WITH ME. THE LAW OF HARVEST\u2026\u2026REMEMBER THIS- IF YOU GIVE LITTLE, YOU WILL GET LITTLE. A FARMER WHO PLANTS JUST FEW SEEDS WILL ONLY GET SMALL CROP. BUT A FARMER THAT PLANTS MUCH, HE REEPS MUCH 11CORINTHIANS 9:6","name":"US Army Video - Apache Helicopter (ACDC - Thunderstruck)","source":"http:\/\/www.youtube.com\/v\/mYMiEU0vKIM?autohide=1&version=3&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/100005172182096\/picture","id":"100005172182096","link":"http:\/\/www.fac
ebook.com\/profile.php?id=100005172182096","name":"Melissa Wright"},"content":"SOMEONE ANYONE PLEASE RE-SHARE THIS AS I HAVE BLOCKED THESE PEOPLE AND I NEED THEM TO KNOW IM ON TO YOU...............ALL FAKERS... IF YOU HATE A SCAM ARTIST THAT TAKES ADVANTAGE OF CAUSES FOR CHILDREN YOU WILL SHARE THIS POST.........\n\nTO THE IMMEDIATE ATTENTION OF THE PERSONS THAT TARGETED MY POST ON ADVOCATING CHILD ABUSE AND NEGLECT AND HUMAN TRAFFICKING AND CLIPED THEM WITHOUT RE-SHARING THEM.\n EVERY SINGEL ONE OF YOU SHOULD BE ASHAMED OF YOUR SELFS FOR CLIPING POST AND PASSING THEM ON AS YOUR WORK AND WORDS IN THE NAME OF ADVOCATING THESE CAUSES. MY SOLE INTENT IN POSTING THE POST WAS TO RAISE AWARENESS TO THESE CAUSES BY RE-SAHRING THEM FOR COMMENTS WHICH IN TURN RAISES AWARENESS TO THE CAUSES THAT I NOT ONLY ADVOCATE FOR BUT EAT SLEEP AND BREATH TO DO SO. \n\nWITH THAT BEING SAID I FURTHER WISH FOR YOU ALL TO KNOW I REFUSE TO LET YOU CLIP MY POST TO DESCISE AS YOUR ON \u201CTO PULL ON THE HEART
STRING OF OTHERS, IN THE NAME OF YOUR GREED AND ILL GOTTEN INTENT. I HAVE BEEN REMOVING ALL OF MY POST AND HAVE SEVERED TIES WITH YOU THAT HAVE DONE THIS. I HAVE FURTHER LOCKED DOWN THE GROUP SOLDIERS FOR CHILDREN AND YOU HAVE BEEN PERMITLY BANNED BECAUSE OF YOUR SHADY NATUER. \n I FURTHER WISH FOR YOU TO KNOW IM ON TO YOU \u201CFAKERS THAT GO AROUND PULLING ON THE HEART STRINGS OF THE PUBLIC FOR PERSONAL GAIN AND FAME AND TRUST ME WHEN I TELL YOU THIS: I WILL PUBLICLY EXPOSE YOU FOR WHAT YOU TRULY ARE. THEREFORE, SONG WRITERS AND MEMORIAL BUILDERS BEWARE BECAUSE I KNOW THINGS ABOUT YOU \u201CDONE DIRT CHEEP DEEDS THAT THE WHOLE WORLD WILL WANT TO KNOW ABOUT. SO WHEN PEOPLE QUESTION ME ABOUT WHO YOU ARE I WILL SIMPLY SAY IT CAN BE TOLD ON YOUR OWN TIMELINES TO BE FOUND THAT YOUR WORKS WAS NEVER YOUR OWN. \n\n IN CLOSING, I WOULD LIKE TO SAY THAT GOD SEES ALL THINGS AND TRUTH WILL ALWAYS BE SO JUST AS IT IS ALSO TRUE IN THE LAW OF HARVEST IT WILL BE SO WITH ME. THE LAW OF HARVEST\u2
026\u2026REMEMBER THIS- IF YOU GIVE LITTLE, YOU WILL GET LITTLE. A FARMER WHO PLANTS JUST FEW SEEDS WILL ONLY GET SMALL CROP. BUT A FARMER THAT PLANTS MUCH, HE REEPS MUCH 11CORINTHIANS 9:6","created_at":"Wed, 02 Oct 2013 23:44:29 +0000","id":"1e32bbc89621a780e066fff38196ca16","link":"http:\/\/www.facebook.com\/100005172182096_202896713226067","schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/mYMiEU0vKIM?autohide=1&version=3&autoplay=1","subtype":"video","title":"US Army Video - Apache Helicopter (ACDC - Thunderstruck)","type":"facebook"},"language":{"confidence":100,"tag":"en"},"links":{"code":[200],"created_at":["Wed, 02 Oct 2013 00:11:00 +0000"],"hops":[["http:\/\/www.youtube.com\/attribution_link?a=2ITrTNk-GH0pXMd9WQQZkw&u=%2Fwatch%3Fv%3DmYMiEU0vKIM%26feature%3Dshare","http:\/\/www.youtube.com\/watch?v=mYMiEU0vKIM&feature=share"]],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["US Army Video - Apache Helicopter (ACDC - Thunderstruck)"],"keywo
rds":[["US","Army"]],"lang":["en"],"opengraph":[{"site_name":"YouTube","type":"video","description":"US Army Video - Apache Helicopter (ACDC - Thunderstruck)","url":"http:\/\/www.youtube.com\/watch?v=mYMiEU0vKIM","title":"US Army Video - Apache Helicopter (ACDC - Thunderstruck)","image":"http:\/\/i1.ytimg.com\/vi\/mYMiEU0vKIM\/hqdefault.jpg?feature=og"}],"twitter":[{"player_width":"480","description":"US Army Video - Apache Helicopter (ACDC - Thunderstruck)","player_height":"360","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/mYMiEU0vKIM","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/mYMiEU0vKIM","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=mYMiEU0vKIM","id":"com.google.android.youtube"}},"site":"@youtube","url":"http:\/\/www.youtube.com\/watch?v=mYMiEU0vKIM","title":"US Army Video - Apache Helicopter (ACDC - Thunderstruck)","image":"http:\/\/i1.ytimg.com\/vi\/mYMiEU0vKIM\/hqdefa
ult.jpg","player":"https:\/\/www.youtube.com\/embed\/mYMiEU0vKIM"}]},"normalized_url":["http:\/\/youtube.com\/watch?v=mYMiEU0vKIM"],"retweet_count":[0],"title":["US Army Video - Apache Helicopter (ACDC - Thunderstruck) - YouTube"],"url":["http:\/\/www.youtube.com\/watch?v=mYMiEU0vKIM"]},"salience":{"title":{"sentiment":0,"topics":[{"name":"War","score":0.64078038930893,"hits":0},{"name":"Aviation","score":0.48720294237137,"hits":0},{"name":"Video Games","score":0.45472154021263,"hits":0}]}}}
+{"demographic":{"gender":"male"},"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/100003154316845\/picture","id":"100003154316845","link":"http:\/\/www.facebook.com\/profile.php?id=100003154316845","name":"Artur Rodrigues"},"caption":"www.youtube.com","created_at":"Wed, 02 Oct 2013 23:51:50 +0000","description":"Enquanto a nossa produ\u00E7\u00E3o resolve tomar vergonha na cara e colocar o v\u00EDdeo Oficial BMD [Violeira de Apache] Decidimos colocar nosso v\u00EDdeo pessoal gravado por...","id":"100003154316845_455573131224493","link":"http:\/\/www.youtube.com\/watch?v=jkYw-QuZToI","message":"Lembrei de tu Fellipe Moreira ,kkkkk se liga ai tbm Gleydison Ribeiro , massa d+ pivete ,hahahahahahaha","name":"BMD - Violeira de Apache","source":"http:\/\/www.youtube.com\/v\/jkYw-QuZToI?version=3&autohide=1&autoplay=1","to":{"ids":["100000843657444","100000091430993"],"names":["Fellipe Moreira","Gleydison Ribeiro"]},"type":"video"},"interaction":{"author":{"avatar":"https:\/\/
graph.facebook.com\/100003154316845\/picture","id":"100003154316845","link":"http:\/\/www.facebook.com\/profile.php?id=100003154316845","name":"Artur Rodrigues"},"content":"Lembrei de tu Fellipe Moreira ,kkkkk se liga ai tbm Gleydison Ribeiro , massa d+ pivete ,hahahahahahaha","created_at":"Wed, 02 Oct 2013 23:52:20 +0000","id":"1e32bbd9af7eaf00e0669cbbe5197ab4","link":"http:\/\/www.facebook.com\/100003154316845_455573131224493","mention_ids":["100000843657444","100000091430993"],"mentions":["Fellipe Moreira","Gleydison Ribeiro"],"received_at":4743578126547750067,"schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/jkYw-QuZToI?version=3&autohide=1&autoplay=1","subtype":"video","title":"BMD - Violeira de Apache","type":"facebook"},"language":{"confidence":100,"tag":"pt"},"links":{"code":[200],"created_at":["Wed, 02 Oct 2013 20:00:17 +0000"],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["Enquanto a nossa produ\u00E7\u00E3o resolve tomar vergonha na
cara e colocar o v\u00EDdeo Oficial BMD [Violeira de Apache] Decidimos colocar nosso v\u00EDdeo pessoal gravado por..."],"keywords":[["BMD"]],"lang":["en"],"opengraph":[{"site_name":"YouTube","type":"video","description":"Enquanto a nossa produ\u00E7\u00E3o resolve tomar vergonha na cara e colocar o v\u00EDdeo Oficial BMD [Violeira de Apache] Decidimos colocar nosso v\u00EDdeo pessoal gravado por...","url":"http:\/\/www.youtube.com\/watch?v=jkYw-QuZToI","title":"BMD - Violeira de Apache","image":"http:\/\/i1.ytimg.com\/vi\/jkYw-QuZToI\/hqdefault.jpg?feature=og"}],"twitter":[{"player_width":"640","description":"Enquanto a nossa produ\u00E7\u00E3o resolve tomar vergonha na cara e colocar o v\u00EDdeo Oficial BMD [Violeira de Apache] Decidimos colocar nosso v\u00EDdeo pessoal gravado por...","player_height":"480","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/jkYw-QuZToI","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/jkYw-QuZT
oI","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=jkYw-QuZToI","id":"com.google.android.youtube"}},"site":"@youtube","url":"http:\/\/www.youtube.com\/watch?v=jkYw-QuZToI","title":"BMD - Violeira de Apache","image":"http:\/\/i1.ytimg.com\/vi\/jkYw-QuZToI\/hqdefault.jpg","player":"https:\/\/www.youtube.com\/embed\/jkYw-QuZToI"}]},"normalized_url":["http:\/\/youtube.com\/watch?v=jkYw-QuZToI"],"retweet_count":[0],"title":["BMD - Violeira de Apache - YouTube"],"url":["http:\/\/www.youtube.com\/watch?v=jkYw-QuZToI"]},"salience":{"content":{"entities":[{"name":"Fellipe Moreira","sentiment":0,"confident":1,"label":"Person","evidence":0,"type":"Person","about":1},{"name":"Gleydison Ribeiro","sentiment":0,"confident":1,"label":"Person","evidence":0,"type":"Person","about":1}],"sentiment":0},"title":{"sentiment":0}}}
+{"demographic":{"gender":"male"},"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/100002994995756\/picture","id":"100002994995756","link":"http:\/\/www.facebook.com\/profile.php?id=100002994995756","name":"Armando Leon"},"caption":"www.youtube.com","created_at":"Wed, 02 Oct 2013 23:52:36 +0000","description":"Producci\u00F3n General: Flow Gallery & Rap latino progress Talento: Mar\u00EDa Claudia Talento: Laura Talento: Mar\u00EDa Isabel Talento:Pierina Talento: Arianna Talento: ...","id":"100002994995756_432578163518680","link":"http:\/\/www.youtube.com\/watch?v=BKfeBHu4Yco","message":"Bellezas Caribe\u00F1as!!!\n\nhttp:\/\/www.youtube.com\/watch?v=BKfeBHu4Yco","name":"Apache P\u00F3nmela en el aire","source":"http:\/\/www.youtube.com\/v\/BKfeBHu4Yco?autohide=1&version=3&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/100002994995756\/picture","id":"100002994995756","link":"http:\/\/www.facebook.com\/profile.php?id=1000029949
95756","name":"Armando Leon"},"content":"Bellezas Caribe\u00F1as!!!\n\nhttp:\/\/www.youtube.com\/watch?v=BKfeBHu4Yco","created_at":"Wed, 02 Oct 2013 23:53:09 +0000","id":"1e32bbdb662faa00e0662770f91b1a0e","link":"http:\/\/www.facebook.com\/100002994995756_432578163518680","schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/BKfeBHu4Yco?autohide=1&version=3&autoplay=1","subtype":"video","title":"Apache P\u00F3nmela en el aire","type":"facebook"},"language":{"confidence":100,"tag":"es"},"links":{"code":[200],"created_at":["Wed, 02 Oct 2013 18:24:10 +0000"],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["Producci\u00F3n General: Flow Gallery & Rap latino progress Talento: Mar\u00EDa Claudia Talento: Laura Talento: Mar\u00EDa Isabel Talento:Pierina Talento: Arianna Talento: ..."],"keywords":[["apache","apache las minas","rap Latino","hip hip Latino","Latin rap","Latin hip hop","rap","Venezuela","hip hop","latino"]],"lang":["en"],"opengraph":[{"site_nam
e":"YouTube","type":"video","description":"Producci\u00F3n General: Flow Gallery & Rap latino progress Talento: Mar\u00EDa Claudia Talento: Laura Talento: Mar\u00EDa Isabel Talento:Pierina Talento: Arianna Talento: ...","url":"http:\/\/www.youtube.com\/watch?v=BKfeBHu4Yco","title":"Apache P\u00F3nmela en el aire","image":"http:\/\/i1.ytimg.com\/vi\/BKfeBHu4Yco\/maxresdefault.jpg?feature=og"}],"twitter":[{"player_width":"1920","description":"Producci\u00F3n General: Flow Gallery & Rap latino progress Talento: Mar\u00EDa Claudia Talento: Laura Talento: Mar\u00EDa Isabel Talento:Pierina Talento: Arianna Talento: ...","player_height":"1080","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/BKfeBHu4Yco","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/BKfeBHu4Yco","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=BKfeBHu4Yco","id":"com.google.android.youtube"}},"site":"@youtube","url":"http:\/\/
www.youtube.com\/watch?v=BKfeBHu4Yco","title":"Apache P\u00F3nmela en el aire","image":"http:\/\/i1.ytimg.com\/vi\/BKfeBHu4Yco\/maxresdefault.jpg","player":"https:\/\/www.youtube.com\/embed\/BKfeBHu4Yco"}]},"normalized_url":["http:\/\/youtube.com\/watch?v=BKfeBHu4Yco"],"retweet_count":[0],"title":["Apache P\u00F3nmela en el aire - YouTube"],"url":["http:\/\/www.youtube.com\/watch?v=BKfeBHu4Yco"]},"salience":{"content":{"sentiment":0},"title":{"sentiment":6}}}
+{"demographic":{"gender":"male"},"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/100002679352328\/picture","id":"100002679352328","link":"http:\/\/www.facebook.com\/profile.php?id=100002679352328","name":"Reinaldo Pinheiro"},"caption":"www.youtube.com","created_at":"Wed, 02 Oct 2013 23:54:57 +0000","description":"The Shadows Playing \"Apache\".","id":"100002679352328_476855475747093","link":"http:\/\/www.youtube.com\/watch?v=NoN6AKPGkBo","message":"http:\/\/www.youtube.com\/watch?v=NoN6AKPGkBo","name":"THE SHADOWS - Apache (1969)","source":"http:\/\/www.youtube.com\/v\/NoN6AKPGkBo?autohide=1&version=3&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/100002679352328\/picture","id":"100002679352328","link":"http:\/\/www.facebook.com\/profile.php?id=100002679352328","name":"Reinaldo Pinheiro"},"content":"http:\/\/www.youtube.com\/watch?v=NoN6AKPGkBo","created_at":"Wed, 02 Oct 2013 23:55:28 +0000","id":"1e32bbe0a6ddae80e0661367be
4e942e","link":"http:\/\/www.facebook.com\/100002679352328_476855475747093","received_at":4743578127336446991,"schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/NoN6AKPGkBo?autohide=1&version=3&autoplay=1","subtype":"video","title":"THE SHADOWS - Apache (1969)","type":"facebook"},"language":{"confidence":100,"tag":"en"},"links":{"code":[200],"created_at":["Wed, 02 Oct 2013 19:30:15 +0000"],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["The Shadows Playing \"Apache\"."],"keywords":[["THE","SHADOWS","APACHE","POP","GO","SIXTIES","GREAT","BRITAIN","GERMANY","MUSIC","PROGRAMME","1969","1960s"]],"lang":["en"],"opengraph":[{"site_name":"YouTube","type":"video","description":"The Shadows Playing \"Apache\".","url":"http:\/\/www.youtube.com\/watch?v=NoN6AKPGkBo","title":"THE SHADOWS - Apache (1969)","image":"http:\/\/i1.ytimg.com\/vi\/NoN6AKPGkBo\/hqdefault.jpg?feature=og"}],"twitter":[{"player_width":"640","description":"The Shadows Playing \"Apache\".
","player_height":"480","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/NoN6AKPGkBo","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/NoN6AKPGkBo","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=NoN6AKPGkBo","id":"com.google.android.youtube"}},"site":"@youtube","url":"http:\/\/www.youtube.com\/watch?v=NoN6AKPGkBo","title":"THE SHADOWS - Apache (1969)","image":"http:\/\/i1.ytimg.com\/vi\/NoN6AKPGkBo\/hqdefault.jpg","player":"https:\/\/www.youtube.com\/embed\/NoN6AKPGkBo"}]},"normalized_url":["http:\/\/youtube.com\/watch?v=NoN6AKPGkBo"],"retweet_count":[0],"title":["THE SHADOWS - Apache (1969) - YouTube"],"url":["http:\/\/www.youtube.com\/watch?v=NoN6AKPGkBo"]},"salience":{"content":{"sentiment":0},"title":{"sentiment":0}}}
+{"demographic":{"gender":"male"},"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/100002994995756\/picture","id":"100002994995756","link":"http:\/\/www.facebook.com\/profile.php?id=100002994995756","name":"Armando Leon"},"caption":"www.youtube.com","created_at":"Wed, 02 Oct 2013 23:59:36 +0000","description":"Apache y Canserbero - La del Estribo (Video Official) Apache y Canserbero - La del Estribo (Video Official) Apache y Canserbero - La del Estribo (Video Offic...","id":"100002994995756_432579833518513","link":"http:\/\/www.youtube.com\/watch?v=5h1KJ4Tf2sM","message":"DE mis favoritos de Venezuela Rifan machin !!!\n\nhttp:\/\/www.youtube.com\/watch?v=5h1KJ4Tf2sM","name":"Apache y Canserbero - La del Estribo (Video Official)","source":"http:\/\/www.youtube.com\/v\/5h1KJ4Tf2sM?autohide=1&version=3&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/100002994995756\/picture","id":"100002994995756","link":"http:\/\/www.facebook.co
m\/profile.php?id=100002994995756","name":"Armando Leon"},"content":"DE mis favoritos de Venezuela Rifan machin !!!\n\nhttp:\/\/www.youtube.com\/watch?v=5h1KJ4Tf2sM","created_at":"Wed, 02 Oct 2013 23:59:56 +0000","id":"1e32bbeb0b9ea400e06684cf3ac22de0","link":"http:\/\/www.facebook.com\/100002994995756_432579833518513","received_at":4743578128457765224,"schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/5h1KJ4Tf2sM?autohide=1&version=3&autoplay=1","subtype":"video","title":"Apache y Canserbero - La del Estribo (Video Official)","type":"facebook"},"language":{"confidence":100,"tag":"es"},"links":{"code":[200],"created_at":["Wed, 02 Oct 2013 19:46:43 +0000"],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["Apache y Canserbero - La del Estribo (Video Official) Apache y Canserbero - La del Estribo (Video Official) Apache y Canserbero - La del Estribo (Video Offic..."],"lang":["en"],"opengraph":[{"site_name":"YouTube","type":"video","description":"Apach
e y Canserbero - La del Estribo (Video Official) Apache y Canserbero - La del Estribo (Video Official) Apache y Canserbero - La del Estribo (Video Offic...","url":"http:\/\/www.youtube.com\/watch?v=5h1KJ4Tf2sM","title":"Apache y Canserbero - La del Estribo (Video Official)","image":"http:\/\/i1.ytimg.com\/vi\/5h1KJ4Tf2sM\/hqdefault.jpg?feature=og"}],"twitter":[{"player_width":"640","description":"Apache y Canserbero - La del Estribo (Video Official) Apache y Canserbero - La del Estribo (Video Official) Apache y Canserbero - La del Estribo (Video Offic...","player_height":"360","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/5h1KJ4Tf2sM","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/5h1KJ4Tf2sM","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=5h1KJ4Tf2sM","id":"com.google.android.youtube"}},"site":"@youtube","url":"http:\/\/www.youtube.com\/watch?v=5h1KJ4Tf2sM","title":"Apache y Canse
rbero - La del Estribo (Video Official)","image":"http:\/\/i1.ytimg.com\/vi\/5h1KJ4Tf2sM\/hqdefault.jpg","player":"https:\/\/www.youtube.com\/embed\/5h1KJ4Tf2sM"}]},"normalized_url":["http:\/\/youtube.com\/watch?v=5h1KJ4Tf2sM"],"retweet_count":[0],"title":["Apache y Canserbero - La del Estribo (Video Official) - YouTube"],"url":["http:\/\/www.youtube.com\/watch?v=5h1KJ4Tf2sM"]},"salience":{"content":{"sentiment":6},"title":{"sentiment":-3}}}
+{"demographic":{"gender":"unisex"},"facebook":{"application":"Links","author":{"avatar":"https:\/\/graph.facebook.com\/144929815716102\/picture","id":"144929815716102","link":"http:\/\/www.facebook.com\/profile.php?id=144929815716102","name":"Share If You Care"},"caption":"www.youtube.com","created_at":"Thu, 03 Oct 2013 00:04:59 +0000","description":"This Wikileaks footage shows Reuters journalist Namir Noor-Elden and her driver Saeed Chmagh being shot at by the Apache. The attack occurred in a public squ...","id":"144929815716102_150380198504397","link":"http:\/\/www.youtube.com\/watch?v=25EWUUBjPMo&feature=share&list=TLSvch4ZZmCXJ7vRZW2FrTANv6yPDaZ_QF","message":"Old footage, but if you haven't seen it here it will change your perspective. This is what America does in foreign countries. Unfortunately, to the rest for the world, WE ARE NOT THE GOOD GUYS. And this is why...","name":"WikiLeaks raw US Apache footage","source":"http:\/\/www.youtube.com\/v\/25EWUUBjPMo?version=3&autohid
e=1&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/144929815716102\/picture","id":"144929815716102","link":"http:\/\/www.facebook.com\/profile.php?id=144929815716102","name":"Share If You Care"},"content":"Old footage, but if you haven't seen it here it will change your perspective. This is what America does in foreign countries. Unfortunately, to the rest for the world, WE ARE NOT THE GOOD GUYS. And this is why...","created_at":"Thu, 03 Oct 2013 00:05:15 +0000","id":"1e32bbf713fca780e06655502b85c7be","link":"http:\/\/www.facebook.com\/144929815716102_150380198504397","received_at":4743578129797546718,"schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/25EWUUBjPMo?version=3&autohide=1&autoplay=1","subtype":"video","title":"WikiLeaks raw US Apache footage","type":"facebook"},"language":{"confidence":100,"tag":"en"},"links":{"code":[200],"created_at":["Thu, 03 Oct 2013 00:05:13 +0000"],"hops":[["http:\/\/www.youtube.com\/watch?v=
25EWUUBjPMo&feature=share&list=TLSvch4ZZmCXJ7vRZW2FrTANv6yPDaZ_QF"]],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["This Wikileaks footage shows Reuters journalist Namir Noor-Elden and her driver Saeed Chmagh being shot at by the Apache. The attack occurred in a public squ..."],"keywords":[["namir nor-elden","saeed chmagh","apache helicopter","wikileaks","us military","baghdad","adults","children","shooting","civilian casualties","murder","rt","russia today","..."]],"lang":["en"],"opengraph":[{"site_name":"YouTube","type":"video","description":"This Wikileaks footage shows Reuters journalist Namir Noor-Elden and her driver Saeed Chmagh being shot at by the Apache. The attack occurred in a public squ...","url":"http:\/\/www.youtube.com\/watch?v=25EWUUBjPMo","title":"WikiLeaks raw US Apache footage","image":"http:\/\/i1.ytimg.com\/vi\/25EWUUBjPMo\/hqdefault.jpg?feature=og"}],"twitter":[{"player_width":"640","description":"This Wikileaks footage shows Reuters
journalist Namir Noor-Elden and her driver Saeed Chmagh being shot at by the Apache. The attack occurred in a public squ...","player_height":"480","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/25EWUUBjPMo","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/25EWUUBjPMo","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=25EWUUBjPMo","id":"com.google.android.youtube"}},"site":"@youtube","url":"http:\/\/www.youtube.com\/watch?v=25EWUUBjPMo","title":"WikiLeaks raw US Apache footage","image":"http:\/\/i1.ytimg.com\/vi\/25EWUUBjPMo\/hqdefault.jpg","player":"https:\/\/www.youtube.com\/embed\/25EWUUBjPMo"}]},"normalized_url":["http:\/\/youtube.com\/watch?v=25EWUUBjPMo"],"retweet_count":[0],"title":["WikiLeaks raw US Apache footage - YouTube"],"url":["http:\/\/www.youtube.com\/watch?v=25EWUUBjPMo"]},"salience":{"content":{"sentiment":-4},"title":{"sentiment":0}}}
+{"demographic":{"gender":"mostly_male"},"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/100005956086899\/picture","id":"100005956086899","link":"http:\/\/www.facebook.com\/profile.php?id=100005956086899","name":"Juan Jose Henandez"},"caption":"www.youtube.com","created_at":"Thu, 03 Oct 2013 00:18:47 +0000","description":"Apache Ft. Canserbero - Stop (Con Letra) Disco : Apa y Can A\u00F1o: 2013 :::::Letra:::::: [Coro] stop stop dice la wiu wiu, alerta... esconde la yerba el mio, pe...","id":"100005956086899_157398494468686","link":"http:\/\/www.youtube.com\/watch?v=QueePm5hO4k","message":"http:\/\/www.youtube.com\/watch?v=QueePm5hO4k","name":"Apache Ft. Canserbero - Stop","source":"http:\/\/www.youtube.com\/v\/QueePm5hO4k?version=3&autohide=1&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/100005956086899\/picture","id":"100005956086899","link":"http:\/\/www.facebook.com\/profile.php?id=100005956086899","name":"Juan Jose Hena
ndez"},"content":"http:\/\/www.youtube.com\/watch?v=QueePm5hO4k","created_at":"Thu, 03 Oct 2013 00:19:30 +0000","id":"1e32bc15ec68ad80e0662011d177021c","link":"http:\/\/www.facebook.com\/100005956086899_157398494468686","received_at":4743578133383049170,"schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/QueePm5hO4k?version=3&autohide=1&autoplay=1","subtype":"video","title":"Apache Ft. Canserbero - Stop","type":"facebook"},"language":{"confidence":100,"tag":"en"},"links":{"code":[200],"created_at":["Thu, 03 Oct 2013 00:19:29 +0000"],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["Apache Ft. Canserbero - Stop (Con Letra) Disco : Apa y Can A\u00F1o: 2013 :::::Letra:::::: [Coro] stop stop dice la wiu wiu, alerta... esconde la yerba el mio, pe..."],"keywords":[["Canserbero - Stop","Apache - Stop","Canserbero 2013","Apa y Can","Apache 2013","Apache y canserbero Stop","Stop","Stop canserbero","lo mejor del canserbero","canserbe..."]],"lang":["en"],"open
graph":[{"site_name":"YouTube","type":"video","description":"Apache Ft. Canserbero - Stop (Con Letra) Disco : Apa y Can A\u00F1o: 2013 :::::Letra:::::: [Coro] stop stop dice la wiu wiu, alerta... esconde la yerba el mio, pe...","url":"http:\/\/www.youtube.com\/watch?v=QueePm5hO4k","title":"Apache Ft. Canserbero - Stop","image":"http:\/\/i1.ytimg.com\/vi\/QueePm5hO4k\/hqdefault.jpg?feature=og"}],"twitter":[{"player_width":"853","description":"Apache Ft. Canserbero - Stop (Con Letra) Disco : Apa y Can A\u00F1o: 2013 :::::Letra:::::: [Coro] stop stop dice la wiu wiu, alerta... esconde la yerba el mio, pe...","player_height":"480","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/QueePm5hO4k","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/QueePm5hO4k","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=QueePm5hO4k","id":"com.google.android.youtube"}},"site":"@youtube","url":"http:\/\/www.youtub
e.com\/watch?v=QueePm5hO4k","title":"Apache Ft. Canserbero - Stop","image":"http:\/\/i1.ytimg.com\/vi\/QueePm5hO4k\/hqdefault.jpg","player":"https:\/\/www.youtube.com\/embed\/QueePm5hO4k"}]},"normalized_url":["http:\/\/youtube.com\/watch?v=QueePm5hO4k"],"retweet_count":[0],"title":["Apache Ft. Canserbero - Stop - YouTube"],"url":["http:\/\/www.youtube.com\/watch?v=QueePm5hO4k"]},"salience":{"content":{"sentiment":0},"title":{"sentiment":0}}}
+{"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/100001444219466\/picture","id":"100001444219466","link":"http:\/\/www.facebook.com\/profile.php?id=100001444219466","name":"Enigmang Bio Hazard"},"caption":"www.youtube.com","created_at":"Thu, 03 Oct 2013 00:25:11 +0000","description":"Apache Ft. Canserbero - Stupid Love Story (Con Letra) Disco : Apa y Can A\u00F1o: 2013 :::::Letras::::: Detesto cuando pasa esto, tener un sue\u00F1o y sentir que fue ...","id":"100001444219466_589069114484557","link":"http:\/\/www.youtube.com\/watch?v=ZAgFOdv4knU","message":"http:\/\/www.youtube.com\/watch?v=ZAgFOdv4knU","name":"Apache Ft. Canserbero - Stupid Love Story","source":"http:\/\/www.youtube.com\/v\/ZAgFOdv4knU?autohide=1&version=3&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/100001444219466\/picture","id":"100001444219466","link":"http:\/\/www.facebook.com\/profile.php?id=100001444219466","name":"Enigmang Bio Hazard"},"content":"h
ttp:\/\/www.youtube.com\/watch?v=ZAgFOdv4knU","created_at":"Thu, 03 Oct 2013 00:25:51 +0000","id":"1e32bc243a84ad80e0669d67bed7553a","link":"http:\/\/www.facebook.com\/100001444219466_589069114484557","schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/ZAgFOdv4knU?autohide=1&version=3&autoplay=1","subtype":"video","title":"Apache Ft. Canserbero - Stupid Love Story","type":"facebook"},"language":{"confidence":100,"tag":"en"},"links":{"code":[200],"created_at":["Wed, 02 Oct 2013 18:20:51 +0000"],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["Apache Ft. Canserbero - Stupid Love Story (Con Letra) Disco : Apa y Can A\u00F1o: 2013 :::::Letras::::: Detesto cuando pasa esto, tener un sue\u00F1o y sentir que fue ..."],"keywords":[["Canserbero - Stupid love story","Apache - Stupid Love Story","Apache 2013","Canserbero 2013"]],"lang":["en"],"opengraph":[{"site_name":"YouTube","type":"video","description":"Apache Ft. Canserbero - Stupid Love Story (Con Letra
) Disco : Apa y Can A\u00F1o: 2013 :::::Letras::::: Detesto cuando pasa esto, tener un sue\u00F1o y sentir que fue ...","url":"http:\/\/www.youtube.com\/watch?v=ZAgFOdv4knU","title":"Apache Ft. Canserbero - Stupid Love Story","image":"http:\/\/i1.ytimg.com\/vi\/ZAgFOdv4knU\/hqdefault.jpg?feature=og"}],"twitter":[{"player_width":"853","description":"Apache Ft. Canserbero - Stupid Love Story (Con Letra) Disco : Apa y Can A\u00F1o: 2013 :::::Letras::::: Detesto cuando pasa esto, tener un sue\u00F1o y sentir que fue ...","player_height":"480","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/ZAgFOdv4knU","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/ZAgFOdv4knU","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=ZAgFOdv4knU","id":"com.google.android.youtube"}},"site":"@youtube","url":"http:\/\/www.youtube.com\/watch?v=ZAgFOdv4knU","title":"Apache Ft. Canserbero - Stupid Love Story","image":"h
ttp:\/\/i1.ytimg.com\/vi\/ZAgFOdv4knU\/hqdefault.jpg","player":"https:\/\/www.youtube.com\/embed\/ZAgFOdv4knU"}]},"normalized_url":["http:\/\/youtube.com\/watch?v=ZAgFOdv4knU"],"retweet_count":[0],"title":["Apache Ft. Canserbero - Stupid Love Story - YouTube"],"url":["http:\/\/www.youtube.com\/watch?v=ZAgFOdv4knU"]},"salience":{"content":{"sentiment":0},"title":{"sentiment":0}}}
+{"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/100001611980796\/picture","id":"100001611980796","link":"http:\/\/www.facebook.com\/profile.php?id=100001611980796","name":"Royer Grefa"},"caption":"www.youtube.com","created_at":"Thu, 03 Oct 2013 00:29:14 +0000","description":"STUPID LOVE STORY - APACHE Y CANSERBERO extraido del \u00E1lbum: APA Y CAN","id":"100001611980796_591199984277062","link":"http:\/\/www.youtube.com\/watch?v=0BBOde0Rl1I","message":"Apa Y Can\n\nhttp:\/\/www.youtube.com\/watch?v=0BBOde0Rl1I","name":"STUPID LOVE STORY - APACHE Y CANSERBERO ( LETRA ) 1080p","source":"http:\/\/www.youtube.com\/v\/0BBOde0Rl1I?version=3&autohide=1&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/100001611980796\/picture","id":"100001611980796","link":"http:\/\/www.facebook.com\/profile.php?id=100001611980796","name":"Royer Grefa"},"content":"Apa Y Can\n\nhttp:\/\/www.youtube.com\/watch?v=0BBOde0Rl1I","created_at":"Thu, 03 Oct
2013 00:29:48 +0000","id":"1e32bc2d47f2a900e066d78603ba84c8","link":"http:\/\/www.facebook.com\/100001611980796_591199984277062","schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/0BBOde0Rl1I?version=3&autohide=1&autoplay=1","subtype":"video","title":"STUPID LOVE STORY - APACHE Y CANSERBERO ( LETRA ) 1080p","type":"facebook"},"language":{"confidence":100,"tag":"en"},"links":{"code":[200],"created_at":["Thu, 03 Oct 2013 00:29:54 +0000"],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["STUPID LOVE STORY - APACHE Y CANSERBERO extraido del \u00E1lbum: APA Y CAN"],"keywords":[["STUPID LOVE STORY","APACHE","CANSERBERO","APACHE Y CANSERBERO","LETRA","HD","Lyrics","Full","Story"]],"lang":["en"],"opengraph":[{"site_name":"YouTube","type":"video","description":"STUPID LOVE STORY - APACHE Y CANSERBERO extraido del \u00E1lbum: APA Y CAN","url":"http:\/\/www.youtube.com\/watch?v=0BBOde0Rl1I","title":"STUPID LOVE STORY - APACHE Y CANSERBERO ( LETRA ) 1080p","im
age":"http:\/\/i1.ytimg.com\/vi\/0BBOde0Rl1I\/maxresdefault.jpg?feature=og"}],"twitter":[{"player_width":"1440","description":"STUPID LOVE STORY - APACHE Y CANSERBERO extraido del \u00E1lbum: APA Y CAN","player_height":"1080","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/0BBOde0Rl1I","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/0BBOde0Rl1I","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=0BBOde0Rl1I","id":"com.google.android.youtube"}},"site":"@youtube","url":"http:\/\/www.youtube.com\/watch?v=0BBOde0Rl1I","title":"STUPID LOVE STORY - APACHE Y CANSERBERO ( LETRA ) 1080p","image":"http:\/\/i1.ytimg.com\/vi\/0BBOde0Rl1I\/maxresdefault.jpg","player":"https:\/\/www.youtube.com\/embed\/0BBOde0Rl1I"}]},"normalized_url":["http:\/\/youtube.com\/watch?v=0BBOde0Rl1I"],"retweet_count":[0],"title":["STUPID LOVE STORY - APACHE Y CANSERBERO ( LETRA ) 1080p - YouTube"],"url":["http:\/\/www.youtu
be.com\/watch?v=0BBOde0Rl1I"]},"salience":{"content":{"sentiment":0},"title":{"sentiment":-4}}}
+{"demographic":{"gender":"male"},"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/100002033189713\/picture","id":"100002033189713","link":"http:\/\/www.facebook.com\/profile.php?id=100002033189713","name":"Jesus Ibarra Pino"},"caption":"www.youtube.com","created_at":"Thu, 03 Oct 2013 00:32:28 +0000","description":"Apache Ft. Canserbero - Uno Por Ellas (Con Letra ) Disco : Apa y Can A\u00F1o: 2013 :::::::::::Letras:::::::::::::::::::::: Y es que se acostumbran a salir con tr...","id":"100002033189713_536153523129082","link":"https:\/\/www.youtube.com\/watch?v=fI9Y-nH5yIw","message":"https:\/\/www.youtube.com\/watch?v=fI9Y-nH5yIw","name":"Apache Ft. Canserbero - Uno Por Ellas","source":"http:\/\/www.youtube.com\/v\/fI9Y-nH5yIw?autohide=1&version=3&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/100002033189713\/picture","id":"100002033189713","link":"http:\/\/www.facebook.com\/profile.php?id=100002033189713","name":"Jesus Ibarr
a Pino"},"content":"https:\/\/www.youtube.com\/watch?v=fI9Y-nH5yIw","created_at":"Thu, 03 Oct 2013 00:32:52 +0000","id":"1e32bc348213a600e06698e907c33c3a","link":"http:\/\/www.facebook.com\/100002033189713_536153523129082","schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/fI9Y-nH5yIw?autohide=1&version=3&autoplay=1","subtype":"video","title":"Apache Ft. Canserbero - Uno Por Ellas","type":"facebook"},"language":{"confidence":100,"tag":"en"},"links":{"code":[200],"created_at":["Thu, 03 Oct 2013 00:32:50 +0000"],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["Apache Ft. Canserbero - Uno Por Ellas (Con Letra ) Disco : Apa y Can A\u00F1o: 2013 :::::::::::Letras:::::::::::::::::::::: Y es que se acostumbran a salir con tr..."],"keywords":[["Canserbero - Uno por ellas","Apache - uno por ellas","Apache 2013","Canserbero 2013","lo mejor de canserbero","Stop Canserbero","Agradezco Canserbero","rap 2013 ven..."]],"lang":["en"],"opengraph":[{"site_name":"Yo
uTube","type":"video","description":"Apache Ft. Canserbero - Uno Por Ellas (Con Letra ) Disco : Apa y Can A\u00F1o: 2013 :::::::::::Letras:::::::::::::::::::::: Y es que se acostumbran a salir con tr...","url":"http:\/\/www.youtube.com\/watch?v=fI9Y-nH5yIw","title":"Apache Ft. Canserbero - Uno Por Ellas","image":"https:\/\/i1.ytimg.com\/vi\/fI9Y-nH5yIw\/hqdefault.jpg?feature=og"}],"twitter":[{"player_width":"853","description":"Apache Ft. Canserbero - Uno Por Ellas (Con Letra ) Disco : Apa y Can A\u00F1o: 2013 :::::::::::Letras:::::::::::::::::::::: Y es que se acostumbran a salir con tr...","player_height":"480","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/fI9Y-nH5yIw","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/fI9Y-nH5yIw","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=fI9Y-nH5yIw","id":"com.google.android.youtube"}},"site":"@youtube","url":"http:\/\/www.youtube.com\/watch?v
=fI9Y-nH5yIw","title":"Apache Ft. Canserbero - Uno Por Ellas","image":"https:\/\/i1.ytimg.com\/vi\/fI9Y-nH5yIw\/hqdefault.jpg","player":"https:\/\/www.youtube.com\/embed\/fI9Y-nH5yIw"}]},"normalized_url":["https:\/\/youtube.com\/watch?v=fI9Y-nH5yIw"],"retweet_count":[0],"title":["Apache Ft. Canserbero - Uno Por Ellas - YouTube"],"url":["https:\/\/www.youtube.com\/watch?v=fI9Y-nH5yIw"]},"salience":{"title":{"sentiment":0}}}
+{"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/100001611980796\/picture","id":"100001611980796","link":"http:\/\/www.facebook.com\/profile.php?id=100001611980796","name":"Royer Grefa"},"caption":"www.youtube.com","created_at":"Thu, 03 Oct 2013 00:34:21 +0000","description":"Apache y Canserbero - La del Estribo (Video Official) Apache y Canserbero - La del Estribo (Video Official) Apache y Canserbero - La del Estribo (Video Offic...","id":"100001611980796_591201640943563","link":"http:\/\/www.youtube.com\/watch?v=5h1KJ4Tf2sM","message":"http:\/\/www.youtube.com\/watch?v=5h1KJ4Tf2sM","name":"Apache y Canserbero - La del Estribo (Video Official)","source":"http:\/\/www.youtube.com\/v\/5h1KJ4Tf2sM?autohide=1&version=3&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/100001611980796\/picture","id":"100001611980796","link":"http:\/\/www.facebook.com\/profile.php?id=100001611980796","name":"Royer Grefa"},"content":"http:\/\/www.yo
utube.com\/watch?v=5h1KJ4Tf2sM","created_at":"Thu, 03 Oct 2013 00:34:50 +0000","id":"1e32bc38b7baa480e066dfcb884577a8","link":"http:\/\/www.facebook.com\/100001611980796_591201640943563","schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/5h1KJ4Tf2sM?autohide=1&version=3&autoplay=1","subtype":"video","title":"Apache y Canserbero - La del Estribo (Video Official)","type":"facebook"},"language":{"confidence":100,"tag":"en"},"links":{"code":[200],"created_at":["Wed, 02 Oct 2013 19:46:43 +0000"],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["Apache y Canserbero - La del Estribo (Video Official) Apache y Canserbero - La del Estribo (Video Official) Apache y Canserbero - La del Estribo (Video Offic..."],"lang":["en"],"opengraph":[{"site_name":"YouTube","type":"video","description":"Apache y Canserbero - La del Estribo (Video Official) Apache y Canserbero - La del Estribo (Video Official) Apache y Canserbero - La del Estribo (Video Offic...","url":"http
:\/\/www.youtube.com\/watch?v=5h1KJ4Tf2sM","title":"Apache y Canserbero - La del Estribo (Video Official)","image":"http:\/\/i1.ytimg.com\/vi\/5h1KJ4Tf2sM\/hqdefault.jpg?feature=og"}],"twitter":[{"player_width":"640","description":"Apache y Canserbero - La del Estribo (Video Official) Apache y Canserbero - La del Estribo (Video Official) Apache y Canserbero - La del Estribo (Video Offic...","player_height":"360","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/5h1KJ4Tf2sM","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/5h1KJ4Tf2sM","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=5h1KJ4Tf2sM","id":"com.google.android.youtube"}},"site":"@youtube","url":"http:\/\/www.youtube.com\/watch?v=5h1KJ4Tf2sM","title":"Apache y Canserbero - La del Estribo (Video Official)","image":"http:\/\/i1.ytimg.com\/vi\/5h1KJ4Tf2sM\/hqdefault.jpg","player":"https:\/\/www.youtube.com\/embed\/5h1KJ4Tf2sM"}]},"no
rmalized_url":["http:\/\/youtube.com\/watch?v=5h1KJ4Tf2sM"],"retweet_count":[0],"title":["Apache y Canserbero - La del Estribo (Video Official) - YouTube"],"url":["http:\/\/www.youtube.com\/watch?v=5h1KJ4Tf2sM"]},"salience":{"content":{"sentiment":0},"title":{"entities":[{"name":"Apache","sentiment":0,"confident":1,"label":"Company","evidence":1,"type":"Company","about":1}],"sentiment":0,"topics":[{"name":"Video Games","score":0.49648976325989,"hits":0}]}}}
+{"demographic":{"gender":"unisex"},"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/149835201830545\/picture","id":"149835201830545","link":"http:\/\/www.facebook.com\/profile.php?id=149835201830545","name":"EH. Enyerbeth Hern\u00E1ndez"},"caption":"www.youtube.com","created_at":"Thu, 03 Oct 2013 00:39:01 +0000","description":"Producci\u00F3n General: Flow Gallery & Rap latino progress Talento: Mar\u00EDa Claudia Talento: Laura Talento: Mar\u00EDa Isabel Talento:Pierina Talento: Arianna Talento: ...","id":"149835201830545_248071985340199","link":"http:\/\/www.youtube.com\/watch?v=BKfeBHu4Yco","message":"La Track del D\u00EDa. #Apache\n\nListen to it!! >>>","name":"Apache P\u00F3nmela en el aire","source":"http:\/\/www.youtube.com\/v\/BKfeBHu4Yco?autohide=1&version=3&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/149835201830545\/picture","id":"149835201830545","link":"http:\/\/www.facebook.com\/profile.php?id=149835201830545
","name":"EH. Enyerbeth Hern\u00E1ndez"},"content":"La Track del D\u00EDa. #Apache\n\nListen to it!! >>>","created_at":"Thu, 03 Oct 2013 00:39:27 +0000","hashtags":["#Apache"],"id":"1e32bc432604a080e066aa71ea3dba02","link":"http:\/\/www.facebook.com\/149835201830545_248071985340199","schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/BKfeBHu4Yco?autohide=1&version=3&autoplay=1","subtype":"video","title":"Apache P\u00F3nmela en el aire","type":"facebook"},"language":{"confidence":63,"tag":"en"},"links":{"code":[200],"created_at":["Wed, 02 Oct 2013 18:24:10 +0000"],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["Producci\u00F3n General: Flow Gallery & Rap latino progress Talento: Mar\u00EDa Claudia Talento: Laura Talento: Mar\u00EDa Isabel Talento:Pierina Talento: Arianna Talento: ..."],"keywords":[["apache","apache las minas","rap Latino","hip hip Latino","Latin rap","Latin hip hop","rap","Venezuela","hip hop","latino"]],"lang":["en"],"opengraph":[
{"site_name":"YouTube","type":"video","description":"Producci\u00F3n General: Flow Gallery & Rap latino progress Talento: Mar\u00EDa Claudia Talento: Laura Talento: Mar\u00EDa Isabel Talento:Pierina Talento: Arianna Talento: ...","url":"http:\/\/www.youtube.com\/watch?v=BKfeBHu4Yco","title":"Apache P\u00F3nmela en el aire","image":"http:\/\/i1.ytimg.com\/vi\/BKfeBHu4Yco\/maxresdefault.jpg?feature=og"}],"twitter":[{"player_width":"1920","description":"Producci\u00F3n General: Flow Gallery & Rap latino progress Talento: Mar\u00EDa Claudia Talento: Laura Talento: Mar\u00EDa Isabel Talento:Pierina Talento: Arianna Talento: ...","player_height":"1080","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/BKfeBHu4Yco","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/BKfeBHu4Yco","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=BKfeBHu4Yco","id":"com.google.android.youtube"}},"site":"@youtube","url":
"http:\/\/www.youtube.com\/watch?v=BKfeBHu4Yco","title":"Apache P\u00F3nmela en el aire","image":"http:\/\/i1.ytimg.com\/vi\/BKfeBHu4Yco\/maxresdefault.jpg","player":"https:\/\/www.youtube.com\/embed\/BKfeBHu4Yco"}]},"normalized_url":["http:\/\/youtube.com\/watch?v=BKfeBHu4Yco"],"retweet_count":[0],"title":["Apache P\u00F3nmela en el aire - YouTube"],"url":["http:\/\/www.youtube.com\/watch?v=BKfeBHu4Yco"]},"salience":{"content":{"sentiment":0},"title":{"sentiment":0}}}
+{"demographic":{"gender":"unisex"},"facebook":{"application":"Share_bookmarklet","author":{"avatar":"https:\/\/graph.facebook.com\/100001288606187\/picture","id":"100001288606187","link":"http:\/\/www.facebook.com\/profile.php?id=100001288606187","name":"Willian Rom\u00E1n Castillo"},"caption":"www.youtube.com","created_at":"Thu, 03 Oct 2013 00:47:38 +0000","description":"Apache Ft. Canserbero - Uno Por Ellas (Con Letra ) Disco : Apa y Can A\u00F1o: 2013 :::::::::::Letras:::::::::::::::::::::: Y es que se acostumbran a salir con tr...","id":"100001288606187_639059829480298","link":"http:\/\/www.youtube.com\/watch?v=fI9Y-nH5yIw&sns=fb","message":"para las chiks..","name":"Apache Ft. Canserbero - Uno Por Ellas","source":"http:\/\/www.youtube.com\/v\/fI9Y-nH5yIw?autohide=1&version=3&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/100001288606187\/picture","id":"100001288606187","link":"http:\/\/www.facebook.com\/profile.php?id=10000128860618
7","name":"Willian Rom\u00E1n Castillo"},"content":"para las chiks..","created_at":"Thu, 03 Oct 2013 00:48:10 +0000","id":"1e32bc566883a100e06607df280bf16c","link":"http:\/\/www.facebook.com\/100001288606187_639059829480298","schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/fI9Y-nH5yIw?autohide=1&version=3&autoplay=1","subtype":"video","title":"Apache Ft. Canserbero - Uno Por Ellas","type":"facebook"},"language":{"confidence":64,"tag":"et"},"links":{"code":[200],"created_at":["Wed, 02 Oct 2013 19:39:13 +0000"],"hops":[["http:\/\/www.youtube.com\/watch?v=fI9Y-nH5yIw&sns=fb"]],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["Apache Ft. Canserbero - Uno Por Ellas (Con Letra ) Disco : Apa y Can A\u00F1o: 2013 :::::::::::Letras:::::::::::::::::::::: Y es que se acostumbran a salir con tr..."],"keywords":[["Canserbero - Uno por ellas","Apache - uno por ellas","Apache 2013","Canserbero 2013","lo mejor de canserbero","Stop Canserbero","Agradezco Canserbe
ro","rap 2013 ven..."]],"lang":["en"],"opengraph":[{"site_name":"YouTube","type":"video","description":"Apache Ft. Canserbero - Uno Por Ellas (Con Letra ) Disco : Apa y Can A\u00F1o: 2013 :::::::::::Letras:::::::::::::::::::::: Y es que se acostumbran a salir con tr...","url":"http:\/\/www.youtube.com\/watch?v=fI9Y-nH5yIw","title":"Apache Ft. Canserbero - Uno Por Ellas","image":"http:\/\/i1.ytimg.com\/vi\/fI9Y-nH5yIw\/hqdefault.jpg?feature=og"}],"twitter":[{"player_width":"853","description":"Apache Ft. Canserbero - Uno Por Ellas (Con Letra ) Disco : Apa y Can A\u00F1o: 2013 :::::::::::Letras:::::::::::::::::::::: Y es que se acostumbran a salir con tr...","player_height":"480","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/fI9Y-nH5yIw","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/fI9Y-nH5yIw","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=fI9Y-nH5yIw","id":"com.google.android.you
tube"}},"site":"@youtube","url":"http:\/\/www.youtube.com\/watch?v=fI9Y-nH5yIw","title":"Apache Ft. Canserbero - Uno Por Ellas","image":"http:\/\/i1.ytimg.com\/vi\/fI9Y-nH5yIw\/hqdefault.jpg","player":"https:\/\/www.youtube.com\/embed\/fI9Y-nH5yIw"}]},"normalized_url":["http:\/\/youtube.com\/watch?v=fI9Y-nH5yIw"],"retweet_count":[0],"title":["Apache Ft. Canserbero - Uno Por Ellas - YouTube"],"url":["http:\/\/www.youtube.com\/watch?v=fI9Y-nH5yIw"]}}
+{"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/100000529773118\/picture","id":"100000529773118","link":"http:\/\/www.facebook.com\/profile.php?id=100000529773118","name":"Ghatholl Perkasa"},"caption":"jakartagreater.com","created_at":"Thu, 03 Oct 2013 00:48:32 +0000","description":"Pangdam VI Mulawarman menjelaskan, tahun 2014 mendapatkan dukungan 2 unit tank Leopard-2, serta 2 unit Helikopter Apache, menemani Helikopter Bell 412 EP","id":"100000529773118_709380572422929","link":"http:\/\/jakartagreater.com\/helikopter-apache-dan-tank-leopard-temani-bell-412-ep-di-kalimantan\/","message":"http:\/\/jakartagreater.com\/helikopter-apache-dan-tank-leopard-temani-bell-412-ep-di-kalimantan\/","name":"Helikopter Apache dan Tank Leopard Temani Bell 412 EP di Kalimantan | JakartaGreater","source":"web","type":"link"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/100000529773118\/picture","id":"100000529773118","link":"http:\/\/www.facebook.com\/profile.php?
id=100000529773118","name":"Ghatholl Perkasa"},"content":"http:\/\/jakartagreater.com\/helikopter-apache-dan-tank-leopard-temani-bell-412-ep-di-kalimantan\/","created_at":"Thu, 03 Oct 2013 00:49:02 +0000","id":"1e32bc586b7fa000e066c7349fcb4420","link":"http:\/\/www.facebook.com\/100000529773118_709380572422929","schema":{"version":3},"source":"web","subtype":"link","title":"Helikopter Apache dan Tank Leopard Temani Bell 412 EP di Kalimantan | JakartaGreater","type":"facebook"},"links":{"code":[200],"created_at":["Wed, 02 Oct 2013 14:42:46 +0000"],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["Pangdam VI Mulawarman menjelaskan, tahun 2014 mendapatkan dukungan 2 unit tank Leopard-2, serta 2 unit Helikopter Apache, menemani Helikopter Bell 412 EP"],"keywords":[["Helikopter Apache Kalimantan"]],"lang":["en-us"]},"normalized_url":["http:\/\/jakartagreater.com\/helikopter-apache-dan-tank-leopard-temani-bell-412-ep-di-kalimantan"],"retweet_count":[0],"title":["He
likopter Apache dan Tank Leopard Temani Bell 412 EP di Kalimantan | JakartaGreater"],"url":["http:\/\/jakartagreater.com\/helikopter-apache-dan-tank-leopard-temani-bell-412-ep-di-kalimantan\/"]},"salience":{"content":{"sentiment":0},"title":{"sentiment":0}}}
+{"interaction":{"author":{"link":"http:\/\/youtube.com\/y_4c0uy7ikisinxqjnquva","name":"y_4c0uy7ikisinxqjnquva"},"content":"comenten y subo otros.","contenttype":"html","created_at":"Thu, 03 Oct 2013 01:54:22 +0000","id":"1e32bc7c8499aa80e0612c55f4551676","link":"http:\/\/www.youtube.com\/watch?v=Yj6ckGTNJ7M","schema":{"version":3},"title":"FA! Fuerte Apache NO TE CONFUNDAS 2013] DEMO","type":"youtube"},"language":{"confidence":62,"tag":"es"},"salience":{"content":{"sentiment":7},"title":{"sentiment":0}},"youtube":{"author":{"link":"http:\/\/youtube.com\/y_4c0uy7ikisinxqjnquva","name":"y_4c0uy7ikisinxqjnquva"},"category":"People & Blogs","content":"comenten y subo otros.","contenttype":"html","created_at":"Thu, 03 Oct 2013 01:04:41 +0000","duration":"219","id":"1e32bc7c8499aa80e0612c55f4551676","title":"FA! Fuerte Apache NO TE CONFUNDAS 2013] DEMO","type":"video","videolink":"http:\/\/www.youtube.com\/watch?v=Yj6ckGTNJ7M"}}
+{"interaction":{"author":{"link":"http:\/\/youtube.com\/thercelement","name":"thercelement"},"contenttype":"html","created_at":"Thu, 03 Oct 2013 18:57:23 +0000","id":"1e32bc845750af80e061657d25bd1b16","link":"http:\/\/www.youtube.com\/watch?v=wMthG7WdQtI","schema":{"version":3},"title":"HPI Apache C1 Flux Update - Smoked ESC","type":"youtube"},"salience":{"title":{"sentiment":0,"topics":[{"name":"Economics","score":0.46346464753151,"hits":0}]}},"youtube":{"author":{"link":"http:\/\/youtube.com\/thercelement","name":"thercelement"},"category":"entertainment","contenttype":"html","created_at":"Thu, 03 Oct 2013 01:08:11 +0000","duration":"251","id":"1e32bc845750af80e061657d25bd1b16","title":"HPI Apache C1 Flux Update - Smoked ESC","type":"video","videolink":"http:\/\/www.youtube.com\/watch?v=wMthG7WdQtI"}}
+{"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/100005863352468\/picture","id":"100005863352468","link":"http:\/\/www.facebook.com\/profile.php?id=100005863352468","name":"Rolocris Tonato Veloz"},"caption":"www.youtube.com","created_at":"Thu, 03 Oct 2013 01:20:53 +0000","description":"del barrio mas pobre y marginal nace esta cancion dedicado a los amigos que no estan ya en este mundo pero en la gloria ya se encuentran","id":"100005863352468_166572870214824","link":"http:\/\/www.youtube.com\/watch?v=7n0vyad0Mlk","message":"http:\/\/www.youtube.com\/watch?v=7n0vyad0Mlk","name":"f a fuerte apache queridos amigos video oficial","source":"http:\/\/www.youtube.com\/v\/7n0vyad0Mlk?version=3&autohide=1&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/100005863352468\/picture","id":"100005863352468","link":"http:\/\/www.facebook.com\/profile.php?id=100005863352468","name":"Rolocris Tonato Veloz"},"content":"http:\/\/www.youtube.com\
/watch?v=7n0vyad0Mlk","created_at":"Thu, 03 Oct 2013 01:21:14 +0000","id":"1e32bca0ba50a880e066e164e0b75830","link":"http:\/\/www.facebook.com\/100005863352468_166572870214824","schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/7n0vyad0Mlk?version=3&autohide=1&autoplay=1","subtype":"video","title":"f a fuerte apache queridos amigos video oficial","type":"facebook"},"language":{"confidence":100,"tag":"en"},"links":{"code":[200],"created_at":["Wed, 02 Oct 2013 22:23:15 +0000"],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["del barrio mas pobre y marginal nace esta cancion dedicado a los amigos que no estan ya en este mundo pero en la gloria ya se encuentran"],"keywords":[["fuerte","apache","argentina","barrios","bajos","muerte","tevez","amigos"]],"lang":["en"],"opengraph":[{"site_name":"YouTube","type":"video","description":"del barrio mas pobre y marginal nace esta cancion dedicado a los amigos que no estan ya en este mundo pero en la gloria ya s
e encuentran","url":"http:\/\/www.youtube.com\/watch?v=7n0vyad0Mlk","title":"f a fuerte apache queridos amigos video oficial","image":"http:\/\/i1.ytimg.com\/vi\/7n0vyad0Mlk\/hqdefault.jpg?feature=og"}],"twitter":[{"player_width":"480","description":"del barrio mas pobre y marginal nace esta cancion dedicado a los amigos que no estan ya en este mundo pero en la gloria ya se encuentran","player_height":"360","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/7n0vyad0Mlk","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/7n0vyad0Mlk","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=7n0vyad0Mlk","id":"com.google.android.youtube"}},"site":"@youtube","url":"http:\/\/www.youtube.com\/watch?v=7n0vyad0Mlk","title":"f a fuerte apache queridos amigos video oficial","image":"http:\/\/i1.ytimg.com\/vi\/7n0vyad0Mlk\/hqdefault.jpg","player":"https:\/\/www.youtube.com\/embed\/7n0vyad0Mlk"}]},"normalized_ur
l":["http:\/\/youtube.com\/watch?v=7n0vyad0Mlk"],"retweet_count":[0],"title":["f a fuerte apache queridos amigos video oficial - YouTube"],"url":["http:\/\/www.youtube.com\/watch?v=7n0vyad0Mlk"]},"salience":{"content":{"sentiment":0},"title":{"sentiment":0,"topics":[{"name":"Video Games","score":0.51537579298019,"hits":0}]}}}
+{"interaction":{"author":{"link":"http:\/\/youtube.com\/2fz96yvwuf7mhxsxcwqdkw","name":"2fz96yvwuf7mhxsxcwqdkw"},"content":"This Article Austin Apache is composed of Creative Common Content. The Original Article can be location at WikiPedia.org. ref length width platform layout FF...","contenttype":"html","created_at":"Thu, 03 Oct 2013 05:18:21 +0000","id":"1e32bca152e7a080e061b93f1aebbc1e","link":"http:\/\/www.youtube.com\/watch?v=8tnaFTMm05w","schema":{"version":3},"title":"Austin Apache","type":"youtube"},"salience":{"content":{"sentiment":4},"title":{"sentiment":0}},"youtube":{"author":{"link":"http:\/\/youtube.com\/2fz96yvwuf7mhxsxcwqdkw","name":"2fz96yvwuf7mhxsxcwqdkw"},"category":"education","content":"This Article Austin Apache is composed of Creative Common Content. The Original Article can be location at WikiPedia.org. ref length width platform layout FF...","contenttype":"html","created_at":"Thu, 03 Oct 2013 01:21:09 +0000","duration":"39","id":"1e32bca152e7a080e061b93f1a
ebbc1e","title":"Austin Apache","type":"video","videolink":"http:\/\/www.youtube.com\/watch?v=8tnaFTMm05w"}}
+{"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/100002670373913\/picture","id":"100002670373913","link":"http:\/\/www.facebook.com\/profile.php?id=100002670373913","name":"Jhonatan ZoUnd"},"caption":"www.youtube.com","created_at":"Thu, 03 Oct 2013 01:23:46 +0000","description":"Apache Ft. Canserbero - Uno Por Ellas (Con Letra ) Disco : Apa y Can A\u00F1o: 2013 :::::::::::Letras:::::::::::::::::::::: Y es que se acostumbran a salir con tr...","id":"100002670373913_476818342417174","link":"http:\/\/www.youtube.com\/watch?v=fI9Y-nH5yIw","message":"<3 Oh baby listen now","name":"Apache Ft. Canserbero - Uno Por Ellas","source":"http:\/\/www.youtube.com\/v\/fI9Y-nH5yIw?version=3&autohide=1&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/100002670373913\/picture","id":"100002670373913","link":"http:\/\/www.facebook.com\/profile.php?id=100002670373913","name":"Jhonatan ZoUnd"},"content":"<3 Oh baby listen now","created_at":"Thu, 03
Oct 2013 01:24:08 +0000","id":"1e32bca72c2bad00e0669ad95907cc90","link":"http:\/\/www.facebook.com\/100002670373913_476818342417174","received_at":4743578149649965593,"schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/fI9Y-nH5yIw?version=3&autohide=1&autoplay=1","subtype":"video","title":"Apache Ft. Canserbero - Uno Por Ellas","type":"facebook"},"links":{"code":[200],"created_at":["Wed, 02 Oct 2013 19:39:13 +0000"],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["Apache Ft. Canserbero - Uno Por Ellas (Con Letra ) Disco : Apa y Can A\u00F1o: 2013 :::::::::::Letras:::::::::::::::::::::: Y es que se acostumbran a salir con tr..."],"keywords":[["Canserbero - Uno por ellas","Apache - uno por ellas","Apache 2013","Canserbero 2013","lo mejor de canserbero","Stop Canserbero","Agradezco Canserbero","rap 2013 ven..."]],"lang":["en"],"opengraph":[{"site_name":"YouTube","type":"video","description":"Apache Ft. Canserbero - Uno Por Ellas (Con Letra ) Disco :
Apa y Can A\u00F1o: 2013 :::::::::::Letras:::::::::::::::::::::: Y es que se acostumbran a salir con tr...","url":"http:\/\/www.youtube.com\/watch?v=fI9Y-nH5yIw","title":"Apache Ft. Canserbero - Uno Por Ellas","image":"http:\/\/i1.ytimg.com\/vi\/fI9Y-nH5yIw\/hqdefault.jpg?feature=og"}],"twitter":[{"player_width":"853","description":"Apache Ft. Canserbero - Uno Por Ellas (Con Letra ) Disco : Apa y Can A\u00F1o: 2013 :::::::::::Letras:::::::::::::::::::::: Y es que se acostumbran a salir con tr...","player_height":"480","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/fI9Y-nH5yIw","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/fI9Y-nH5yIw","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=fI9Y-nH5yIw","id":"com.google.android.youtube"}},"site":"@youtube","url":"http:\/\/www.youtube.com\/watch?v=fI9Y-nH5yIw","title":"Apache Ft. Canserbero - Uno Por Ellas","image":"http:\/\/i1.ytimg.com\/vi\
/fI9Y-nH5yIw\/hqdefault.jpg","player":"https:\/\/www.youtube.com\/embed\/fI9Y-nH5yIw"}]},"normalized_url":["http:\/\/youtube.com\/watch?v=fI9Y-nH5yIw"],"retweet_count":[0],"title":["Apache Ft. Canserbero - Uno Por Ellas - YouTube"],"url":["http:\/\/www.youtube.com\/watch?v=fI9Y-nH5yIw"]},"salience":{"title":{"sentiment":0}}}
+{"interaction":{"author":{"link":"http:\/\/youtube.com\/RedSoftVideos","name":"RedSoftVideos"},"content":"\u041F\u043E\u0448\u0430\u0433\u043E\u0432\u044B\u0435 \u0438\u043D\u0441\u0442\u0440\u0443\u043A\u0446\u0438\u0438 \u0434\u043B\u044F \u043D\u043E\u0432\u0438\u0447\u043A\u043E\u0432. \u0414\u0435\u043B\u0430\u0435\u043C \u0438\u0437 \u0441\u0442\u0430\u0440\u043E\u0433\u043E \u043A\u043E\u043C\u043F\u0430 \u0434\u043E\u043C\u0430\u0448\u043D\u0438\u0439 web \u0441\u0435\u0440\u0432\u0435\u0440 \u043D\u0430 \u043E\u0441\u043D\u043E\u0432\u0435 Centos 6.4. \u0412 \u044D\u0442\u043E\u0439 \u0441\u0435\u0440\u0438\u0438 \u0443\u0441\u0442\u0430\u043D\u0430\u0432\u043B\u0438\u0432\u0430\u0435\u043C web \u0441\u0435\u0440\u0432\u0435\u0440 Apache, PHP \u0434\u043B\u044F \u0434\u0438...","contenttype":"html","created_at":"Thu, 03 Oct 2013 15:03:35 +0000","id":"1e32bcb1ad88a600e0610e0f18ad029a","link":"http:\/\/www.youtube.com\/watch?v=xlLGlVcw7qQ","schema":{"version":3},"title":"[iMi
ke.ru] - \u0443\u0441\u0442\u0430\u043D\u043E\u0432\u043A\u0430 LAMP web \u0441\u0435\u0440\u0432\u0435\u0440\u0430 \u0432\u0438\u0434\u0435\u043E\u0438\u043D\u0441\u0442\u0440\u0443\u043A\u0446\u0438\u044F. \u0423\u0440\u043E\u043A 2. Apache, PHP, MySQL, Wordpress.","type":"youtube"},"language":{"confidence":88,"tag":"ru"},"youtube":{"author":{"link":"http:\/\/youtube.com\/RedSoftVideos","name":"RedSoftVideos"},"category":"education","content":"\u041F\u043E\u0448\u0430\u0433\u043E\u0432\u044B\u0435 \u0438\u043D\u0441\u0442\u0440\u0443\u043A\u0446\u0438\u0438 \u0434\u043B\u044F \u043D\u043E\u0432\u0438\u0447\u043A\u043E\u0432. \u0414\u0435\u043B\u0430\u0435\u043C \u0438\u0437 \u0441\u0442\u0430\u0440\u043E\u0433\u043E \u043A\u043E\u043C\u043F\u0430 \u0434\u043E\u043C\u0430\u0448\u043D\u0438\u0439 web \u0441\u0435\u0440\u0432\u0435\u0440 \u043D\u0430 \u043E\u0441\u043D\u043E\u0432\u0435 Centos 6.4. \u0412 \u044D\u0442\u043E\u0439 \u0441\u0435\u0440\u0438\u0438 \u0443\u0441\u0442\u043
0\u043D\u0430\u0432\u043B\u0438\u0432\u0430\u0435\u043C web \u0441\u0435\u0440\u0432\u0435\u0440 Apache, PHP \u0434\u043B\u044F \u0434\u0438...","contenttype":"html","created_at":"Thu, 03 Oct 2013 01:28:28 +0000","duration":"3147","id":"1e32bcb1ad88a600e0610e0f18ad029a","title":"[iMike.ru] - \u0443\u0441\u0442\u0430\u043D\u043E\u0432\u043A\u0430 LAMP web \u0441\u0435\u0440\u0432\u0435\u0440\u0430 \u0432\u0438\u0434\u0435\u043E\u0438\u043D\u0441\u0442\u0440\u0443\u043A\u0446\u0438\u044F. \u0423\u0440\u043E\u043A 2. Apache, PHP, MySQL, Wordpress.","type":"video","videolink":"http:\/\/www.youtube.com\/watch?v=xlLGlVcw7qQ"}}
+{"demographic":{"gender":"female"},"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/100001626092766\/picture","id":"100001626092766","link":"http:\/\/www.facebook.com\/profile.php?id=100001626092766","name":"Fernanda Beltr\u00E0n"},"caption":"www.youtube.com","created_at":"Thu, 03 Oct 2013 01:31:05 +0000","description":"Nuevo album 2013","id":"100001626092766_593084220755776","link":"http:\/\/www.youtube.com\/watch?v=cDALFX6lQX8#_=","message":"te ve\u00EDas igual de \u00A1AHH! por supuesto, juro haber sentido el olor de tu cabello en mi pecho .. y de hecho el cigarro en tu recuerdo amargo, es todo lo que dejo y regresas sin embargo \u266B uf.","name":"Cancerbero y Apache - Stupid love story","source":"http:\/\/www.youtube.com\/v\/cDALFX6lQX8?version=3&autohide=1&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/100001626092766\/picture","id":"100001626092766","link":"http:\/\/www.facebook.com\/profile.php?id=100001626092766","
name":"Fernanda Beltr\u00E0n"},"content":"te ve\u00EDas igual de \u00A1AHH! por supuesto, juro haber sentido el olor de tu cabello en mi pecho .. y de hecho el cigarro en tu recuerdo amargo, es todo lo que dejo y regresas sin embargo \u266B uf.","created_at":"Thu, 03 Oct 2013 01:31:38 +0000","id":"1e32bcb786cda280e0664a7d4c566eec","link":"http:\/\/www.facebook.com\/100001626092766_593084220755776","received_at":4743578151536100481,"schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/cDALFX6lQX8?version=3&autohide=1&autoplay=1","subtype":"video","title":"Cancerbero y Apache - Stupid love story","type":"facebook"},"language":{"confidence":100,"tag":"es"},"links":{"code":[200],"created_at":["Wed, 02 Oct 2013 17:54:38 +0000"],"hops":[["http:\/\/www.youtube.com\/watch?v=cDALFX6lQX8#_="]],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["Nuevo album 2013"],"keywords":[["cancerbero","apache","2013","sptupid love story"]],"lang":["en"],"opengraph":[{"site_n
ame":"YouTube","type":"video","description":"Nuevo album 2013","url":"http:\/\/www.youtube.com\/watch?v=cDALFX6lQX8","title":"Cancerbero y Apache - Stupid love story","image":"http:\/\/i1.ytimg.com\/vi\/cDALFX6lQX8\/hqdefault.jpg?feature=og"}],"twitter":[{"player_width":"640","description":"Nuevo album 2013","player_height":"480","card":"player","app":{"iphone":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/cDALFX6lQX8","id":"544007664"},"ipad":{"name":"YouTube","url":"vnd.youtube:\/\/watch\/cDALFX6lQX8","id":"544007664"},"googleplay":{"name":"YouTube","url":"http:\/\/www.youtube.com\/watch?v=cDALFX6lQX8","id":"com.google.android.youtube"}},"site":"@youtube","url":"http:\/\/www.youtube.com\/watch?v=cDALFX6lQX8","title":"Cancerbero y Apache - Stupid love story","image":"http:\/\/i1.ytimg.com\/vi\/cDALFX6lQX8\/hqdefault.jpg","player":"https:\/\/www.youtube.com\/embed\/cDALFX6lQX8"}]},"normalized_url":["http:\/\/youtube.com\/watch?v=cDALFX6lQX8"],"retweet_count":[0],"title":["Cancerbe
ro y Apache - Stupid love story - YouTube"],"url":["http:\/\/www.youtube.com\/watch?v=cDALFX6lQX8"]},"salience":{"content":{"sentiment":0},"title":{"sentiment":0}}}
+{"interaction":{"author":{"link":"http:\/\/youtube.com\/9ux759qmx_tehxdm8g0zmq","name":"9ux759qmx_tehxdm8g0zmq"},"content":"Dance off during a trivia night at Bender's Sports Bar in Milwaukee. This happened at America's Pub Quiz night. www.americaspubquiz.com for the free answer o...","contenttype":"html","created_at":"Thu, 03 Oct 2013 02:06:13 +0000","id":"1e32bcbf335ead80e061be6ca7aac4cc","link":"http:\/\/www.youtube.com\/watch?v=VsnfhGD8KcI","schema":{"version":3},"title":"Apache Dance at an America's Pub Quiz show in Milwaukee","type":"youtube"},"language":{"confidence":100,"tag":"en"},"salience":{"content":{"entities":[{"name":"Milwaukee","sentiment":0,"confident":1,"label":"Place","evidence":1,"type":"Place","about":1,"themes":["trivia night"]},{"name":"America","sentiment":0,"confident":1,"label":"Place","evidence":1,"type":"Place","about":0,"themes":["trivia night"]}],"sentiment":0,"topics":[{"name":"Sports","score":0.5362588763237001,"hits":0}]},"title":{"entities":[{"name"
:"Milwaukee","sentiment":0,"confident":1,"label":"Place","evidence":1,"type":"Place","about":1}],"sentiment":0}},"youtube":{"author":{"link":"http:\/\/youtube.com\/9ux759qmx_tehxdm8g0zmq","name":"9ux759qmx_tehxdm8g0zmq"},"category":"People & Blogs","content":"Dance off during a trivia night at Bender's Sports Bar in Milwaukee. This happened at America's Pub Quiz night. www.americaspubquiz.com for the free answer o...","contenttype":"html","created_at":"Thu, 03 Oct 2013 01:34:31 +0000","duration":"30","id":"1e32bcbf335ead80e061be6ca7aac4cc","title":"Apache Dance at an America's Pub Quiz show in Milwaukee","type":"video","videolink":"http:\/\/www.youtube.com\/watch?v=VsnfhGD8KcI"}}
+{"demographic":{"gender":"male"},"facebook":{"author":{"avatar":"https:\/\/graph.facebook.com\/100001811016208\/picture","id":"100001811016208","link":"http:\/\/www.facebook.com\/profile.php?id=100001811016208","name":"Alan Giovanni Gonzaga"},"caption":"www.youtube.com","created_at":"Thu, 03 Oct 2013 01:36:04 +0000","description":"festividad en honor a el principe san miguel arcangel..","id":"100001811016208_538494686220894","link":"http:\/\/www.youtube.com\/v\/WuX7qu39YvA?version=3&autohide=1&autoplay=1","message":"http:\/\/www.youtube.com\/watch?v=WuX7qu39YvA&feature=youtu.be&hd=1","name":"san miguel toliman Qro...(Danza Apache Los Halcones) 29-SEP-2013","source":"http:\/\/www.youtube.com\/v\/WuX7qu39YvA?version=3&autohide=1&autoplay=1","type":"video"},"interaction":{"author":{"avatar":"https:\/\/graph.facebook.com\/100001811016208\/picture","id":"100001811016208","link":"http:\/\/www.facebook.com\/profile.php?id=100001811016208","name":"Alan Giovanni Gonzaga"},"content":"http:\/\
/www.youtube.com\/watch?v=WuX7qu39YvA&feature=youtu.be&hd=1","created_at":"Thu, 03 Oct 2013 01:36:31 +0000","id":"1e32bcc2aa49aa00e066e2c7d568b828","link":"http:\/\/www.facebook.com\/100001811016208_538494686220894","schema":{"version":3},"source":"http:\/\/www.youtube.com\/v\/WuX7qu39YvA?version=3&autohide=1&autoplay=1","subtype":"video","title":"san miguel toliman Qro...(Danza Apache Los Halcones) 29-SEP-2013","type":"facebook"},"language":{"confidence":100,"tag":"en"},"links":{"code":[200],"created_at":["Thu, 03 Oct 2013 01:36:28 +0000"],"hops":[["http:\/\/www.youtube.com\/watch?v=WuX7qu39YvA&feature=youtu.be&hd=1"]],"meta":{"charset":["UTF-8"],"content_type":["text\/html"],"description":["festividad en honor a el principe san miguel arcangel.."],"lang":["en"],"opengraph":[{"site_name":"YouTube","type":"video","description":"festividad en honor a el principe san miguel arcangel..","url":"http:\/\/www.youtube.com\/watch?v=WuX7qu39YvA","title":"san miguel toliman Qro...(Danza Apach
e Los Halcones) 29-SEP-2013","image":"http:\/\/i1.
<TRUNCATED>
[04/13] adding some uncommitted modules reorganizing pom improvements
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-provider-datasift/src/test/resources/datasift_jsons.txt
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/resources/datasift_jsons.txt b/streams-contrib/streams-provider-datasift/src/test/resources/datasift_jsons.txt
deleted file mode 100644
index 858756a..0000000
--- a/streams-contrib/streams-provider-datasift/src/test/resources/datasift_jsons.txt
+++ /dev/null
@@ -1,101 +0,0 @@
-{"w2o":{"tags":{"brand":["ArmAndHammer"]}},"twitter":{"id":"361363177973366787","filter_level":"medium","text":"Arm and Hammer sounds like a good porno. Not baking powder","source":"<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android<\/a>","created_at":"Sun, 28 Jul 2013 05:51:10 +0000","lang":"en","user":{"geo_enabled":true,"statuses_count":7761,"lang":"en","utc_offset":-14400,"id":472480397,"time_zone":"Eastern Time (US & Canada)","favourites_count":5062,"description":"I hate everything. Especially you.","friends_count":385,"name":"Alexander","created_at":"Tue, 24 Jan 2012 00:22:21 +0000","screen_name":"InsulinLover","id_str":"472480397","profile_image_url":"http://a0.twimg.com/profile_images/3078961290/b86c753148bdaf13fcf7f0ea3ea2ff57_normal.jpeg","followers_count":241}},"salience":{"content":{"sentiment":5}},"klout":{"score":42},"interaction":{"schema":{"version":3},"content":"Arm and Hammer sounds like a good porno. Not baking powder","id":"1e2f7
49b40d9ab00e0742908a9ff4d74","author":{"id":472480397,"username":"InsulinLover","name":"Alexander","link":"http://twitter.com/InsulinLover","language":"en","avatar":"http://a0.twimg.com/profile_images/3078961290/b86c753148bdaf13fcf7f0ea3ea2ff57_normal.jpeg"},"source":"Twitter for Android","link":"http://twitter.com/InsulinLover/statuses/361363177973366787","created_at":"Sun, 28 Jul 2013 05:51:10 +0000","type":"twitter"},"language":{"tag":"en","confidence":100},"demographic":{"gender":"male"}}
-{"w2o":{"tags":{"brand":["ArmAndHammer"]}},"twitter":{"id":"361366536625201152","retweeted":{"id":"361363177973366787","source":"<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android<\/a>","created_at":"Sun, 28 Jul 2013 05:51:10 +0000","user":{"geo_enabled":true,"statuses_count":7765,"lang":"en","utc_offset":-14400,"id":472480397,"time_zone":"Eastern Time (US & Canada)","favourites_count":5064,"description":"I hate everything. Especially you.","friends_count":386,"name":"Alexander","created_at":"Tue, 24 Jan 2012 00:22:21 +0000","screen_name":"InsulinLover","id_str":"472480397","profile_image_url":"http://a0.twimg.com/profile_images/3078961290/b86c753148bdaf13fcf7f0ea3ea2ff57_normal.jpeg","followers_count":242}},"retweet":{"id":"361366536625201152","text":"Arm and Hammer sounds like a good porno. Not baking powder","source":"<a href=\"http://twitter.com/#!/download/ipad\" rel=\"nofollow\">Twitter for iPad<\/a>","count":1,"created_at":"Sun, 28 Jul 2013 0
6:04:30 +0000","user":{"location":"Henderson, NV","statuses_count":4851,"lang":"en","url":"https://www.facebook.com/DonMfPatchUFemaleDogs","utc_offset":-25200,"id":537777703,"time_zone":"Pacific Time (US & Canada)","favourites_count":1235,"description":"Otherwise known as @Slender_Sir","friends_count":240,"name":"Iram Gonzalez","created_at":"Tue, 27 Mar 2012 02:35:56 +0000","screen_name":"gonzalez_iram","id_str":"537777703","profile_image_url":"http://a0.twimg.com/profile_images/3325430053/65b2d200115a97ff3f9061e9cc672136_normal.jpeg","followers_count":232},"lang":"en"}},"salience":{"content":{"sentiment":5}},"klout":{"score":40},"interaction":{"schema":{"version":3},"content":"RT @InsulinLover: Arm and Hammer sounds like a good porno. Not baking powder","id":"1e2f74b90e3eab00e07452795b51705e","author":{"id":537777703,"username":"gonzalez_iram","name":"Iram Gonzalez","link":"http://twitter.com/gonzalez_iram","language":"en","avatar":"http://a0.twimg.com/profile_images/3325430053/65b
2d200115a97ff3f9061e9cc672136_normal.jpeg"},"source":"Twitter for iPad","link":"http://twitter.com/gonzalez_iram/statuses/361366536625201152","created_at":"Sun, 28 Jul 2013 06:04:30 +0000","type":"twitter"},"language":{"tag":"en","confidence":100}}
-{"w2o":{"tags":{"topic":["Sustainability","LaundryDetergent"],"brand":["ArmAndHammer"]}},"salience":{"content":{"sentiment":0,"topics":[{"hits":0,"name":"Weather","score":0.65636122226715}]}},"facebook":{"id":"557546084284515_565003810205409","message":"To get arm and hammer laundry detergent for 1.29 go to Walgreens and get a rain check for this product. One you have the rain check go to www.thecouponclippers.com and order the arm and hammer detergent that's $1 off. The coupons are around 12 cent each. If you have any questions please post","author":{"id":"557546084284515","name":"Coupon Savings STL","link":"http://www.facebook.com/profile.php?id=557546084284515","avatar":"https://graph.facebook.com/557546084284515/picture"},"application":"Pages Manager for Android","source":"Pages Manager for Android (121876164619130)","created_at":"Fri, 26 Jul 2013 19:39:11 +0000","type":"status"},"interaction":{"schema":{"version":3},"id":"1e2f62b0b691a980e066608cf545f2d6","content":"To get arm
and hammer laundry detergent for 1.29 go to Walgreens and get a rain check for this product. One you have the rain check go to www.thecouponclippers.com and order the arm and hammer detergent that's $1 off. The coupons are around 12 cent each. If you have any questions please post","author":{"id":"557546084284515","name":"Coupon Savings STL","link":"http://www.facebook.com/profile.php?id=557546084284515","avatar":"https://graph.facebook.com/557546084284515/picture"},"source":"Pages Manager for Android (121876164619130)","link":"http://www.facebook.com/557546084284515_565003810205409","subtype":"status","created_at":"Fri, 26 Jul 2013 19:39:25 +0000","type":"facebook"},"language":{"tag":"en","confidence":100}}
-{"w2o":{"beyatch":"yo","tags":{"topic":["Sustainability","LaundryDetergent"],"brand":["ArmAndHammer"]}},"salience":{"content":{"sentiment":6}},"facebook":{"id":"741220477_10153006009870478","message":"Arm and Hammer-sensitive laundry detergent is really my friend right now...#shehastolearnonewayoranother","author":{"id":"741220477","name":"Nicole Reed","link":"http://www.facebook.com/profile.php?id=741220477","avatar":"https://graph.facebook.com/741220477/picture"},"source":"web","created_at":"Thu, 25 Jul 2013 16:51:28 +0000","type":"status"},"interaction":{"schema":{"version":3},"id":"1e2f54a72fb6a800e06682e7cdbf4b5c","content":"Arm and Hammer-sensitive laundry detergent is really my friend right now...#shehastolearnonewayoranother","author":{"id":"741220477","name":"Nicole Reed","link":"http://www.facebook.com/profile.php?id=741220477","avatar":"https://graph.facebook.com/741220477/picture"},"source":"web","subtype":"status","created_at":"Thu, 25 Jul 2013 16:51:56 +0000","type":"f
acebook"},"language":{"tag":"en","confidence":100},"demographic":{"gender":"female"}}
-{"w2o":{"tags":{"topic":["LaundryDetergent"],"brand":["ArmAndHammer"]}},"facebook":{"id":"165282606905632_387817134652177","message":"Still Time To Get On This Deal>>> Hot Deal At County Market On Arm & Hammer Laundry Soap w/Coupon!>>>>>>>>>>>>http://crazyforcouponsandfreebies.com/blog/?p=60571","author":{"id":"165282606905632","name":"Crazy for coupons & freebies","link":"http://www.facebook.com/profile.php?id=165282606905632","avatar":"https://graph.facebook.com/165282606905632/picture"},"source":"web","created_at":"Sat, 27 Jul 2013 02:20:01 +0000","type":"status"},"interaction":{"schema":{"version":3},"id":"1e2f6630a53dae80e06600495e73832a","content":"Still Time To Get On This Deal>>> Hot Deal At County Market On Arm & Hammer Laundry Soap w/Coupon!>>>>>>>>>>>>http://crazyforcouponsandfreebies.com/blog/?p=60571","author":{"id":"165282606905632","name":"Crazy for coupons & freebies","link":"http://www.facebook.com/profile.php?id=165282606905632","avatar":"https://graph.facebook.c
om/165282606905632/picture"},"source":"web","link":"http://www.facebook.com/165282606905632_387817134652177","subtype":"status","created_at":"Sat, 27 Jul 2013 02:20:17 +0000","type":"facebook"},"links":{"title":["Hot Deal At County Market On Arm & Hammer Laundry Soap w/Coupon! | The Daily Blog"],"normalized_url":["http://crazyforcouponsandfreebies.com/blog?p=60571"],"created_at":["Sat, 27 Jul 2013 02:20:27 +0000"],"retweet_count":[0],"code":[200],"meta":{"charset":["UTF-8"],"opengraph":[{"title":"Hot Deal At County Market On Arm & Hammer Laundry Soap w/Coupon! | The Daily Blog","description":" You Can Get Arm & Hammer On Sale At County Market For $1.99 Each Go Here And Print Your $1/3 OP-$4.96 WYB 3 + Tax GO HERE HOT DEAL!~Cheryl","image":"http://crazyforcouponsandfreebies.com/blog/wp-content/uploads/2013/07/2013-07-24_1918.png","site_name":"The Daily Blog","type":"article","url":"http://crazyforcouponsandfreebies.com/blog/?p=60571"}],"content_type":["text/html"],"lang":["en-us"]},"
url":["http://crazyforcouponsandfreebies.com/blog/?p=60571"]},"language":{"tag":"mt","confidence":62}}
-{"w2o":{"tags":{"topic":["Sustainability","LaundryDetergent"],"brand":["Tide","ArmAndHammer"]}},"twitter":{"id":"360731330335608833","filter_level":"medium","text":"You'd have to use your ARM & HAMMER to get me to drink laundry detergent. What do I GAIN from it? The TIDE comes in & SUNLIGHT disappears?","source":"<a href=\"http://favstar.fm\" rel=\"nofollow\">Favstar.FM<\/a>","created_at":"Fri, 26 Jul 2013 12:00:25 +0000","lang":"en","user":{"location":"Halifax, Canada","statuses_count":26676,"lang":"en","url":"http://favstar.fm/users/CindyMeakin/recent","id":267645953,"utc_offset":-14400,"time_zone":"Eastern Time (US & Canada)","favourites_count":156111,"description":"I wear sun glasses at night so people don't talk to me.","friends_count":2162,"name":"~Cindy~","created_at":"Thu, 17 Mar 2011 08:57:31 +0000","screen_name":"CindyMeakin","id_str":"267645953","profile_image_url":"http://a0.twimg.com/profile_images/378800000068549393/1045d699f983568a884240bf6e07ca8a_normal.jpeg
","followers_count":7604,"listed_count":407}},"salience":{"content":{"sentiment":0}},"klout":{"score":61},"interaction":{"schema":{"version":3},"content":"You'd have to use your ARM & HAMMER to get me to drink laundry detergent. What do I GAIN from it? The TIDE comes in & SUNLIGHT disappears?","id":"1e2f5eaf4a2da280e074c2f2b84aa4f8","author":{"id":267645953,"username":"CindyMeakin","name":"~Cindy~","link":"http://twitter.com/CindyMeakin","language":"en","avatar":"http://a0.twimg.com/profile_images/378800000068549393/1045d699f983568a884240bf6e07ca8a_normal.jpeg"},"source":"Favstar.FM","link":"http://twitter.com/CindyMeakin/statuses/360731330335608833","created_at":"Fri, 26 Jul 2013 12:00:25 +0000","type":"twitter"},"language":{"tag":"en","confidence":100}}
-{"w2o":{"tags":{"brand":["ArmAndHammer"]}},"twitter":{"id":"361614949908361217","filter_level":"medium","text":"@THEmrReynolds probably a mixture of prunes, icy hot, and arm and hammer foot powder","mention_ids":[106891542],"source":"<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone<\/a>","in_reply_to_screen_name":"THEmrReynolds","in_reply_to_status_id":"361603144695820288","created_at":"Sun, 28 Jul 2013 22:31:37 +0000","in_reply_to_user_id":"106891542","lang":"en","mentions":["THEmrReynolds"],"user":{"geo_enabled":true,"location":"Stanford, CA","statuses_count":22223,"lang":"en","url":"http://memoirsofaGAYsh.tumblr.com","id":49340813,"utc_offset":-25200,"time_zone":"Pacific Time (US & Canada)","favourites_count":4278,"description":"?Dance is the hidden language of the soul of the body? CSRE/DRAMA Major @Stanford. #playwright #dancer","friends_count":436,"name":"Dominique Delacroix","created_at":"Sun, 21 Jun 2009 15:29:26 +0000","screen_name":"Memoi
rsofaGAYsh","id_str":"49340813","profile_image_url":"http://a0.twimg.com/profile_images/2258352218/MemoirsofaGAYsh_normal.jpg","followers_count":423,"listed_count":10}},"salience":{"content":{"sentiment":0}},"klout":{"score":57},"interaction":{"schema":{"version":3},"content":"@THEmrReynolds probably a mixture of prunes, icy hot, and arm and hammer foot powder","id":"1e2f7d576eeeaa80e074ae7154d45ee6","author":{"id":49340813,"username":"MemoirsofaGAYsh","name":"Dominique Delacroix","link":"http://twitter.com/MemoirsofaGAYsh","language":"en","avatar":"http://a0.twimg.com/profile_images/2258352218/MemoirsofaGAYsh_normal.jpg"},"source":"Twitter for iPhone","mention_ids":[106891542],"link":"http://twitter.com/MemoirsofaGAYsh/statuses/361614949908361217","created_at":"Sun, 28 Jul 2013 22:31:37 +0000","type":"twitter","mentions":["THEmrReynolds"]},"language":{"tag":"en","confidence":100},"demographic":{"gender":"unisex"}}
-{"w2o":{"tags":{"topic":["Sustainability","LaundryDetergent"],"brand":["ArmAndHammer"]}},"salience":{"content":{"sentiment":0},"title":{"sentiment":0}},"youtube":{"id":"1e2e7740c1d3a700e061c997741cc686","content":"1 irish spring or softsoap body wash 15-18oz. $.50 cvs brand antacid $1/2 arm and hammer laundry detergent Blog:averagejoecouponing.blogspot.com.","author":{"name":"averagejoecouponing","link":"http://youtube.com/averagejoecouponing"},"title":"Cvs red box coupons week of 07/07","duration":"58","category":"entertainment","videolink":"http://www.youtube.com/watch?v=zRPZqYpAvnc","created_at":"Mon, 08 Jul 2013 02:13:58 +0000","contenttype":"html","type":"video"},"interaction":{"schema":{"version":3},"id":"1e2e7740c1d3a700e061c997741cc686","content":"1 irish spring or softsoap body wash 15-18oz. $.50 cvs brand antacid $1/2 arm and hammer laundry detergent Blog:averagejoecouponing.blogspot.com.","author":{"name":"averagejoecouponing","link":"http://youtube.com/averagejoecouponin
g"},"title":"Cvs red box coupons week of 07/07","link":"http://www.youtube.com/watch?v=zRPZqYpAvnc","created_at":"Mon, 29 Jul 2013 01:22:47 +0000","type":"youtube","contenttype":"html"},"language":{"tag":"en","confidence":62}}
-{"w2o":{"tags":{"topic":["StainRemoval"],"brand":["ArmAndHammer"]}},"salience":{"content":{"sentiment":-5}},"facebook":{"id":"602766199_10151557023101200","message":"Dust, dirt, chip crumbs, and God only knows what else are my mortal enemies as I wade in to this life or death battle armed only with this vacuum and arm and hammer odor eliminating powder. To battle!!!!","author":{"id":"602766199","name":"Chris Boothroy","link":"http://www.facebook.com/profile.php?id=602766199","avatar":"https://graph.facebook.com/602766199/picture"},"application":"Facebook for iPhone","source":"Facebook for iPhone (6628568379)","created_at":"Tue, 30 Jul 2013 00:06:43 +0000","type":"status"},"interaction":{"schema":{"version":3},"id":"1e2f8abea632ab80e066b5fddc080fa8","content":"Dust, dirt, chip crumbs, and God only knows what else are my mortal enemies as I wade in to this life or death battle armed only with this vacuum and arm and hammer odor eliminating powder. To battle!!!!","author":{"id":"602766
199","name":"Chris Boothroy","link":"http://www.facebook.com/profile.php?id=602766199","avatar":"https://graph.facebook.com/602766199/picture"},"source":"Facebook for iPhone (6628568379)","subtype":"status","created_at":"Tue, 30 Jul 2013 00:06:51 +0000","type":"facebook"},"language":{"tag":"en","confidence":100},"demographic":{"gender":"mostly_male"}}
-{"w2o":{"tags":{"topic":["Sustainability","Scent","LaundryDetergent"],"brand":["ArmAndHammer"]}},"twitter":{"id":"361950303878254592","filter_level":"medium","text":"New ARM and HAMMER Sensitive Skin and Sensitive Scents?laundry detergent http://t.co/Yk2d3zqihm via @MyBitsandBleeps","mention_ids":[151080077],"source":"<a href=\"http://twitter.com/tweetbutton\" rel=\"nofollow\">Tweet Button<\/a>","domains":["wp.me"],"created_at":"Mon, 29 Jul 2013 20:44:11 +0000","links":["http://wp.me/p12Gyx-358"],"lang":"en","mentions":["MyBitsandBleeps"],"user":{"location":"Alberta,Canada ","statuses_count":208685,"lang":"en","url":"http://rocketgirl00.blogspot.com","id":23870867,"utc_offset":-21600,"time_zone":"Mountain Time (US & Canada)","favourites_count":105,"description":"I'm married,like to travel,think that friends and family are the most important thing in life.","friends_count":2842,"name":"Doris ","created_at":"Thu, 12 Mar 2009 01:36:56 +0000","screen_name":"dewinner","id_str":"23870867"
,"profile_image_url":"http://a0.twimg.com/profile_images/3066398820/1ac0eedaa1099ea172540dff0a4524de_normal.jpeg","followers_count":2591,"listed_count":81}},"salience":{"content":{"sentiment":0}},"klout":{"score":47},"interaction":{"schema":{"version":3},"content":"New ARM and HAMMER Sensitive Skin and Sensitive Scents?laundry detergent http://t.co/Yk2d3zqihm via @MyBitsandBleeps","id":"1e2f88f9f3b1af80e074bd3a2171a578","author":{"id":23870867,"username":"dewinner","name":"Doris ","link":"http://twitter.com/dewinner","language":"en","avatar":"http://a0.twimg.com/profile_images/3066398820/1ac0eedaa1099ea172540dff0a4524de_normal.jpeg"},"source":"Tweet Button","mention_ids":[151080077],"link":"http://twitter.com/dewinner/statuses/361950303878254592","created_at":"Mon, 29 Jul 2013 20:44:11 +0000","type":"twitter","mentions":["MyBitsandBleeps"]},"links":{"title":["New ARM and HAMMER Sensitive Skin and Sensitive Scents laundry detergent | Mybitsandbleeps's Blog"],"normalized_url":["h
ttp://mybitsandbleeps.com/2013/07/14/new-arm-and-hammer-sensitive-skin-and-sensitive-scents-laundry-detergent"],"created_at":["Sun, 28 Jul 2013 22:08:38 +0000"],"retweet_count":[0],"code":[200],"hops":[["http://wp.me/p12Gyx-358"]],"meta":{"twitter":[{"site":"@wordpressdotcom","card":"gallery","creator":"@MyBitsandBleeps"}],"description":["A mom blog creating more buzz about your product!"],"charset":["UTF-8"],"opengraph":[{"title":"New ARM and HAMMER Sensitive Skin and Sensitive Scents laundry detergent","description":"School?s out and families are taking the fun outside, which means more muddy knees and grass-stained tees. Laundry will be piling up in no time but many Canadians will be forced to reach for unscen...","image":"http://i2.wp.com/mybitsandbleeps.files.wordpress.com/2013/07/img_00002130.jpg?fit=1000%2C1000","type":"article","site_name":"Mybitsandbleeps's Blog","url":"http://mybitsandbleeps.com/2013/07/14/new-arm-and-hammer-sensitive-skin-and-sensitive-scents-laundry
-detergent/"}],"content_type":["text/html"],"lang":["en"]},"url":["http://mybitsandbleeps.com/2013/07/14/new-arm-and-hammer-sensitive-skin-and-sensitive-scents-laundry-detergent/"]},"language":{"tag":"en","confidence":62},"demographic":{"gender":"female"}}
-{"w2o":{"tags":{"topic":["Scent"],"brand":["ArmAndHammer"]}},"salience":{"content":{"sentiment":0},"title":{"sentiment":0}},"interaction":{"schema":{"version":3},"id":"1e2f455369a1a000e0619912c7bf4656","content":"I like the arm and hammer scent free powder.<br /> Not worried about UV as I,m either in a blind, blind around stand or high up in a climber.<br /> Scent and movement are the killers","author":{},"title":"Scent free detergent on sale","link":"http://forum.gon.com/showthread.php?t=762688#post7957680","created_at":"Tue, 30 Jul 2013 09:44:17 +0000","type":"board","contenttype":"html"},"language":{"tag":"en","confidence":95},"board":{"id":"1e2f455369a1a000e0619912c7bf4656","content":"I like the arm and hammer scent free powder.<br /> Not worried about UV as I,m either in a blind, blind around stand or high up in a climber.<br /> Scent and movement are the killers","title":"Scent free detergent on sale","link":"http://forum.gon.com/showthread.php?t=762688#post7957680","thread":"
http://forum.gon.com/showthread.php?t=762688","domain":"www.gon.com","created_at":"Wed, 24 Jul 2013 11:36:00 +0000","contenttype":"html","type":"post"}}
-{"w2o":{"tags":{"topic":["ConcentratedFormulas"],"brand":["ArmAndHammer"]}},"twitter":{"id":"363008881179369473","filter_level":"medium","text":"Missed a scoop? No problem! ARM & HAMMER? Ultra Last? delivers long lasting odor control even when you can't scoop. http://t.co/3DvTv6UOnp","source":"<a href=\"http://twitter.com/tweetbutton\" rel=\"nofollow\">Tweet Button<\/a>","domains":["www.armandhammer.com"],"created_at":"Thu, 01 Aug 2013 18:50:36 +0000","links":["http://www.armandhammer.com/pet-care/cat-care/Products/arm-and-hammer-ultra-last.aspx"],"lang":"en","user":{"id":1262903550,"friends_count":1,"name":"serenecerulean","created_at":"Tue, 12 Mar 2013 21:31:35 +0000","screen_name":"serenecerulean","id_str":"1262903550","statuses_count":6,"profile_image_url":"http://a0.twimg.com/sticky/default_profile_images/default_profile_6_normal.png","lang":"en"}},"salience":{"content":{"sentiment":2}},"interaction":{"schema":{"version":3},"content":"Missed a scoop? No problem! ARM & H
AMMER? Ultra Last? delivers long lasting odor control even when you can't scoop. http://t.co/3DvTv6UOnp","id":"1e2fadb4069ea600e074e3962995fde4","author":{"id":1262903550,"username":"serenecerulean","name":"serenecerulean","link":"http://twitter.com/serenecerulean","language":"en","avatar":"http://a0.twimg.com/sticky/default_profile_images/default_profile_6_normal.png"},"source":"Tweet Button","link":"http://twitter.com/serenecerulean/statuses/363008881179369473","created_at":"Thu, 01 Aug 2013 18:50:36 +0000","type":"twitter"},"links":{"title":["Arm & Hammer Multi-Brand - ARM & HAMMER? Ultra Last?"],"normalized_url":["http://armandhammer.com/pet-care/cat-care/Products/arm-and-hammer-ultra-last.aspx"],"created_at":["Thu, 01 Aug 2013 12:23:26 +0000"],"retweet_count":[0],"code":[200],"meta":{"keywords":[["Missed a scoop? No problem! ARM & HAMMER? Ultra Last? delivers long lasting odor control even when you can't scoop."]],"description":["Missed a scoop? No problem! ARM & HAMMER? Ultra
Last? delivers long lasting odor control even when you can't scoop."],"charset":["UTF-8"],"content_type":["text/html"],"lang":["unknown"]},"url":["http://www.armandhammer.com/pet-care/cat-care/Products/arm-and-hammer-ultra-last.aspx"]},"language":{"tag":"en","confidence":100}}
-{"w2o":{"tags":{"topic":["Sustainability"],"brand":["ArmAndHammer"]}},"salience":{"content":{"sentiment":-2,"topics":[{"hits":0,"name":"Beverages","score":1.0265922546387}],"entities":[{"sentiment":0,"confident":1,"name":"Danny","evidence":1,"about":1,"label":"Person","themes":["washing soda"],"type":"Person"}]}},"facebook":{"id":"506582967_10151624053632968","message":"Hate when you are getting ready to do something productive and realize you are missing something....need arm and hammer washing soda before I can make the detergent and Danny is napping.guess I'll have the hubs get it on his way home...","author":{"id":"506582967","name":"Bethany Bennett","link":"http://www.facebook.com/profile.php?id=506582967","avatar":"https://graph.facebook.com/506582967/picture"},"source":"web","created_at":"Sat, 03 Aug 2013 17:30:40 +0000","type":"status"},"interaction":{"schema":{"version":3},"id":"1e2fc626a99ea000e0664766345bbd9e","content":"Hate when you are getting ready to do something pro
ductive and realize you are missing something....need arm and hammer washing soda before I can make the detergent and Danny is napping.guess I'll have the hubs get it on his way home...","author":{"id":"506582967","name":"Bethany Bennett","link":"http://www.facebook.com/profile.php?id=506582967","avatar":"https://graph.facebook.com/506582967/picture"},"source":"web","subtype":"status","created_at":"Sat, 03 Aug 2013 17:30:50 +0000","type":"facebook"},"language":{"tag":"en","confidence":100},"demographic":{"gender":"female"}}
-{"w2o":{"tags":{"topic":["Scent"],"brand":["ArmAndHammer"]}},"twitter":{"id":"362021838882209792","filter_level":"medium","text":"Luv da smell of gain and arm&hammer washing powder","source":"<a href=\"https://path.com/\" rel=\"nofollow\">Path<\/a>","created_at":"Tue, 30 Jul 2013 01:28:27 +0000","lang":"en","user":{"statuses_count":6590,"lang":"en","utc_offset":-10800,"id":762376266,"time_zone":"Atlantic Time (Canada)","favourites_count":133,"description":"#teamproudmommy #hardworking #independent #teamjrrockojc: Thru it all I still maintain and smile, with God on my side I don't care who's against me","friends_count":280,"name":"toya","created_at":"Thu, 16 Aug 2012 21:36:25 +0000","screen_name":"1ofakind_87","id_str":"762376266","profile_image_url":"http://a0.twimg.com/profile_images/378800000133672589/4c46242befe6daf5f05e1136df5cfe19_normal.jpeg","followers_count":137}},"salience":{"content":{"sentiment":0}},"klout":{"score":30},"interaction":{"schema":{"version":3},"content":
"Luv da smell of gain and arm&hammer washing powder","id":"1e2f8b755662af80e0745b9f6321114e","author":{"id":762376266,"username":"1ofakind_87","name":"toya","link":"http://twitter.com/1ofakind_87","language":"en","avatar":"http://a0.twimg.com/profile_images/378800000133672589/4c46242befe6daf5f05e1136df5cfe19_normal.jpeg"},"source":"Path","link":"http://twitter.com/1ofakind_87/statuses/362021838882209792","created_at":"Tue, 30 Jul 2013 01:28:27 +0000","type":"twitter"},"language":{"tag":"en","confidence":100},"demographic":{"gender":"female"}}
-{"w2o":{"tags":{"brand":["ArmAndHammer"]}},"twitter":{"id":"362230621323526145","retweeted":{"id":"362229784211750912","source":"<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone<\/a>","created_at":"Tue, 30 Jul 2013 15:14:45 +0000","user":{"location":"LA","statuses_count":18221,"lang":"en","utc_offset":-18000,"id":223342789,"time_zone":"Central Time (US & Canada)","favourites_count":6766,"description":"Onesie gang or die","friends_count":231,"name":"Carlage","created_at":"Mon, 06 Dec 2010 03:23:42 +0000","screen_name":"legitCarll","id_str":"223342789","profile_image_url":"http://a0.twimg.com/profile_images/378800000049299664/a636d7d245455eeb8b335ed4bd6480f8_normal.jpeg","followers_count":220}},"retweet":{"id":"362230621323526145","text":"We're eating breakfast and @TheVikkiMinaj sits down and says the arm and hammer powder she put on her vagina burns. THANKS FOR THAT","source":"<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter f
or iPhone<\/a>","count":3,"mention_ids":[279691748],"created_at":"Tue, 30 Jul 2013 15:18:04 +0000","user":{"geo_enabled":true,"statuses_count":40607,"lang":"en","utc_offset":-18000,"id":442045360,"time_zone":"Central Time (US & Canada)","favourites_count":13074,"friends_count":190,"name":"sM","created_at":"Tue, 20 Dec 2011 17:58:45 +0000","screen_name":"SeanMeier_","id_str":"442045360","profile_image_url":"http://a0.twimg.com/profile_images/378800000156618013/8008e98a1d4634a27e65bf3517f2bf3f_normal.jpeg","followers_count":1034,"listed_count":1},"mentions":["TheVikkiMinaj"],"lang":"en"}},"salience":{"content":{"sentiment":0,"topics":[{"hits":0,"name":"Food","score":0.50305610895157}]}},"klout":{"score":38},"interaction":{"schema":{"version":3},"content":"RT @legitCarll: We're eating breakfast and @TheVikkiMinaj sits down and says the arm and hammer powder she put on her vagina burns. THANKS FOR THAT","id":"1e2f92b3acd9ae00e0745976b013340a","author":{"id":442045360,"username":"SeanMei
er_","name":"sM","link":"http://twitter.com/SeanMeier_","language":"en","avatar":"http://a0.twimg.com/profile_images/378800000156618013/8008e98a1d4634a27e65bf3517f2bf3f_normal.jpeg"},"source":"Twitter for iPhone","mention_ids":[279691748],"link":"http://twitter.com/SeanMeier_/statuses/362230621323526145","created_at":"Tue, 30 Jul 2013 15:18:04 +0000","type":"twitter","mentions":["TheVikkiMinaj"]},"language":{"tag":"en","confidence":100}}
-{"w2o":{"tags":{"brand":["ArmAndHammer"]}},"twitter":{"id":"362037402786729986","filter_level":"medium","text":"Whippin that DIY wheat paste. So this is how the Arm & Hammer powder slangers feel. I'm Pusha P lykabit","source":"<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android<\/a>","created_at":"Tue, 30 Jul 2013 02:30:17 +0000","lang":"en","user":{"geo_enabled":true,"location":"North Texas","statuses_count":24770,"lang":"en","url":"http://d1ligence.tumblr.com","id":224686933,"utc_offset":-18000,"time_zone":"Central Time (US & Canada)","favourites_count":638,"description":"Jussa yung nicca on the virtue of internet prosperity. Artist; ? of Elev?ted collective.","friends_count":670,"name":"Paul, The Apostle ","created_at":"Thu, 09 Dec 2010 16:26:43 +0000","screen_name":"MasterChefP","id_str":"224686933","profile_image_url":"http://a0.twimg.com/profile_images/3674596098/7a4252c6a43db5cb373102b73b90e21d_normal.jpeg","followers_count":939,"listed_co
unt":6}},"salience":{"content":{"sentiment":0}},"klout":{"score":41},"interaction":{"schema":{"version":3},"content":"Whippin that DIY wheat paste. So this is how the Arm & Hammer powder slangers feel. I'm Pusha P lykabit","id":"1e2f8bff8bb4a280e074878bb70afd4a","author":{"id":224686933,"username":"MasterChefP","name":"Paul, The Apostle ","link":"http://twitter.com/MasterChefP","language":"en","avatar":"http://a0.twimg.com/profile_images/3674596098/7a4252c6a43db5cb373102b73b90e21d_normal.jpeg"},"source":"Twitter for Android","link":"http://twitter.com/MasterChefP/statuses/362037402786729986","created_at":"Tue, 30 Jul 2013 02:30:17 +0000","type":"twitter"},"language":{"tag":"en","confidence":100},"demographic":{"gender":"male"}}
-{"w2o":{"tags":{"topic":["Sustainability","Advertising","ConcentratedFormulas","CouponUsage","LaundryDetergent"],"brand":["ArmAndHammer"]}},"salience":{"content":{"sentiment":4,"topics":[{"hits":0,"name":"Science","score":1.2123092412949},{"hits":0,"name":"Technology","score":0.59644603729248},{"hits":0,"name":"Hardware","score":0.4546263217926}],"entities":[{"sentiment":0,"confident":1,"name":"Procter & gamble","evidence":1,"about":1,"label":"Company","themes":["liquid laundry detergent","commercial products","ea dawn","liquid offers","grease-cutting ingredients","improved formula","longer-lasting suds","sink changeover"],"type":"Company"},{"sentiment":0,"confident":1,"name":"\"Liquid Detergents (Surfactant Science)\"","evidence":1,"about":0,"label":"Quote","themes":["liquid detergents","helpful search engine","different books","actual prices","fast search"],"type":"Quote"}]}},"facebook":{"id":"1406941246186240_1407041779509520","message":"Liquid Detergents (Surfactant Science)\n\n
If you want to find any book including \"Liquid Detergents (Surfactant Science)\" or if you want to read reviews on different books or to see the actual prices on books you may visit this website - AZBookFinder.com\n\nAZBookFinder.com is a helpful search engine and I want to share the way of fast search of books.\nBest regards.\n\nSome other books that may be interesting:\nArm & hammer 84555 liquid laundry detergent plus oxiclean he 35-loads (case of 6);\nProcter & gamble commercial products - dishwashing liquid, concentrated, 38 oz - sold as 1 ea - dawn dishwashing liquid offers more grease-cutting ingredients in this improved formula. longer-lasting suds reduce sink changeover. high-surfactant formula seeks out greasy soil, cuts through it, surrounds the removed soil and lifts it out of the way.;\nFramed print of soap bubbles from science photo library;\nPhoto jigsaw puzzle of soap bubbles from science photo library;\nHandbook of detergents, part f: production: 142 (surfactant sci
ence);\nChemical product design: towards a perspective through case studies, volume 23 (computer aided chemical engineering).","author":{"id":"1406941246186240","name":"Bookformats","link":"http://www.facebook.com/profile.php?id=1406941246186240","avatar":"https://graph.facebook.com/1406941246186240/picture"},"application":"Filipspacename","source":"Filipspacename (104453219725181)","created_at":"Tue, 06 Aug 2013 22:40:46 +0000","type":"status"},"interaction":{"schema":{"version":3},"id":"1e2fee93be16ab00e066dedfa2d1c568","content":"Liquid Detergents (Surfactant Science)\n\nIf you want to find any book including \"Liquid Detergents (Surfactant Science)\" or if you want to read reviews on different books or to see the actual prices on books you may visit this website - AZBookFinder.com\n\nAZBookFinder.com is a helpful search engine and I want to share the way of fast search of books.\nBest regards.\n\nSome other books that may be interesting:\nArm & hammer 84555 liquid laundry deterg
ent plus oxiclean he 35-loads (case of 6);\nProcter & gamble commercial products - dishwashing liquid, concentrated, 38 oz - sold as 1 ea - dawn dishwashing liquid offers more grease-cutting ingredients in this improved formula. longer-lasting suds reduce sink changeover. high-surfactant formula seeks out greasy soil, cuts through it, surrounds the removed soil and lifts it out of the way.;\nFramed print of soap bubbles from science photo library;\nPhoto jigsaw puzzle of soap bubbles from science photo library;\nHandbook of detergents, part f: production: 142 (surfactant science);\nChemical product design: towards a perspective through case studies, volume 23 (computer aided chemical engineering).","author":{"id":"1406941246186240","name":"Bookformats","link":"http://www.facebook.com/profile.php?id=1406941246186240","avatar":"https://graph.facebook.com/1406941246186240/picture"},"source":"Filipspacename (104453219725181)","subtype":"status","created_at":"Tue, 06 Aug 2013 22:41:01 +0
000","type":"facebook"},"language":{"tag":"en","confidence":99}}
-{"w2o":{"tags":{"brand":["ArmAndHammer"]}},"twitter":{"id":"364861019413164032","filter_level":"medium","text":"Fresh laundry solution for sensitive skin Arm & Hammer http://t.co/pvJC5hX28b via @Gingermommy","mention_ids":[126764623],"source":"<a href=\"http://triberr.com\" rel=\"nofollow\">Triberr<\/a>","domains":["goo.gl"],"created_at":"Tue, 06 Aug 2013 21:30:20 +0000","links":["http://goo.gl/DIOspt"],"lang":"en","mentions":["Gingermommy"],"user":{"location":"Ontario, Canada","statuses_count":19585,"lang":"en","url":"http://yeewittlethings.com","id":438700992,"utc_offset":-18000,"time_zone":"Central Time (US & Canada)","favourites_count":132,"description":"#CDN Mom of 2 wonderful kids. Wife to an amazing hubby. #PR friendly #Blogger. #Reviews, #Giveaways, Random Musings & Every day living! #KinderMom","friends_count":7875,"name":"Yee Wittle Things","created_at":"Fri, 16 Dec 2011 22:33:08 +0000","screen_name":"yeewittlethings","id_str":"438700992","profile_image_url":"http://a0
.twimg.com/profile_images/2618536105/862tu9w2orhn68rc82lk_normal.jpeg","followers_count":8712,"listed_count":58}},"salience":{"content":{"sentiment":0}},"klout":{"score":63},"interaction":{"schema":{"version":3},"content":"Fresh laundry solution for sensitive skin Arm & Hammer http://t.co/pvJC5hX28b via @Gingermommy","id":"1e2fedf64fcfae00e074af57bfa66a5c","author":{"id":438700992,"username":"yeewittlethings","name":"Yee Wittle Things","link":"http://twitter.com/yeewittlethings","language":"en","avatar":"http://a0.twimg.com/profile_images/2618536105/862tu9w2orhn68rc82lk_normal.jpeg"},"source":"Triberr","mention_ids":[126764623],"link":"http://twitter.com/yeewittlethings/statuses/364861019413164032","created_at":"Tue, 06 Aug 2013 21:30:20 +0000","type":"twitter","mentions":["Gingermommy"]},"links":{"title":["Fresh laundry solution for sensitive skin Arm & Hammer - Tales of a Ranting Ginger"],"normalized_url":["http://talesofarantingginger.com/2013/08/fresh-laundry-solution-for-se
nsitive-skin-arm-hammer.html"],"created_at":["Tue, 06 Aug 2013 06:53:13 +0000"],"retweet_count":[0],"code":[200],"hops":[["http://goo.gl/DIOspt"]],"meta":{"description":["sensitive skin arm & hammer"],"charset":["UTF-8"],"content_type":["text/html"],"lang":["en-us"]},"url":["http://www.talesofarantingginger.com/2013/08/fresh-laundry-solution-for-sensitive-skin-arm-hammer.html"]},"language":{"tag":"en","confidence":100},"demographic":{"gender":"unisex"}}
-{"w2o":{"tags":{"brand":["ArmAndHammer"]}},"twitter":{"id":"364845410017230848","filter_level":"medium","text":"Hannah Montana without the arm & hammer orrrrrr? RT @ThatGuy_Jesse10: Got that powder. Hmu","mention_ids":[326314186],"source":"<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone<\/a>","in_reply_to_screen_name":"ThatGuy_Jesse10","in_reply_to_status_id":"364845085571031041","created_at":"Tue, 06 Aug 2013 20:28:18 +0000","in_reply_to_user_id":"326314186","lang":"en","mentions":["ThatGuy_Jesse10"],"user":{"geo_enabled":true,"location":"Phoeniquera","statuses_count":41253,"lang":"en","utc_offset":-25200,"id":403930347,"time_zone":"Arizona","favourites_count":4362,"description":"You're gonna like THIS guy he's alright, He's a Good Fella.. 93 til.","friends_count":330,"name":"Simba","created_at":"Thu, 03 Nov 2011 06:10:20 +0000","screen_name":"JuanGTho","id_str":"403930347","profile_image_url":"http://a0.twimg.com/profile_images/37880000018266
5838/026c863eec52a3af8e3e40af8ab5c56f_normal.jpeg","followers_count":359}},"salience":{"content":{"sentiment":0,"entities":[{"sentiment":0,"confident":1,"name":"Hannah Montana","evidence":1,"about":1,"label":"Person","type":"Person"}]}},"klout":{"score":38},"interaction":{"schema":{"version":3},"content":"Hannah Montana without the arm & hammer orrrrrr? RT @ThatGuy_Jesse10: Got that powder. Hmu","id":"1e2fed6ba80dad00e074579fbdc03880","author":{"id":403930347,"username":"JuanGTho","name":"Simba","link":"http://twitter.com/JuanGTho","language":"en","avatar":"http://a0.twimg.com/profile_images/378800000182665838/026c863eec52a3af8e3e40af8ab5c56f_normal.jpeg"},"source":"Twitter for iPhone","mention_ids":[326314186],"link":"http://twitter.com/JuanGTho/statuses/364845410017230848","created_at":"Tue, 06 Aug 2013 20:28:18 +0000","type":"twitter","mentions":["ThatGuy_Jesse10"]},"language":{"tag":"en","confidence":100}}
-{"w2o":{"tags":{"topic":["Convenience","LaundryDetergent"],"brand":["ArmAndHammer"]}},"salience":{"content":{"sentiment":-6,"topics":[{"hits":0,"name":"Beverages","score":1.056524515152}],"entities":[{"sentiment":-12,"confident":1,"name":"Target","evidence":1,"about":1,"label":"Company","themes":["brand laundry soap","cleaning team","washing soda"],"type":"Company"}]}},"facebook":{"id":"100005166043472_180980985417435","message":"A rogue ink pen exploded in my clothes dryer this morning :( ruining two sets of my work clothes..... I cleaned the inkout of the dryer drum with rubbing alcohol, and am trying to salvage my clothes. Rigth now the greatest cleaning team ever assembled is at work in the washer..... Target brand laundry soap, Fabuloso surface cleaner, half a cup of borax, and whatever was left in the box of Arm and Hammer washing soda on a small load cycle..... Will keep posted.....","author":{"id":"100005166043472","name":"Zach Watts","link":"http://www.facebook.com/pro
file.php?id=100005166043472","avatar":"https://graph.facebook.com/100005166043472/picture"},"application":"Mobile","source":"Mobile (2915120374)","created_at":"Wed, 07 Aug 2013 02:32:06 +0000","type":"status"},"interaction":{"schema":{"version":3},"id":"1e2ff098d015a700e066f80d5c4c0dd0","content":"A rogue ink pen exploded in my clothes dryer this morning :( ruining two sets of my work clothes..... I cleaned the inkout of the dryer drum with rubbing alcohol, and am trying to salvage my clothes. Rigth now the greatest cleaning team ever assembled is at work in the washer..... Target brand laundry soap, Fabuloso surface cleaner, half a cup of borax, and whatever was left in the box of Arm and Hammer washing soda on a small load cycle..... Will keep posted.....","author":{"id":"100005166043472","name":"Zach Watts","link":"http://www.facebook.com/profile.php?id=100005166043472","avatar":"https://graph.facebook.com/100005166043472/picture"},"source":"Mobile (2915120374)","link":"http
://www.facebook.com/100005166043472_180980985417435","subtype":"status","created_at":"Wed, 07 Aug 2013 02:32:23 +0000","type":"facebook"},"language":{"tag":"en","confidence":100},"demographic":{"gender":"male"}}
-{"w2o":{"tags":{"topic":["LaundryDetergent"],"brand":["ArmAndHammer"]}},"salience":{"content":{"sentiment":2}},"facebook":{"id":"753060211_10153029318745212","message":"The best carpet cleaner is arm and hammer plus oxi clean laundry soap plus a scoop of powder oxi clean hot water and elbow grease. My carpet looks almost brand new. =)","author":{"id":"753060211","name":"Sarah McCormack","link":"http://www.facebook.com/profile.php?id=753060211","avatar":"https://graph.facebook.com/753060211/picture"},"application":"Facebook for Android","source":"Facebook for Android (350685531728)","created_at":"Thu, 25 Jul 2013 18:08:27 +0000","type":"status"},"interaction":{"schema":{"version":3},"id":"1e2f555341edaf80e066003d5ea01da0","content":"The best carpet cleaner is arm and hammer plus oxi clean laundry soap plus a scoop of powder oxi clean hot water and elbow grease. My carpet looks almost brand new. =)","author":{"id":"753060211","name":"Sarah McCormack","link":"http://www.facebook.com/
profile.php?id=753060211","avatar":"https://graph.facebook.com/753060211/picture"},"source":"Facebook for Android (350685531728)","subtype":"status","created_at":"Thu, 25 Jul 2013 18:08:51 +0000","type":"facebook"},"language":{"tag":"en","confidence":100},"demographic":{"gender":"female"}}
-{"w2o":{"tags":{"topic":["Sustainability","ConcentratedFormulas"],"brand":["ArmAndHammer"]}},"twitter":{"id":"361171385718018048","filter_level":"medium","text":"20 $1/1 ARM&HAMMER 4x concentrate detergent 8/31/13 - http://t.co/DwFm08iyOj http://t.co/7XvuTjDd0v","source":"<a href=\"http://satehut.com\" rel=\"nofollow\">Satehut<\/a>","domains":["dish-washers-dryers.com"],"created_at":"Sat, 27 Jul 2013 17:09:03 +0000","links":["http://dish-washers-dryers.com/?p=3858"],"lang":"en","media":[{"id":361171385722212352,"sizes":{"small":{"w":104,"h":140,"resize":"fit"},"thumb":{"w":104,"h":140,"resize":"crop"},"large":{"w":104,"h":140,"resize":"fit"},"medium":{"w":104,"h":140,"resize":"fit"}},"media_url_https":"https://pbs.twimg.com/media/BQMjdjUCEAAMlDL.jpg","media_url":"http://pbs.twimg.com/media/BQMjdjUCEAAMlDL.jpg","expanded_url":"http://twitter.com/Sate_Hut/status/361171385718018048/photo/1","id_str":"361171385722212352","display_url":"pic.twitter.com/7XvuTjDd0v","type":"photo","url
":"http://t.co/7XvuTjDd0v"}],"user":{"location":"Miami, FL","statuses_count":20248,"lang":"en","url":"http://SateHut.com","utc_offset":-14400,"id":342319003,"time_zone":"Eastern Time (US & Canada)","description":"For good sate (or Sateh) you can be anywhere! We give you the recipes and ingredients to make it at home!","name":"Sate Hut","created_at":"Mon, 25 Jul 2011 20:37:12 +0000","screen_name":"Sate_Hut","id_str":"342319003","profile_image_url":"http://a0.twimg.com/profile_images/1460610380/sate_hut_logo_normal.jpg","followers_count":15}},"salience":{"content":{"sentiment":0,"entities":[{"sentiment":0,"confident":1,"name":"8/31/13","evidence":1,"about":1,"label":"Date","type":"Pattern"}]}},"klout":{"score":13},"interaction":{"schema":{"version":3},"content":"20 $1/1 ARM&HAMMER 4x concentrate detergent 8/31/13 - http://t.co/DwFm08iyOj http://t.co/7XvuTjDd0v","id":"1e2f6df3ca32a180e0745c1dc0cceda4","author":{"id":342319003,"username":"Sate_Hut","name":"Sate Hut","link":"http://
twitter.com/Sate_Hut","language":"en","avatar":"http://a0.twimg.com/profile_images/1460610380/sate_hut_logo_normal.jpg"},"source":"Satehut","link":"http://twitter.com/Sate_Hut/statuses/361171385718018048","created_at":"Sat, 27 Jul 2013 17:09:03 +0000","type":"twitter"},"links":{"title":["20 $1/1 ARM&HAMMER 4x concentrate detergent 8/31/13 | Dish-Washers & Dryers"],"normalized_url":["http://dish-washers-dryers.com/?p=3858"],"created_at":["Sat, 27 Jul 2013 17:09:07 +0000"],"retweet_count":[0],"code":[200],"meta":{"charset":["UTF-8"],"opengraph":[{"title":"20 $1/1 ARM&HAMMER 4x concentrate detergent 8/31/13 | Dish-Washers & Dryers","description":"$0.99 (0 Bids)End Date: Saturday Jul-27-2013 19:47:26 PDTBuy It Now for only: $2.50Buy It Now | Bid now | Add to watch list","image":"http://thumbs1.ebaystatic.com/m/m5EkUMKNxAoIDYVzVioQCRg/140.jpg","site_name":"Dish-Washers & Dryers","type":"article","url":"http://dish-washers-dryers.com/?p=3858"}],"content_type":["text/html"],"lang":["en-us"
]},"url":["http://dish-washers-dryers.com/?p=3858"]},"language":{"tag":"en","confidence":62}}
-{"w2o":{"tags":{"topic":["Sustainability","LaundryDetergent"],"brand":["ArmAndHammer"]}},"salience":{"content":{"sentiment":0}},"facebook":{"id":"297050503659016_614567205240676","message":"Arm & Hammer Laundry Detergent Just $1.92 at Walgreens (til 7/27)\n\nhttp://wp.me/p22sqw-j1g","author":{"id":"297050503659016","name":"Cook, Coupon, Work","link":"http://www.facebook.com/profile.php?id=297050503659016","avatar":"https://graph.facebook.com/297050503659016/picture"},"source":"web","created_at":"Fri, 26 Jul 2013 23:40:01 +0000","type":"status"},"interaction":{"schema":{"version":3},"id":"1e2f64cb0481ae80e0668a518d22dfe2","content":"Arm & Hammer Laundry Detergent Just $1.92 at Walgreens (til 7/27)\n\nhttp://wp.me/p22sqw-j1g","author":{"id":"297050503659016","name":"Cook, Coupon, Work","link":"http://www.facebook.com/profile.php?id=297050503659016","avatar":"https://graph.facebook.com/297050503659016/picture"},"source":"web","link":"http://www.facebook.com/297050503659016_614567205240
676","subtype":"status","created_at":"Fri, 26 Jul 2013 23:40:22 +0000","type":"facebook"},"links":{"title":["Arm & Hammer Laundry Detergent Just $1.92 at Walgreens (til 7/27) - Cook, Coupon, Work"],"normalized_url":["http://cookcouponwork.com/arm-hammer-laundry-detergent-just-1-92-at-walgreens-til-727"],"created_at":["Fri, 26 Jul 2013 23:40:16 +0000"],"retweet_count":[0],"code":[200],"hops":[["http://wp.me/p22sqw-j1g","http://www.cookcouponwork.com/?p=73114"]],"meta":{"twitter":[{"site":"@CookCouponWork","card":"summary","creator":"@CookCouponWork"}],"charset":["UTF-8"],"opengraph":[{"title":"Arm & Hammer Laundry Detergent Just $1.92 at Walgreens (til 7/27) - Cook, Coupon, Work","description":"Need Laundry Detergent? Pick up a good deal on Arm & Hammer at Walgreens (til 7/27) Deal Idea: Buy (2) Arm & Hammer 2x Liquid Detergent, 25 ? 32 loads, $2.29 = $4.58 Use (1) $0.75/2 Arm & Hammer Liquid or Powder Detergent, or Fabric Softener Sheets, exp. 9/30/13 (SS 07/14/13) Pay $3.83 =$1.92
EACH!","image":"http://www.cookcouponwork.com/wp-content/uploads/2013/07/543-300x213.jpg","type":"article","site_name":"Cook, Coupon, Work","url":"http://www.cookcouponwork.com/arm-hammer-laundry-detergent-just-1-92-at-walgreens-til-727/"}],"content_type":["text/html"],"lang":["en-us"]},"url":["http://www.cookcouponwork.com/arm-hammer-laundry-detergent-just-1-92-at-walgreens-til-727/"]},"language":{"tag":"en","confidence":100},"demographic":{"gender":"unisex"}}
-{"w2o":{"tags":{"topic":["Sustainability","LaundryDetergent"],"brand":["ArmAndHammer"]}},"twitter":{"id":"362022978235531264","filter_level":"medium","text":"Arm and hammer laundry detergent gets the whites to be super white. #notracist","source":"<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone<\/a>","created_at":"Tue, 30 Jul 2013 01:32:58 +0000","hashtags":["notracist"],"lang":"en","user":{"location":"Knoxville, TN","statuses_count":3202,"lang":"en","utc_offset":-14400,"id":442907220,"time_zone":"Eastern Time (US & Canada)","favourites_count":379,"description":"Never had a headache. Unable to grow a full beard.","friends_count":52,"name":"Uncle_D","created_at":"Wed, 21 Dec 2011 16:05:22 +0000","screen_name":"_UncleD_","id_str":"442907220","profile_image_url":"http://a0.twimg.com/profile_images/378800000144278522/e2c248ef7011fe240871ee3beab46bfd_normal.jpeg","followers_count":59}},"salience":{"content":{"sentiment":6}},"klout":{"score":40},"interac
tion":{"schema":{"version":3},"content":"Arm and hammer laundry detergent gets the whites to be super white. #notracist","id":"1e2f8b7f6ed8a100e074a98a4f29f3bc","author":{"id":442907220,"username":"_UncleD_","name":"Uncle_D","link":"http://twitter.com/_UncleD_","language":"en","avatar":"http://a0.twimg.com/profile_images/378800000144278522/e2c248ef7011fe240871ee3beab46bfd_normal.jpeg"},"source":"Twitter for iPhone","link":"http://twitter.com/_UncleD_/statuses/362022978235531264","created_at":"Tue, 30 Jul 2013 01:32:58 +0000","type":"twitter"},"language":{"tag":"en","confidence":100}}
-{"w2o":{"tags":{"topic":["Scent"],"brand":["ArmAndHammer"]}},"salience":{"content":{"sentiment":4}},"facebook":{"id":"100000487215935_704899129536321","message":"Arm & hammer carpet powder $2.75.... The feeling & smell of clean priceless..... Trying to sprinkle powder on carpet in front of a fan.... Stupid! I crack myself up!","author":{"id":"100000487215935","name":"Joyce Warren Cantrell","link":"http://www.facebook.com/profile.php?id=100000487215935","avatar":"https://graph.facebook.com/100000487215935/picture"},"application":"Mobile","source":"Mobile (2915120374)","created_at":"Sat, 27 Jul 2013 16:30:08 +0000","type":"status"},"interaction":{"schema":{"version":3},"id":"1e2f6d9ccde6a800e06674caf5177576","content":"Arm & hammer carpet powder $2.75.... The feeling & smell of clean priceless..... Trying to sprinkle powder on carpet in front of a fan.... Stupid! I crack myself up!","author":{"id":"100000487215935","name":"Joyce Warren Cantrell","link":"http://www.facebook.com/profile
.php?id=100000487215935","avatar":"https://graph.facebook.com/100000487215935/picture"},"source":"Mobile (2915120374)","link":"http://www.facebook.com/100000487215935_704899129536321","subtype":"status","created_at":"Sat, 27 Jul 2013 16:30:26 +0000","type":"facebook"},"language":{"tag":"en","confidence":100},"demographic":{"gender":"mostly_female"}}
-{"w2o":{"tags":{"brand":["ArmAndHammer"]}},"twitter":{"id":"361615185913462785","retweeted":{"id":"361614949908361217","source":"<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone<\/a>","in_reply_to_screen_name":"THEmrReynolds","in_reply_to_status_id":"361603144695820288","in_reply_to_user_id":"106891542","created_at":"Sun, 28 Jul 2013 22:31:37 +0000","user":{"geo_enabled":true,"location":"Stanford, CA","statuses_count":22223,"lang":"en","url":"http://memoirsofaGAYsh.tumblr.com","id":49340813,"utc_offset":-25200,"time_zone":"Pacific Time (US & Canada)","favourites_count":4278,"description":"?Dance is the hidden language of the soul of the body? CSRE/DRAMA Major @Stanford. #playwright #dancer","friends_count":436,"name":"Dominique Delacroix","created_at":"Sun, 21 Jun 2009 15:29:26 +0000","screen_name":"MemoirsofaGAYsh","id_str":"49340813","profile_image_url":"http://a0.twimg.com/profile_images/2258352218/MemoirsofaGAYsh_normal.jpg","followers_count":4
23,"listed_count":10}},"retweet":{"id":"361615185913462785","text":"@THEmrReynolds probably a mixture of prunes, icy hot, and arm and hammer foot powder","source":"<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android<\/a>","count":1,"mention_ids":[106891542],"created_at":"Sun, 28 Jul 2013 22:32:33 +0000","user":{"location":"soundcloud.com/mrReynoldsMusic","statuses_count":13351,"lang":"en","url":"http://www.THEmrReynolds.com","id":106891542,"utc_offset":-14400,"time_zone":"Eastern Time (US & Canada)","favourites_count":1353,"description":"I Mock The Party","friends_count":1554,"name":"Mr Reynolds","created_at":"Thu, 21 Jan 2010 00:37:45 +0000","screen_name":"THEmrReynolds","id_str":"106891542","profile_image_url":"http://a0.twimg.com/profile_images/2828785203/a097d038f964b0a6125a95c0a0e8ff7d_normal.jpeg","followers_count":7938,"listed_count":49},"mentions":["THEmrReynolds"],"lang":"en"}},"salience":{"content":{"sentiment":0}},"klout":{"score":59},"int
eraction":{"schema":{"version":3},"content":"RT @MemoirsofaGAYsh: @THEmrReynolds probably a mixture of prunes, icy hot, and arm and hammer foot powder","id":"1e2f7d5984fda680e074135a4b824dd0","author":{"id":106891542,"username":"THEmrReynolds","name":"Mr Reynolds","link":"http://twitter.com/THEmrReynolds","language":"en","avatar":"http://a0.twimg.com/profile_images/2828785203/a097d038f964b0a6125a95c0a0e8ff7d_normal.jpeg"},"source":"Twitter for Android","mention_ids":[106891542],"link":"http://twitter.com/THEmrReynolds/statuses/361615185913462785","created_at":"Sun, 28 Jul 2013 22:32:33 +0000","type":"twitter","mentions":["THEmrReynolds"]},"language":{"tag":"en","confidence":100}}
-{"w2o":{"tags":{"brand":["ArmAndHammer"]}},"twitter":{"id":"362467813958500354","filter_level":"medium","text":"Arm & Hammer Sensitive Skin?Review & Giveaway http://t.co/pS413KqHv5","source":"<a href=\"http://www.networkedblogs.com/\" rel=\"nofollow\">NetworkedBlogs<\/a>","domains":["nblo.gs"],"created_at":"Wed, 31 Jul 2013 07:00:35 +0000","links":["http://nblo.gs/NEwDq"],"lang":"en","user":{"location":"Canada","statuses_count":2034,"lang":"en","url":"http://www.zeemaid.com","id":25435820,"utc_offset":-25200,"time_zone":"Pacific Time (US & Canada)","favourites_count":17,"description":"I am Zeemaid. Momma to 3, an avid reader and an all around sassy momma. I own my own video business, am married to my Mr. Darcy, love to throw pots and write","friends_count":1976,"name":"Zeemaid","created_at":"Fri, 20 Mar 2009 01:59:59 +0000","screen_name":"Zeemaid","id_str":"25435820","profile_image_url":"http://a0.twimg.com/profile_images/1732650108/Zeemaid_normal.jpg","followers_count":10
27,"listed_count":30}},"salience":{"content":{"sentiment":0}},"klout":{"score":41},"interaction":{"schema":{"version":3},"content":"Arm & Hammer Sensitive Skin?Review & Giveaway http://t.co/pS413KqHv5","id":"1e2f9aee5d36ab80e0745a0744aef83a","author":{"id":25435820,"username":"Zeemaid","name":"Zeemaid","link":"http://twitter.com/Zeemaid","language":"en","avatar":"http://a0.twimg.com/profile_images/1732650108/Zeemaid_normal.jpg"},"source":"NetworkedBlogs","link":"http://twitter.com/Zeemaid/statuses/362467813958500354","created_at":"Wed, 31 Jul 2013 07:00:35 +0000","type":"twitter"},"links":{"title":["Arm & Hammer Sensitive Skin?Review & Giveaway"],"normalized_url":["http://networkedblogs.com/NEwDq"],"created_at":["Wed, 31 Jul 2013 07:00:43 +0000"],"retweet_count":[0],"code":[200],"hops":[["http://nblo.gs/NEwDq"]],"meta":{"charset":["UTF-8"],"opengraph":[{"title":"Arm & Hammer Sensitive Skin?Review & Giveaway","description":"When you love a product you want to stick with it bu
t as we all know price is a huge factor as to whether we continue to buy the brand we love or go elsewhere. Fortunately, Arm & Hammer?s affordability makes it easy for me to keep my cupboard well stocked with the Arm & Hammer products I love. Not","image":"http://lh6.ggpht.com/-8PeRiK68OMk/UfhgV8guCFI/AAAAAAAACgs/ubCE0V6ffqQ/CAN_AHLL_Sensitive_203L_thumb7.jpg?imgmax=800","site_name":"In the Mommy Trenches","type":"article","url":"http://networkedblogs.com/NEwDq"}],"content_type":["text/html"],"lang":["unknown"]},"url":["http://networkedblogs.com/NEwDq"]},"language":{"tag":"en","confidence":63}}
-{"w2o":{"tags":{"topic":["Sustainability","LaundryDetergent"],"brand":["ArmAndHammer"]}},"salience":{"content":{"sentiment":4}},"facebook":{"id":"478623348890311_497707713648541","message":"Example of what you can save......\n(regular retail)\n1 Febeze Air Effects $2.99 each ($1)\n1 Gillette Mach 3 disposables $7.49 each ($3)\n1 Gillette Shaving Cream $2.29 each ($1)\n1 Woolite High Traffic Cleaner $6.79 each ($2)\n1 Schick Women's Razors $5.69 each ($2)\n1 Colgate Toothbrush $1.89 each ($1)\n1 Arm and Hammer Laundry Detergent $4.99 each ($2)\n1 Snuggle Fabric Softener $4.99 each ($2)\n1 Bayer Aspirin $2.99 each ($1)\n1 Herbal Essence Shampoo $2.99 each ($1)\n1 Herbal Essence Conditioner $2.99 each ($1)\n1 Herbal Essence Styling Products $3.39 each ($1)\nTotal: $52.48\nDOLLAR DEALS TOTAL: $18.00","author":{"id":"478623348890311","name":"Dollar Deals Delivered","link":"http://www.facebook.com/profile.php?id=478623348890311","avatar":"https://graph.facebook.com/478623348890311/pictu
re"},"source":"web","created_at":"Wed, 31 Jul 2013 06:44:42 +0000","type":"status"},"interaction":{"schema":{"version":3},"id":"1e2f9acadcb2a900e066515ce688d7d0","content":"Example of what you can save......\n(regular retail)\n1 Febeze Air Effects $2.99 each ($1)\n1 Gillette Mach 3 disposables $7.49 each ($3)\n1 Gillette Shaving Cream $2.29 each ($1)\n1 Woolite High Traffic Cleaner $6.79 each ($2)\n1 Schick Women's Razors $5.69 each ($2)\n1 Colgate Toothbrush $1.89 each ($1)\n1 Arm and Hammer Laundry Detergent $4.99 each ($2)\n1 Snuggle Fabric Softener $4.99 each ($2)\n1 Bayer Aspirin $2.99 each ($1)\n1 Herbal Essence Shampoo $2.99 each ($1)\n1 Herbal Essence Conditioner $2.99 each ($1)\n1 Herbal Essence Styling Products $3.39 each ($1)\nTotal: $52.48\nDOLLAR DEALS TOTAL: $18.00","author":{"id":"478623348890311","name":"Dollar Deals Delivered","link":"http://www.facebook.com/profile.php?id=478623348890311","avatar":"https://graph.facebook.com/478623348890311/picture"},"source":"we
b","link":"http://www.facebook.com/478623348890311_497707713648541","subtype":"status","created_at":"Wed, 31 Jul 2013 06:44:57 +0000","type":"facebook"},"language":{"tag":"en","confidence":74}}
-{"w2o":{"tags":{"topic":["Sustainability","Scent","LaundryDetergent"],"brand":["ArmAndHammer"]}},"twitter":{"id":"362269562365943809","filter_level":"medium","text":"New ARM and HAMMER Sensitive Skin and Sensitive Scents?laundry detergent http://t.co/2HvWoZVjTt via @MyBitsandBleeps","mention_ids":[151080077],"source":"<a href=\"http://twitter.com/tweetbutton\" rel=\"nofollow\">Tweet Button<\/a>","domains":["wp.me"],"created_at":"Tue, 30 Jul 2013 17:52:48 +0000","links":["http://wp.me/p12Gyx-358"],"lang":"en","mentions":["MyBitsandBleeps"],"user":{"location":"Toronto,Ontario","statuses_count":29115,"lang":"en","utc_offset":-14400,"id":574859993,"time_zone":"Eastern Time (US & Canada)","favourites_count":13,"friends_count":1727,"name":"Andrew P","created_at":"Tue, 08 May 2012 23:07:45 +0000","screen_name":"apham17","id_str":"574859993","profile_image_url":"http://a0.twimg.com/profile_images/3226883575/148106bd4d6dcb4e8f06d8e5f0cd197f_normal.png","followers_count":455,"listed_count":7}
},"salience":{"content":{"sentiment":0}},"klout":{"score":48},"interaction":{"schema":{"version":3},"content":"New ARM and HAMMER Sensitive Skin and Sensitive Scents?laundry detergent http://t.co/2HvWoZVjTt via @MyBitsandBleeps","id":"1e2f940d87f9a800e074438bcad545f2","author":{"id":574859993,"username":"apham17","name":"Andrew P","link":"http://twitter.com/apham17","language":"en","avatar":"http://a0.twimg.com/profile_images/3226883575/148106bd4d6dcb4e8f06d8e5f0cd197f_normal.png"},"source":"Tweet Button","mention_ids":[151080077],"link":"http://twitter.com/apham17/statuses/362269562365943809","created_at":"Tue, 30 Jul 2013 17:52:48 +0000","type":"twitter","mentions":["MyBitsandBleeps"]},"links":{"title":["New ARM and HAMMER Sensitive Skin and Sensitive Scents laundry detergent | Mybitsandbleeps's Blog"],"normalized_url":["http://mybitsandbleeps.com/2013/07/14/new-arm-and-hammer-sensitive-skin-and-sensitive-scents-laundry-detergent"],"created_at":["Mon, 29 Jul 2013 23:20:31 +00
00"],"retweet_count":[0],"code":[200],"hops":[["http://wp.me/p12Gyx-358"]],"meta":{"twitter":[{"site":"@wordpressdotcom","card":"gallery","creator":"@MyBitsandBleeps"}],"description":["A mom blog creating more buzz about your product!"],"charset":["UTF-8"],"opengraph":[{"title":"New ARM and HAMMER Sensitive Skin and Sensitive Scents laundry detergent","description":"School?s out and families are taking the fun outside, which means more muddy knees and grass-stained tees. Laundry will be piling up in no time but many Canadians will be forced to reach for unscen...","image":"http://i2.wp.com/mybitsandbleeps.files.wordpress.com/2013/07/img_00002130.jpg?fit=1000%2C1000","type":"article","site_name":"Mybitsandbleeps's Blog","url":"http://mybitsandbleeps.com/2013/07/14/new-arm-and-hammer-sensitive-skin-and-sensitive-scents-laundry-detergent/"}],"content_type":["text/html"],"lang":["en"]},"url":["http://mybitsandbleeps.com/2013/07/14/new-arm-and-hammer-sensitive-skin-and-sensitive-sce
nts-laundry-detergent/"]},"language":{"tag":"en","confidence":62},"demographic":{"gender":"male"}}
-{"w2o":{"tags":{"brand":["ArmAndHammer"]}},"twitter":{"id":"362229784211750912","filter_level":"medium","text":"We're eating breakfast and @TheVikkiMinaj sits down and says the arm and hammer powder she put on her vagina burns. THANKS FOR THAT","mention_ids":[279691748],"source":"<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone<\/a>","created_at":"Tue, 30 Jul 2013 15:14:45 +0000","lang":"en","mentions":["TheVikkiMinaj"],"user":{"location":"LA","statuses_count":18221,"lang":"en","utc_offset":-18000,"id":223342789,"time_zone":"Central Time (US & Canada)","favourites_count":6766,"description":"Onesie gang or die","friends_count":231,"name":"Carlage","created_at":"Mon, 06 Dec 2010 03:23:42 +0000","screen_name":"legitCarll","id_str":"223342789","profile_image_url":"http://a0.twimg.com/profile_images/378800000049299664/a636d7d245455eeb8b335ed4bd6480f8_normal.jpeg","followers_count":220}},"salience":{"content":{"sentiment":0,"topics":[{"hits":0,"name":"Foo
d","score":0.50475257635117}]}},"klout":{"score":43},"interaction":{"schema":{"version":3},"content":"We're eating breakfast and @TheVikkiMinaj sits down and says the arm and hammer powder she put on her vagina burns. THANKS FOR THAT","id":"1e2f92ac430aa080e0748f373f80284e","author":{"id":223342789,"username":"legitCarll","name":"Carlage","link":"http://twitter.com/legitCarll","language":"en","avatar":"http://a0.twimg.com/profile_images/378800000049299664/a636d7d245455eeb8b335ed4bd6480f8_normal.jpeg"},"source":"Twitter for iPhone","mention_ids":[279691748],"link":"http://twitter.com/legitCarll/statuses/362229784211750912","created_at":"Tue, 30 Jul 2013 15:14:45 +0000","type":"twitter","mentions":["TheVikkiMinaj"]},"language":{"tag":"en","confidence":100}}
-{"w2o":{"tags":{"topic":["ConcentratedFormulas"],"brand":["ArmAndHammer"]}},"twitter":{"id":"365738027701645314","filter_level":"medium","text":"Missed a scoop? No problem! ARM & HAMMER? Ultra Last? delivers long lasting odor control even when you can't scoop. http://t.co/VGNQ0i0KGb","source":"<a href=\"http://twitter.com/tweetbutton\" rel=\"nofollow\">Tweet Button<\/a>","domains":["www.armandhammer.com"],"created_at":"Fri, 09 Aug 2013 07:35:15 +0000","links":["http://www.armandhammer.com/pet-care/cat-care/Products/arm-and-hammer-ultra-last.aspx"],"lang":"en","user":{"id":1571552280,"friends_count":2,"name":"tim tunner","created_at":"Fri, 05 Jul 2013 23:43:28 +0000","screen_name":"timtunner0987","id_str":"1571552280","statuses_count":32,"profile_image_url":"http://a0.twimg.com/sticky/default_profile_images/default_profile_4_normal.png","lang":"en"}},"salience":{"content":{"sentiment":2}},"klout":{"score":10},"interaction":{"schema":{"version":3},"content":"Missed a scoop? No pro
blem! ARM & HAMMER? Ultra Last? delivers long lasting odor control even when you can't scoop. http://t.co/VGNQ0i0KGb","id":"1e300c63b51fab80e074e71f77da87ca","author":{"id":1571552280,"username":"timtunner0987","name":"tim tunner","link":"http://twitter.com/timtunner0987","language":"en","avatar":"http://a0.twimg.com/sticky/default_profile_images/default_profile_4_normal.png"},"source":"Tweet Button","link":"http://twitter.com/timtunner0987/statuses/365738027701645314","created_at":"Fri, 09 Aug 2013 07:35:15 +0000","type":"twitter"},"links":{"title":["Arm & Hammer Multi-Brand - ARM & HAMMER? Ultra Last?"],"normalized_url":["http://armandhammer.com/pet-care/cat-care/Products/arm-and-hammer-ultra-last.aspx"],"created_at":["Fri, 09 Aug 2013 07:35:18 +0000"],"retweet_count":[0],"code":[200],"meta":{"keywords":[["Missed a scoop? No problem! ARM & HAMMER? Ultra Last? delivers long lasting odor control even when you can't scoop."]],"description":["Missed a scoop? No problem! ARM & HAMM
ER? Ultra Last? delivers long lasting odor control even when you can't scoop."],"charset":["UTF-8"],"content_type":["text/html"],"lang":["unknown"]},"url":["http://www.armandhammer.com/pet-care/cat-care/Products/arm-and-hammer-ultra-last.aspx"]},"language":{"tag":"en","confidence":100},"demographic":{"gender":"male"}}
-{"w2o":{"tags":{"topic":["Sustainability","StainRemoval","LaundryDetergent"],"brand":["ArmAndHammer"]}},"twitter":{"id":"364421048600510464","filter_level":"medium","text":"10 bottles Arm & Hammer Plus Oxi Clean Stain Fighters Liquid Laundry Detergent $0.99 http://t.co/89pkfqEgvl #laundry","source":"<a href=\"http://uTweetDeals.com\" rel=\"nofollow\">U Tweet Deals<\/a>","domains":["bit.ly"],"created_at":"Mon, 05 Aug 2013 16:22:03 +0000","hashtags":["laundry"],"links":["http://bit.ly/15zY9H5"],"lang":"en","user":{"statuses_count":59620,"lang":"en","utc_offset":-25200,"id":1131374341,"time_zone":"Arizona","description":"We search for deals on Home and Garden related products. Bath, Kitchen, Furniture, Garden and more. *** I Follow Back ***","friends_count":1939,"name":"Home and Garden","created_at":"Tue, 29 Jan 2013 15:37:21 +0000","screen_name":"HomeGardenBuys","id_str":"1131374341","followers_count":1706,"profile_image_url":"http://a0.twimg.com/profile_images/3531065622/bc956b2a
9da482ed071f2dbcdabea80b_normal.jpeg","listed_count":95}},"salience":{"content":{"sentiment":0}},"klout":{"score":49},"interaction":{"schema":{"version":3},"content":"10 bottles Arm & Hammer Plus Oxi Clean Stain Fighters Liquid Laundry Detergent $0.99 http://t.co/89pkfqEgvl #laundry","id":"1e2fdeb29810a780e0747af2c3544504","author":{"id":1131374341,"username":"HomeGardenBuys","name":"Home and Garden","link":"http://twitter.com/HomeGardenBuys","language":"en","avatar":"http://a0.twimg.com/profile_images/3531065622/bc956b2a9da482ed071f2dbcdabea80b_normal.jpeg"},"source":"U Tweet Deals","link":"http://twitter.com/HomeGardenBuys/statuses/364421048600510464","created_at":"Mon, 05 Aug 2013 16:22:03 +0000","type":"twitter"},"links":{"title":["10 Bottles Arm Hammer Plus Oxi Clean Stain Fighters Liquid Laundry Detergent | eBay"],"normalized_url":["http://ebay.com/itm/ws/eBayISAPI.dll?ViewItem&item=121154908808&item=121154908808"],"created_at":["Mon, 05 Aug 2013 16:22:47 +0000"],"retweet_
count":[0],"code":[200],"hops":[["http://bit.ly/15zY9H5","http://rover.ebay.com/rover/1/711-53200-19255-0/1/?type=2&campid=5337311970&toolid=10001&ext=121154908808&item=121154908808"]],"meta":{"keywords":[["10 bottles Arm & Hammer Plus Oxi Clean Stain Fighters Liquid Laundry Detergent","Home & Garden","Housekeeping & Organization","Laundry Supplies"]],"description":["10 bottles Arm & Hammer Plus Oxi Clean Stain Fighters Liquid Laundry Detergent in Home & Garden, Housekeeping & Organization, Laundry Supplies | eBay"],"charset":["UTF-8"],"opengraph":[{"title":"10 bottles Arm & Hammer Plus Oxi Clean Stain Fighters Liquid Laundry Detergent","description":"10 bottles Arm & Hammer Plus Oxi Clean Stain Fighters Liquid Laundry Detergent in Home & Garden, Housekeeping & Organization, Laundry Supplies | eBay","image":"http://thumbs1.ebaystatic.com/d/l225/m/mSRDHPf7_rrZnR75JtQxcCA.jpg","type":"ebay-objects:item","site_name":"eBay","url":"http://www.ebay.com/itm/10-bottles-Arm-Hammer-Plus-Oxi-C
lean-Stain-Fighters-Liquid-Laundry-Detergent-/121154908808"}],"content_type":["text/html"],"lang":["en-gb"]},"url":["http://www.ebay.com/itm/ws/eBayISAPI.dll?ViewItem&item=121154908808&item=121154908808"]},"language":{"tag":"en","confidence":63}}
-{"w2o":{"tags":{"brand":["ArmAndHammer"]}},"twitter":{"id":"364879533691772928","filter_level":"medium","text":"Arm and hammer carpet powder is amazing....#justsayin","source":"<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android<\/a>","created_at":"Tue, 06 Aug 2013 22:43:54 +0000","hashtags":["justsayin"],"lang":"en","user":{"location":"Indiana US of A","statuses_count":1400,"lang":"en","url":"http://www.quirkycreative.blogspot.com","utc_offset":-14400,"id":93704952,"time_zone":"Eastern Time (US & Canada)","favourites_count":9,"description":"Writer~ Random ~Quirky ~Creative ~Wife ~Believer ~Extroverted Introvert ~Animal Person ~Brony ~Girly-Girl w/ Tomboy Tendencies @JAInspired","friends_count":493,"name":"Rachel M","created_at":"Mon, 30 Nov 2009 20:20:15 +0000","screen_name":"thisrandomgirl","id_str":"93704952","profile_image_url":"http://a0.twimg.com/profile_images/3376486852/bfc1a1d4c44cf09978fb2c5c0fb33892_normal.jpeg","followers_count":102}},"sa
lience":{"content":{"sentiment":4}},"klout":{"score":18},"interaction":{"schema":{"version":3},"content":"Arm and hammer carpet powder is amazing....#justsayin","id":"1e2fee9abeffa100e074fa86d21b91e6","author":{"id":93704952,"username":"thisrandomgirl","name":"Rachel M","link":"http://twitter.com/thisrandomgirl","language":"en","avatar":"http://a0.twimg.com/profile_images/3376486852/bfc1a1d4c44cf09978fb2c5c0fb33892_normal.jpeg"},"source":"Twitter for Android","link":"http://twitter.com/thisrandomgirl/statuses/364879533691772928","created_at":"Tue, 06 Aug 2013 22:43:54 +0000","type":"twitter"},"language":{"tag":"en","confidence":61},"demographic":{"gender":"female"}}
-{"w2o":{"tags":{"brand":["ArmAndHammer"]}},"twitter":{"id":"364749039100633088","filter_level":"medium","text":"Fresh laundry solution for sensitive skin Arm & Hammer http://t.co/MmPYCeqmgN via @Gingermommy","mention_ids":[126764623],"source":"<a href=\"http://triberr.com\" rel=\"nofollow\">Triberr<\/a>","domains":["goo.gl"],"created_at":"Tue, 06 Aug 2013 14:05:22 +0000","links":["http://goo.gl/DIOspt"],"lang":"en","mentions":["Gingermommy"],"user":{"statuses_count":4853,"lang":"en","url":"http://www.mykidsguide.com/","id":612438542,"favourites_count":3,"description":"Everything you need to know about educational games and activities for kids.","friends_count":613,"name":"MyKidsGuide","created_at":"Tue, 19 Jun 2012 10:11:57 +0000","screen_name":"MyKidsGuide","id_str":"612438542","followers_count":1542,"profile_image_url":"http://a0.twimg.com/profile_images/2322033061/ryougq6v4o8gvddcx0ru_normal.jpeg","listed_count":14}},"salience":{"content":{"sentiment":0}},"klout":{"score":49
},"interaction":{"schema":{"version":3},"content":"Fresh laundry solution for sensitive skin Arm & Hammer http://t.co/MmPYCeqmgN via @Gingermommy","id":"1e2fea13bbd7ad00e0742b3a16342e3e","author":{"id":612438542,"username":"MyKidsGuide","name":"MyKidsGuide","link":"http://twitter.com/MyKidsGuide","language":"en","avatar":"http://a0.twimg.com/profile_images/2322033061/ryougq6v4o8gvddcx0ru_normal.jpeg"},"source":"Triberr","mention_ids":[126764623],"link":"http://twitter.com/MyKidsGuide/statuses/364749039100633088","created_at":"Tue, 06 Aug 2013 14:05:22 +0000","type":"twitter","mentions":["Gingermommy"]},"links":{"title":["Fresh laundry solution for sensitive skin Arm & Hammer - Tales of a Ranting Ginger"],"normalized_url":["http://talesofarantingginger.com/2013/08/fresh-laundry-solution-for-sensitive-skin-arm-hammer.html"],"created_at":["Tue, 06 Aug 2013 06:53:13 +0000"],"retweet_count":[0],"code":[200],"hops":[["http://goo.gl/DIOspt"]],"meta":{"description":["sensitive skin arm
& hammer"],"charset":["UTF-8"],"content_type":["text/html"],"lang":["en-us"]},"url":["http://www.talesofarantingginger.com/2013/08/fresh-laundry-solution-for-sensitive-skin-arm-hammer.html"]},"language":{"tag":"en","confidence":100}}
-{"w2o":{"tags":{"topic":["Scent","CouponUsage"],"brand":["Tide","ArmAndHammer"]}},"salience":{"content":{"sentiment":0,"entities":[{"sentiment":0,"confident":1,"name":"Gain","evidence":1,"about":1,"label":"Company","themes":["3 gallon gain"],"type":"Company"},{"sentiment":0,"confident":1,"name":"Tide","evidence":1,"about":1,"label":"Company","themes":["3 gallon gain"],"type":"Company"}]}},"facebook":{"id":"100001334708168_529689457085540","message":"SAVE SAVE SAVE NOW\n\nProduct Price Sheet\n\n25 lb powder Gain or Tide: $20\n\n3 gallon Gain or Tide: $25\n\n3 gallon Clear Arm & Hammer (no dye, no perfume) $25\n\n4 gallon Bleach $15\n\nAngel Soft Tissue 32 rolls $25, 96 rolls $65\n\n1 gallon Pine-sol Multi-Purpose Soap $12\n\n1 gallon Dove bodywash $15\n\n1 gallon Dawn Dish liquid $12\n\n1 gallon Downy Fabric Softener $15\n\n25 bars of Dove Soap $15\n\nTo place orders, call or text me at 678-531-1965\n\nSponsored by CarDM Mobile Mechanics and Hawaiian Breeze Auto Detailers","author"
:{"id":"100001334708168","name":"Albert Wells","link":"http://www.facebook.com/profile.php?id=100001334708168","avatar":"https://graph.facebook.com/100001334708168/picture"},"icon":"http://static.ak.fbcdn.net/rsrc.php/v2/yx/r/og8V99JVf8G.gif","application":"Facebook for Android","source":"Facebook for Android (350685531728)","link":"http://www.facebook.com/photo.php?fbid=529689133752239&set=pcb.529689457085540&type=1&relevant_count=3","created_at":"Mon, 05 Aug 2013 13:41:25 +0000","type":"photo"},"interaction":{"schema":{"version":3},"id":"1e2fdd4b8cefa080e0663432906416b4","content":"SAVE SAVE SAVE NOW\n\nProduct Price Sheet\n\n25 lb powder Gain or Tide: $20\n\n3 gallon Gain or Tide: $25\n\n3 gallon Clear Arm & Hammer (no dye, no perfume) $25\n\n4 gallon Bleach $15\n\nAngel Soft Tissue 32 rolls $25, 96 rolls $65\n\n1 gallon Pine-sol Multi-Purpose Soap $12\n\n1 gallon Dove bodywash $15\n\n1 gallon Dawn Dish liquid $12\n\n1 gallon Downy Fabric Softener $15\n\n25 bars of Dove Soap $1
5\n\nTo place orders, call or text me at 678-531-1965\n\nSponsored by CarDM Mobile Mechanics and Hawaiian Breeze Auto Detailers","author":{"id":"100001334708168","name":"Albert Wells","link":"http://www.facebook.com/profile.php?id=100001334708168","avatar":"https://graph.facebook.com/100001334708168/picture"},"source":"Facebook for Android (350685531728)","link":"http://www.facebook.com/100001334708168_529689457085540","subtype":"photo","created_at":"Mon, 05 Aug 2013 13:41:44 +0000","type":"facebook"},"links":{"title":["Photos from Albert Wells's post | Facebook"],"normalized_url":["https://facebook.com/photo.php?fbid=529689133752239&set=pcb.529689457085540&type=1&relevant_count=3"],"created_at":["Mon, 05 Aug 2013 13:41:51 +0000"],"retweet_count":[0],"code":[200],"hops":[["http://www.facebook.com/photo.php?fbid=529689133752239&set=pcb.529689457085540&type=1&relevant_count=3"]],"meta":{"charset":["UTF-8"],"content_type":["text/html"],"lang":["en"]},"url":["https://www.facebook.c
om/photo.php?fbid=529689133752239&set=pcb.529689457085540&type=1&relevant_count=3"]},"language":{"tag":"en","confidence":100},"demographic":{"gender":"male"}}
-{"w2o":{"tags":{"topic":["Sustainability","LaundryDetergent"],"brand":["ArmAndHammer"]}},"twitter":{"id":"364070259025649666","filter_level":"medium","text":"Somebody Come Do My Laundry. I'll Pay You. U Dont Need Detergent. I Got This Arm And Hammer ?","source":"<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone<\/a>","created_at":"Sun, 04 Aug 2013 17:08:08 +0000","lang":"en","user":{"geo_enabled":true,"statuses_count":31708,"lang":"en","utc_offset":-18000,"id":57967571,"time_zone":"Quito","favourites_count":38,"description":"http://ask.fm/thatDudeBiggz","friends_count":727,"name":"Sen?r","created_at":"Sat, 18 Jul 2009 16:18:33 +0000","screen_name":"thatDudeBiggz","id_str":"57967571","profile_image_url":"http://a0.twimg.com/profile_images/378800000020493320/c5b8669b417c8e362e0d92ad68bb24f9_normal.jpeg","followers_count":658,"listed_count":5}},"salience":{"content":{"sentiment":0}},"klout":{"score":43},"interaction":{"schema":{"version":3},"content":"S
omebody Come Do My Laundry. I'll Pay You. U Dont Need Detergent. I Got This Arm And Hammer ?","id":"1e2fd286f28cac00e074ec9c56b152ac","author":{"id":57967571,"username":"thatDudeBiggz","name":"Sen?r","link":"http://twitter.com/thatDudeBiggz","language":"en","avatar":"http://a0.twimg.com/profile_images/378800000020493320/c5b8669b417c8e362e0d92ad68bb24f9_normal.jpeg"},"source":"Twitter for iPhone","link":"http://twitter.com/thatDudeBiggz/statuses/364070259025649666","created_at":"Sun, 04 Aug 2013 17:08:08 +0000","type":"twitter"},"language":{"tag":"en","confidence":62}}
-{"w2o":{"tags":{"brand":["ArmAndHammer"]}},"twitter":{"id":"364267076807950339","filter_level":"medium","text":"Fresh laundry solution for sensitive skin Arm & Hammer http://t.co/hkHzh1oFnq via @Gingermommy","mention_ids":[126764623],"source":"<a href=\"http://triberr.com\" rel=\"nofollow\">Triberr<\/a>","domains":["goo.gl"],"created_at":"Mon, 05 Aug 2013 06:10:13 +0000","links":["http://goo.gl/DIOspt"],"lang":"en","mentions":["Gingermommy"],"user":{"geo_enabled":true,"location":"Lumsden, Saskatchewan","statuses_count":18479,"lang":"en","url":"http://www.mommykatandkids.com/","id":106185357,"utc_offset":-21600,"time_zone":"Mountain Time (US & Canada)","favourites_count":29,"description":"Mother of two active boys, blogger and writer. Lover of hot coffee, organic chocolate and sleeping in late whenever I get the chance.","friends_count":7894,"name":"Kathryn Lavallee","created_at":"Mon, 18 Jan 2010 19:29:48 +0000","screen_name":"mommykatandkids","id_str":"106185357","profile_image
_url":"http://a0.twimg.com/profile_images/1611498528/corel_touched-up_headshot_normal.jpg","followers_count":15439,"listed_count":306}},"salience":{"content":{"sentiment":0}},"klout":{"score":65},"interaction":{"schema":{"version":3},"content":"Fresh laundry solution for sensitive skin Arm & Hammer http://t.co/hkHzh1oFnq via @Gingermommy","id":"1e2fd95b0a39a080e07405af37275d4a","author":{"id":106185357,"username":"mommykatandkids","name":"Kathryn Lavallee","link":"http://twitter.com/mommykatandkids","language":"en","avatar":"http://a0.twimg.com/profile_images/1611498528/corel_touched-up_headshot_normal.jpg"},"source":"Triberr","mention_ids":[126764623],"link":"http://twitter.com/mommykatandkids/statuses/364267076807950339","created_at":"Mon, 05 Aug 2013 06:10:13 +0000","type":"twitter","mentions":["Gingermommy"]},"links":{"title":["Fresh laundry solution for sensitive skin Arm & Hammer - Tales of a Ranting Ginger"],"normalized_url":["http://talesofarantingginger.com/2013/08/fres
h-laundry-solution-for-sensitive-skin-arm-hammer.html"],"created_at":["Sun, 04 Aug 2013 21:00:19 +0000"],"retweet_count":[0],"code":[200],"hops":[["http://goo.gl/DIOspt"]],"meta":{"description":["sensitive skin arm & hammer"],"charset":["UTF-8"],"content_type":["text/html"],"lang":["en-us"]},"url":["http://www.talesofarantingginger.com/2013/08/fresh-laundry-solution-for-sensitive-skin-arm-hammer.html"]},"language":{"tag":"en","confidence":100},"demographic":{"gender":"female"}}
-{"w2o":{"tags":{"topic":["Sustainability"],"brand":["ArmAndHammer"]}},"twitter":{"id":"367633815277035520","filter_level":"medium","text":"Arm & Hammer Detergent Only $1.99 Starting Saturday!\n\nTime to Stock Up!! :) ~Ava~... http://t.co/LznOl9Me9k","source":"<a href=\"http://www.facebook.com/twitter\" rel=\"nofollow\">Facebook<\/a>","domains":["fb.me"],"created_at":"Wed, 14 Aug 2013 13:08:26 +0000","links":["http://fb.me/2jIdLRYWm"],"lang":"en","user":{"location":"Ontario, Canada","statuses_count":11244,"lang":"en","url":"http://savebiglivebetter.com","id":465174690,"description":"The best Canadian deals, sales, coupons, freebies, recipes, money saving articles and more updated daily at http://SaveBigLiveBetter.com! HAPPY SAVING:)","friends_count":71,"name":"Save Big Live Better","created_at":"Mon, 16 Jan 2012 01:56:38 +0000","screen_name":"SBLBCanada","id_str":"465174690","followers_count":476,"profile_image_url":"http://a0.twimg.com/profile_images/3279225640/270245999d75d401
c696258306991b51_normal.png","listed_count":4}},"salience":{"content":{"sentiment":3}},"klout":{"score":37},"interaction":{"schema":{"version":3},"content":"Arm & Hammer Detergent Only $1.99 Starting Saturday!\n\nTime to Stock Up!! :) ~Ava~... http://t.co/LznOl9Me9k","id":"1e304e29af34a100e0742d305e733fa6","author":{"id":465174690,"username":"SBLBCanada","name":"Save Big Live Better","link":"http://twitter.com/SBLBCanada","language":"en","avatar":"http://a0.twimg.com/profile_images/3279225640/270245999d75d401c696258306991b51_normal.png"},"source":"Facebook","link":"http://twitter.com/SBLBCanada/statuses/367633815277035520","created_at":"Wed, 14 Aug 2013 13:08:26 +0000","type":"twitter"},"links":{"title":["Timeline Photos | Facebook"],"normalized_url":["https://facebook.com/photo.php?fbid=507697515973605&l=337aea86ef"],"created_at":["Wed, 14 Aug 2013 13:08:58 +0000"],"retweet_count":[0],"code":[200],"hops":[["http://fb.me/2jIdLRYWm"]],"meta":{"charset":["UTF-8"],"content_type":[
"text/html"],"lang":["en"]},"url":["https://www.facebook.com/photo.php?fbid=507697515973605&l=337aea86ef"]},"language":{"tag":"en","confidence":62},"demographic":{"gender":"male"}}
-{"w2o":{"tags":{"topic":["Sustainability","Scent"],"brand":["ArmAndHammer"]}},"salience":{"content":{"sentiment":1,"entities":[{"sentiment":-2,"confident":1,"name":"Vinegar","evidence":1,"about":0,"label":"Person","themes":["funky smell","washing machine","go-to product","regular detergent","vinegar smell"],"type":"Person"}]}},"facebook":{"id":"234885799896401_583141561737488","message":"One of my fans asked me how to get the funky smell out of her washing machine. Vinegar was the go-to product for this in the old days. Although you would need to run the load a second time with regular detergent to get the vinegar smell out. It is possible to mix the detergent with vinegar if you want to try that.\n \nI've personally found that Arm & Hammer detergent works for me. I used to have the same problem with some of my thick towels. The A&H removed the smell. To avoid a washing machine smell, do the following:\n1) Make sure you clean the washing machine on a regular basis. The easiest
thing to do is to run an empty load with bleach.\n2) Leave the door open on the front loader between uses (if you have small children, though, make sure it's in a room here you can close the door).\n3) Make sure that the washing machine is set on the highest speed spin cycle. Some machines allow you to crank the spin cycle to over 1000 RPM's. That's what you need to do. The more effective the spin cycle, the more water that is removed and the less chance of mold and mildew.","author":{"id":"234885799896401","name":"Tipman Tim - Tips, tools and tricks for life","link":"http://www.facebook.com/profile.php?id=234885799896401","avatar":"https://graph.facebook.com/234885799896401/picture"},"source":"web","created_at":"Wed, 14 Aug 2013 17:23:45 +0000","type":"status"},"interaction":{"schema":{"version":3},"id":"1e3050645c92a680e066aef68cf33be0","content":"One of my fans asked me how to get the funky smell out of her washing machine. Vinegar was the go-to product for this in the old day
s. Although you would need to run the load a second time with regular detergent to get the vinegar smell out. It is possible to mix the detergent with vinegar if you want to try that.\n \nI've personally found that Arm & Hammer detergent works for me. I used to have the same problem with some of my thick towels. The A&H removed the smell. To avoid a washing machine smell, do the following:\n1) Make sure you clean the washing machine on a regular basis. The easiest thing to do is to run an empty load with bleach.\n2) Leave the door open on the front loader between uses (if you have small children, though, make sure it's in a room here you can close the door).\n3) Make sure that the washing machine is set on the highest speed spin cycle. Some machines allow you to crank the spin cycle to over 1000 RPM's. That's what you need to do. The more effective the spin cycle, the more water that is removed and the less chance of mold and mildew.","author":{"id":"234885799896401","name":
"Tipman Tim - Tips, tools and tricks for life","link":"http://www.facebook.com/profile.php?id=234885799896401","avatar":"https://graph.facebook.com/234885799896401/picture"},"source":"web","link":"http://www.facebook.com/234885799896401_583141561737488","subtype":"status","created_at":"Wed, 14 Aug 2013 17:24:00 +0000","type":"facebook"},"language":{"tag":"en","confidence":100},"demographic":{"gender":"male"}}
-{"w2o":{"tags":{"topic":["Sustainability","CouponUsage","LaundryDetergent"],"brand":["ArmAndHammer","Purex"]}},"salience":{"content":{"sentiment":0,"topics":[{"hits":0,"name":"Beverages","score":0.92570048570633}]}},"facebook":{"id":"100000254074556_643746722310458","message":"Ashley, that was the recipe for the powder detergent (which is amazing by the way) It won't last as long as the liquid and is more expensive. Sorry. I didn't mean to copy that one. Here is the liquid\nLIQUID SOAP MAKES 10 GALLONS\n? C PER LOAD ? 640 LOADS\n4 cups of Hot Tap Water\n1 Fels Naptha Soap Bar\n1 Cup Arm & Hammer Super Washing Soda\n? Cup Borax\n1 cup Purex liquid softener\nGrate soap, add to sauce pan with 4 cups of water. Heat and stir continually until soap dissolves & is melted.\nFill 5 gallon Bucket ? full with Hot Tap Water, Add melted soap, soda, Purex and borax. Stir well until powder is dissolved. Fill bucket to top with more hot water, stir, cover, and let sit overnight to thicken. S
tir then fill a container (used laundry bottles or empty water jugs) ? full with soap, fill the remainder with water. Shake well.","author":{"id":"100000254074556","name":"Sharon Bedwell Schluter","link":"http://www.facebook.com/profile.php?id=100000254074556","avatar":"https://graph.facebook.com/100000254074556/picture"},"source":"web","created_at":"Fri, 09 Aug 2013 15:07:21 +0000","type":"status"},"interaction":{"schema":{"version":3},"id":"1e3010563ad1aa80e0660e6752dee246","content":"Ashley, that was the recipe for the powder detergent (which is amazing by the way) It won't last as long as the liquid and is more expensive. Sorry. I didn't mean to copy that one. Here is the liquid\nLIQUID SOAP MAKES 10 GALLONS\n? C PER LOAD ? 640 LOADS\n4 cups of Hot Tap Water\n1 Fels Naptha Soap Bar\n1 Cup Arm & Hammer Super Washing Soda\n? Cup Borax\n1 cup Purex liquid softener\nGrate soap, add to sauce pan with 4 cups of water. Heat and stir continually until soap dissolves & is melted.\nF
ill 5 gallon Bucket ? full with Hot Tap Water, Add melted soap, soda, Purex and borax. Stir well until powder is dissolved. Fill bucket to top with more hot water, stir, cover, and let sit overnight to thicken. Stir then fill a container (used laundry bottles or empty water jugs) ? full with soap, fill the remainder with water. Shake well.","author":{"id":"100000254074556","name":"Sharon Bedwell Schluter","link":"http://www.facebook.com/profile.php?id=100000254074556","avatar":"https://graph.facebook.com/100000254074556/picture"},"source":"web","link":"http://www.facebook.com/100000254074556_643746722310458","subtype":"status","created_at":"Fri, 09 Aug 2013 15:07:35 +0000","type":"facebook"},"language":{"tag":"en","confidence":100},"demographic":{"gender":"unisex"}}
-{"w2o":{"tags":{"topic":["ConcentratedFormulas"],"brand":["ArmAndHammer"]}},"twitter":{"id":"359694365049896960","filter_level":"medium","text":"Missed a scoop? No problem! ARM & HAMMER? Ultra Last? delivers long lasting odor control even when you can't scoop. http://t.co/eZdNss2ClQ","source":"<a href=\"http://twitter.com/tweetbutton\" rel=\"nofollow\">Tweet Button<\/a>","domains":["www.armandhammer.com"],"created_at":"Tue, 23 Jul 2013 15:19:54 +0000","links":["http://www.armandhammer.com/pet-care/cat-care/Products/arm-and-hammer-ultra-last.aspx"],"lang":"en","user":{"id":1546236864,"friends_count":82,"name":"Kyra Mitchell","created_at":"Tue, 25 Jun 2013 17:32:24 +0000","screen_name":"ArmyRebel0","id_str":"1546236864","statuses_count":14,"followers_count":10,"profile_image_url":"http://a0.twimg.com/profile_images/378800000084101648/95eff16837f6e9fd03585630e8e6b562_normal.jpeg","lang":"en"}},"salience":{"content":{"sentiment":2}},"klout":{"score":12},"interaction":{"schema":{"ver
sion":3},"content":"Missed a scoop? No problem! ARM & HAMMER? Ultra Last? delivers long lasting odor control even when you can't scoop. http://t.co/eZdNss2ClQ","id":"1e2f3ab537a0a900e074ec3e0933e560","author":{"id":1546236864,"username":"ArmyRebel0","name":"Kyra Mitchell","link":"http://twitter.com/ArmyRebel0","language":"en","avatar":"http://a0.twimg.com/profile_images/378800000084101648/95eff16837f6e9fd03585630e8e6b562_normal.jpeg"},"source":"Tweet Button","link":"http://twitter.com/ArmyRebel0/statuses/359694365049896960","created_at":"Tue, 23 Jul 2013 15:19:54 +0000","type":"twitter"},"links":{"title":["Arm & Hammer Multi-Brand - ARM & HAMMER? Ultra Last?"],"normalized_url":["http://armandhammer.com/pet-care/cat-care/Products/arm-and-hammer-ultra-last.aspx"],"created_at":["Tue, 23 Jul 2013 15:20:08 +0000"],"retweet_count":[0],"code":[200],"meta":{"keywords":[["Missed a scoop? No problem! ARM & HAMMER? Ultra Last? delivers long lasting odor control even when you can't scoop."]
],"description":["Missed a scoop? No problem! ARM & HAMMER? Ultra Last? delivers long lasting odor control even when you can't scoop."],"charset":["UTF-8"],"content_type":["text/html"],"lang":["unknown"]},"url":["http://www.armandhammer.com/pet-care/cat-care/Products/arm-and-hammer-ultra-last.aspx"]},"language":{"tag":"en","confi
<TRUNCATED>
[08/13] git commit: adding uncommitted modules
Posted by sb...@apache.org.
adding uncommitted modules
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/89ff615e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/89ff615e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/89ff615e
Branch: refs/heads/springcleaning
Commit: 89ff615eaa9281540f5d9187fee51ef6f0a8cfa9
Parents: 1993313
Author: Steve Blackmon <sb...@w2odigital.com>
Authored: Sun Mar 23 18:27:20 2014 -0500
Committer: Steve Blackmon <sb...@w2odigital.com>
Committed: Sun Mar 23 18:27:20 2014 -0500
----------------------------------------------------------------------
streams-contrib/streams-persist-hbase/pom.xml | 109 +++
.../apache/streams/hbase/HbaseConfigurator.java | 43 +
.../streams/hbase/HbasePersistWriter.java | 189 ++++
.../streams/hbase/HbasePersistWriterTask.java | 38 +
.../streams/hbase/HbaseConfiguration.json | 41 +
.../src/main/resources/reference.properties | 10 +
.../provider/DatasiftEventProcessor.java | 105 +++
.../provider/DatasiftStreamConfigurator.java | 26 +
.../provider/DatasiftStreamProvider.java | 201 +++++
.../serializer/DatasiftActivitySerializer.java | 181 ++++
.../src/test/resources/part-r-00000.json | 862 +++++++++++++++++++
.../org/apache/streams/core/DatumStatus.java | 7 +
.../apache/streams/core/DatumStatusCounter.java | 41 +
streams-runtimes/pom.xml | 39 +
streams-runtimes/streams-runtime-local/pom.xml | 103 +++
.../jsonschema/LocalRuntimeConfiguration.json | 12 +
.../src/test/resources/TestFile.txt | 4 +
17 files changed, 2011 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/89ff615e/streams-contrib/streams-persist-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-hbase/pom.xml b/streams-contrib/streams-persist-hbase/pom.xml
new file mode 100644
index 0000000..8dc9618
--- /dev/null
+++ b/streams-contrib/streams-persist-hbase/pom.xml
@@ -0,0 +1,109 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>streams-contrib</artifactId>
+ <groupId>org.apache.streams</groupId>
+ <version>0.1-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>streams-persist-hbase</artifactId>
+
+ <repositories>
+ <repository>
+ <id>cloudera</id>
+ <url>https://repository.cloudera.com/artifactory/repo</url>
+ </repository>
+ </repositories>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-pojo</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-util</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>2.0.0-cdh4.5.0</version>
+ <scope>compile</scope>
+ <type>jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase</artifactId>
+ <version>0.94.6-cdh4.5.0</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>2.5.0</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-sources/jsonschema2pojo</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-maven-plugin</artifactId>
+ <configuration>
+ <addCompileSourceRoot>true</addCompileSourceRoot>
+ <generateBuilders>true</generateBuilders>
+ <sourcePaths>
+ <sourcePath>src/main/jsonschema/org/apache/streams/hbase/HbaseConfiguration.json</sourcePath>
+ </sourcePaths>
+ <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
+ <targetPackage>org.apache.streams.hbase.pojo</targetPackage>
+ <useLongIntegers>true</useLongIntegers>
+ <useJodaDates>false</useJodaDates>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/89ff615e/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbaseConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbaseConfigurator.java b/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbaseConfigurator.java
new file mode 100644
index 0000000..283307f
--- /dev/null
+++ b/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbaseConfigurator.java
@@ -0,0 +1,43 @@
+package org.apache.streams.hbase;
+
+import com.typesafe.config.Config;
+import org.apache.streams.config.StreamsConfigurator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Created by sblackmon on 12/10/13.
+ */
+public class HbaseConfigurator {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(HbaseConfigurator.class);
+
+ public static HbaseConfiguration detectConfiguration() {
+
+ Config zookeeper = StreamsConfigurator.config.getConfig("zookeeper");
+ Config hbase = StreamsConfigurator.config.getConfig("hbase");
+
+ String rootdir = hbase.getString("rootdir");
+
+ Config znode = zookeeper.getConfig("znode");
+
+ String rootserver = znode.getString("rootserver");
+ String parent = znode.getString("parent");
+ Integer clientPort = hbase.getConfig("zookeeper").getConfig("property").getInt("clientPort");
+ String quorum = hbase.getConfig("zookeeper").getString("quorum");
+
+ HbaseConfiguration hbaseConfiguration = new HbaseConfiguration();
+
+ hbaseConfiguration.setRootdir(rootdir);
+ hbaseConfiguration.setRootserver(rootserver);
+ hbaseConfiguration.setParent(parent);
+ hbaseConfiguration.setQuorum(quorum);
+ hbaseConfiguration.setClientPort(clientPort.longValue());
+ hbaseConfiguration.setTable(hbase.getString("table"));
+ hbaseConfiguration.setFamily(hbase.getString("family"));
+ hbaseConfiguration.setQualifier(hbase.getString("qualifier"));
+
+ return hbaseConfiguration;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/89ff615e/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistWriter.java b/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistWriter.java
new file mode 100644
index 0000000..4754c0d
--- /dev/null
+++ b/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistWriter.java
@@ -0,0 +1,189 @@
+package org.apache.streams.hbase;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsPersistWriter;
+import org.apache.streams.util.GuidUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.Flushable;
+import java.io.IOException;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+public class HbasePersistWriter implements StreamsPersistWriter, Flushable, Closeable
+{
+ private final static Logger LOGGER = LoggerFactory.getLogger(HbasePersistWriter.class);
+
+ protected HConnection connection;
+ protected HTablePool pool;
+ protected HTableInterface table;
+ protected HTableDescriptor descriptor;
+
+ protected volatile Queue<StreamsDatum> persistQueue;
+
+ private ObjectMapper mapper = new ObjectMapper();
+
+ private HbaseConfiguration config;
+
+ public HbasePersistWriter() {
+ this.config = HbaseConfigurator.detectConfiguration();
+ this.persistQueue = new ConcurrentLinkedQueue<StreamsDatum>();
+ }
+
+ public HbasePersistWriter(Queue<StreamsDatum> persistQueue) {
+ this.config = HbaseConfigurator.detectConfiguration();
+ this.persistQueue = persistQueue;
+ }
+
+ private synchronized void connectToHbase()
+ {
+ Configuration configuration = new Configuration();
+ configuration.set("hbase.rootdir", config.getRootdir());
+ configuration.set("zookeeper.znode.parent", config.getParent());
+ configuration.set("zookeeper.znode.rootserver", config.getRootserver());
+ //configuration.set("hbase.master", config.getRootserver());
+ //configuration.set("hbase.cluster.distributed", "false");
+ configuration.set("hbase.zookeeper.quorum", config.getQuorum());
+ configuration.set("hbase.zookeeper.property.clientPort", Long.toString(config.getClientPort()));
+ configuration.setInt("zookeeper.session.timeout", 1000);
+
+ configuration.setInt("timeout", 1000);
+
+ //pool = new HTablePool(configuration, 10);
+ try {
+ connection = HConnectionManager.createConnection(configuration);
+ } catch (Exception e) {
+ e.printStackTrace();
+ return;
+ }
+
+ try {
+ // table = new HTable(configuration, config.getTable());
+ // table = (HTable) pool.getTable(config.getTable());
+ table = new HTable(configuration, config.getTable().getBytes());
+ table.setAutoFlush(true);
+ } catch (Exception e) {
+ e.printStackTrace();
+ return;
+ }
+ //
+
+ try {
+ descriptor = table.getTableDescriptor();
+ } catch (Exception e) {
+ e.printStackTrace();
+ return;
+ }
+
+ try
+ {
+ LOGGER.info("Table : {}", descriptor);
+ }
+ catch (Exception e)
+ {
+ LOGGER.error("There was an error connecting to HBase, please check your settings and try again");
+ e.printStackTrace();
+ return;
+ }
+ }
+
+ @Override
+ public void write(StreamsDatum streamsDatum) {
+
+ ObjectNode node;
+ Put put = new Put();
+ if( streamsDatum.getDocument() instanceof String ) {
+ try {
+ node = mapper.readValue((String)streamsDatum.getDocument(), ObjectNode.class);
+ } catch (IOException e) {
+ e.printStackTrace();
+ LOGGER.warn("Invalid json: {}", streamsDatum.getDocument().toString());
+ return;
+ }
+ put.setId(GuidUtils.generateGuid(node.toString()));
+ try {
+ byte[] value = node.binaryValue();
+ put.add(config.getFamily().getBytes(), config.getQualifier().getBytes(), value);
+ } catch (IOException e) {
+ e.printStackTrace();
+ LOGGER.warn("Failure adding object: {}", streamsDatum.getDocument().toString());
+ return;
+ }
+ } else {
+ try {
+ node = mapper.valueToTree(streamsDatum.getDocument());
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.warn("Invalid json: {}", streamsDatum.getDocument().toString());
+ return;
+ }
+ put.setId(GuidUtils.generateGuid(node.toString()));
+ try {
+ byte[] value = node.binaryValue();
+ put.add(config.getFamily().getBytes(), config.getQualifier().getBytes(), value);
+ } catch (IOException e) {
+ e.printStackTrace();
+ LOGGER.warn("Failure preparing put: {}", streamsDatum.getDocument().toString());
+ return;
+ }
+ }
+ try {
+ table.put(put);
+ } catch (IOException e) {
+ e.printStackTrace();
+ LOGGER.warn("Failure executin put: {}", streamsDatum.getDocument().toString());
+ return;
+ }
+
+ }
+
+ public void flush() throws IOException
+ {
+ table.flushCommits();
+ }
+
+ public synchronized void close() throws IOException
+ {
+ table.close();
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+
+ connectToHbase();
+
+ Thread task = new Thread(new HbasePersistWriterTask(this));
+ task.start();
+
+ try {
+ task.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ return;
+ }
+
+ }
+
+ @Override
+ public void cleanUp() {
+
+ try {
+ flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/89ff615e/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistWriterTask.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistWriterTask.java b/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistWriterTask.java
new file mode 100644
index 0000000..a09c8ab
--- /dev/null
+++ b/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistWriterTask.java
@@ -0,0 +1,38 @@
+package org.apache.streams.hbase;
+
+import org.apache.streams.core.StreamsDatum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Random;
+
+public class HbasePersistWriterTask implements Runnable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(HbasePersistWriterTask.class);
+
+ private HbasePersistWriter writer;
+
+ public HbasePersistWriterTask(HbasePersistWriter writer) {
+ this.writer = writer;
+ }
+
+ @Override
+ public void run() {
+
+ while(true) {
+ if( writer.persistQueue.peek() != null ) {
+ try {
+ StreamsDatum entry = writer.persistQueue.remove();
+ writer.write(entry);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ try {
+ Thread.sleep(new Random().nextInt(1));
+ } catch (InterruptedException e) {}
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/89ff615e/streams-contrib/streams-persist-hbase/src/main/jsonschema/org/apache/streams/hbase/HbaseConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-hbase/src/main/jsonschema/org/apache/streams/hbase/HbaseConfiguration.json b/streams-contrib/streams-persist-hbase/src/main/jsonschema/org/apache/streams/hbase/HbaseConfiguration.json
new file mode 100644
index 0000000..d3ac8db
--- /dev/null
+++ b/streams-contrib/streams-persist-hbase/src/main/jsonschema/org/apache/streams/hbase/HbaseConfiguration.json
@@ -0,0 +1,41 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType" : "org.apache.streams.hbase.HbaseConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "rootdir": {
+ "type": "string",
+ "description": "Hbase host"
+ },
+ "parent": {
+ "type": "string",
+ "description": "WebHdfs port"
+ },
+ "rootserver": {
+ "type": "string",
+ "description": "Base path"
+ },
+ "quorum": {
+ "type": "string",
+ "description": "User"
+ },
+ "clientPort": {
+ "type": "integer",
+ "description": "ZK Port"
+ },
+ "table": {
+ "type": "string",
+ "description": "Table"
+ },
+ "family": {
+ "type": "string",
+ "description": "Column Family"
+ },
+ "qualifier": {
+ "type": "string",
+ "description": "Qualifier"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/89ff615e/streams-contrib/streams-persist-hbase/src/main/resources/reference.properties
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-hbase/src/main/resources/reference.properties b/streams-contrib/streams-persist-hbase/src/main/resources/reference.properties
new file mode 100644
index 0000000..699f655
--- /dev/null
+++ b/streams-contrib/streams-persist-hbase/src/main/resources/reference.properties
@@ -0,0 +1,10 @@
+hbase.rootdir = "hdfs://localhost:8020/hbase"
+
+zookeeper.znode.parent = "/hbase"
+
+zookeeper.znode.rootserver = "localhost"
+
+hbase.zookeeper.quorum = "localhost"
+
+hbase.zookeeper.property.clientPort = 2181
+
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/89ff615e/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftEventProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftEventProcessor.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftEventProcessor.java
new file mode 100644
index 0000000..3c0aa8b
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftEventProcessor.java
@@ -0,0 +1,105 @@
+package org.apache.streams.datasift.provider;
+
+import com.datasift.client.stream.Interaction;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.datasift.Datasift;
+import org.apache.streams.datasift.serializer.DatasiftActivitySerializer;
+import org.apache.streams.datasift.twitter.Twitter;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.twitter.pojo.Tweet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Queue;
+import java.util.Random;
+
+/**
+ * Created by sblackmon on 12/10/13.
+ */
+public class DatasiftEventProcessor implements Runnable {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftEventProcessor.class);
+
+ private ObjectMapper mapper = new ObjectMapper();
+
+ private Queue<Interaction> inQueue;
+ private Queue<StreamsDatum> outQueue;
+
+ private Class inClass;
+ private Class outClass;
+
+ private DatasiftActivitySerializer datasiftInteractionActivitySerializer = new DatasiftActivitySerializer();
+
+ public final static String TERMINATE = new String("TERMINATE");
+
+ public DatasiftEventProcessor(Queue<Interaction> inQueue, Queue<StreamsDatum> outQueue, Class inClass, Class outClass) {
+ this.inQueue = inQueue;
+ this.outQueue = outQueue;
+ this.inClass = inClass;
+ this.outClass = outClass;
+ }
+
+ public DatasiftEventProcessor(Queue<Interaction> inQueue, Queue<StreamsDatum> outQueue, Class outClass) {
+ this.inQueue = inQueue;
+ this.outQueue = outQueue;
+ this.outClass = outClass;
+ }
+
+ @Override
+ public void run() {
+
+ while(true) {
+ Object item;
+ try {
+ item = inQueue.poll();
+ if(item instanceof String && item.equals(TERMINATE)) {
+ LOGGER.info("Terminating!");
+ break;
+ }
+
+ Thread.sleep(new Random().nextInt(100));
+
+ org.apache.streams.datasift.Datasift datasift = mapper.convertValue(item, Datasift.class);
+
+ // if the target is string, just pass-through
+ if( String.class.equals(outClass)) {
+ outQueue.offer(new StreamsDatum(datasift.toString()));
+
+ }
+ else if( Interaction.class.equals(outClass))
+ {
+ outQueue.offer(new StreamsDatum(item));
+ }
+ else if( Tweet.class.equals(outClass))
+ {
+ // convert to desired format
+ Twitter twitter = datasift.getTwitter();
+
+ Tweet tweet = mapper.convertValue(twitter, Tweet.class);
+
+ if( tweet != null ) {
+
+ outQueue.offer(new StreamsDatum(tweet));
+
+ }
+ }
+ else if( Activity.class.equals(outClass))
+ {
+ // convert to desired format
+ Interaction entry = (Interaction) item;
+ if( entry != null ) {
+ Activity out = datasiftInteractionActivitySerializer.deserialize(datasift);
+
+ if( out != null )
+ outQueue.offer(new StreamsDatum(out));
+ }
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+};
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/89ff615e/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java
new file mode 100644
index 0000000..71c4633
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java
@@ -0,0 +1,26 @@
+package org.apache.streams.datasift.provider;
+
+import com.typesafe.config.Config;
+import org.apache.streams.datasift.DatasiftConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Created by sblackmon on 12/10/13.
+ */
+public class DatasiftStreamConfigurator {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftStreamConfigurator.class);
+
+ public static DatasiftConfiguration detectConfiguration(Config datasift) {
+
+ DatasiftConfiguration datasiftConfiguration = new DatasiftConfiguration();
+
+ datasiftConfiguration.setApiKey(datasift.getString("apiKey"));
+ datasiftConfiguration.setUserName(datasift.getString("userName"));
+ datasiftConfiguration.setStreamHash(datasift.getStringList("hashes"));
+
+ return datasiftConfiguration;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/89ff615e/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java
new file mode 100644
index 0000000..d339385
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java
@@ -0,0 +1,201 @@
+package org.apache.streams.datasift.provider;
+
+import com.datasift.client.DataSiftClient;
+import com.datasift.client.DataSiftConfig;
+import com.datasift.client.core.Stream;
+import com.datasift.client.stream.*;
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.typesafe.config.Config;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProvider;
+import org.apache.streams.core.StreamsResultSet;
+import org.apache.streams.datasift.DatasiftConfiguration;
+import org.joda.time.DateTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.math.BigInteger;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Created by sblackmon on 12/10/13.
+ */
+public class DatasiftStreamProvider implements StreamsProvider {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftStreamProvider.class);
+
+ protected DatasiftConfiguration config = null;
+
+ protected DataSiftClient client;
+
+ private Class klass;
+
+ public DatasiftConfiguration getConfig() {
+ return config;
+ }
+
+ public void setConfig(DatasiftConfiguration config) {
+ this.config = config;
+ }
+
+ protected BlockingQueue inQueue = new LinkedBlockingQueue<Interaction>(10000);
+
+ protected volatile Queue<StreamsDatum> providerQueue = new ConcurrentLinkedQueue<StreamsDatum>();
+
+ public BlockingQueue<Object> getInQueue() {
+ return inQueue;
+ }
+
+ protected ListeningExecutorService executor = MoreExecutors.listeningDecorator(newFixedThreadPoolWithQueueSize(100, 100));
+
+ protected List<String> streamHashes;
+
+ private static ExecutorService newFixedThreadPoolWithQueueSize(int nThreads, int queueSize) {
+ return new ThreadPoolExecutor(nThreads, nThreads,
+ 5000L, TimeUnit.MILLISECONDS,
+ new ArrayBlockingQueue<Runnable>(queueSize, true), new ThreadPoolExecutor.CallerRunsPolicy());
+ }
+
+ public DatasiftStreamProvider() {
+ Config datasiftConfig = StreamsConfigurator.config.getConfig("datasift");
+ this.config = DatasiftStreamConfigurator.detectConfiguration(datasiftConfig);
+ }
+
+ public DatasiftStreamProvider(DatasiftConfiguration config) {
+ this.config = config;
+ }
+
+ public DatasiftStreamProvider(Class klass) {
+ Config config = StreamsConfigurator.config.getConfig("datasift");
+ this.config = DatasiftStreamConfigurator.detectConfiguration(config);
+ this.klass = klass;
+ }
+
+ public DatasiftStreamProvider(DatasiftConfiguration config, Class klass) {
+ this.config = config;
+ this.klass = klass;
+ }
+
+ @Override
+ public void startStream() {
+
+ Preconditions.checkNotNull(this.klass);
+
+ Preconditions.checkNotNull(config);
+
+ Preconditions.checkNotNull(config.getStreamHash());
+
+ Preconditions.checkNotNull(config.getStreamHash().get(0));
+
+ for( String hash : config.getStreamHash()) {
+
+ client.liveStream().subscribe(new Subscription(Stream.fromString(hash)));
+
+ }
+
+ for( int i = 0; i < ((config.getStreamHash().size() / 5) + 1); i++ )
+ executor.submit(new DatasiftEventProcessor(inQueue, providerQueue, klass));
+
+ }
+
+ public void stop() {
+
+ for( String hash : config.getStreamHash()) {
+
+ client.liveStream().subscribe(new Subscription(Stream.fromString(hash)));
+
+ }
+ }
+
+ public Queue<StreamsDatum> getProviderQueue() {
+ return this.providerQueue;
+ }
+
+ @Override
+ public StreamsResultSet readCurrent() {
+
+ return (StreamsResultSet) providerQueue;
+
+ }
+
+ @Override
+ public StreamsResultSet readNew(BigInteger sequence) {
+ return null;
+ }
+
+ @Override
+ public StreamsResultSet readRange(DateTime start, DateTime end) {
+ return null;
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+
+ Preconditions.checkNotNull(config);
+
+ String apiKey = config.getApiKey();
+ String userName = config.getUserName();
+
+ DataSiftConfig config = new DataSiftConfig(userName, apiKey);
+
+ client = new DataSiftClient(config);
+
+ client.liveStream().onError(new ErrorHandler());
+
+ //handle delete message
+ client.liveStream().onStreamEvent(new DeleteHandler());
+
+ }
+
+ @Override
+ public void cleanUp() {
+ stop();
+ }
+
+ public class Subscription extends StreamSubscription {
+ AtomicLong count = new AtomicLong();
+
+ public Subscription(Stream stream) {
+ super(stream);
+ }
+
+ public void onDataSiftLogMessage(DataSiftMessage di) {
+ //di.isWarning() is also available
+ System.out.println((di.isError() ? "Error" : di.isInfo() ? "Info" : "Warning") + ":\n" + di);
+ }
+
+ public void onMessage(Interaction i) {
+
+ LOGGER.debug("Processing:\n" + i);
+
+ inQueue.offer(i);
+
+ if (count.incrementAndGet() % 1000 == 0) {
+ LOGGER.info("Processed {}:\n " + count.get());
+
+ }
+
+ }
+ }
+
+ public class DeleteHandler extends StreamEventListener {
+ public void onDelete(DeletedInteraction di) {
+ //go off and delete the interaction if you have it stored. This is a strict requirement!
+ LOGGER.info("DELETED:\n " + di);
+ }
+ }
+
+ public class ErrorHandler extends ErrorListener {
+ public void exceptionCaught(Throwable t) {
+ LOGGER.warn(t.getMessage());
+ //do something useful...
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/89ff615e/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java
new file mode 100644
index 0000000..c73abd7
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java
@@ -0,0 +1,181 @@
+package org.apache.streams.datasift.serializer;
+
+import com.fasterxml.jackson.databind.AnnotationIntrospector;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
+import org.apache.streams.data.ActivitySerializer;
+import org.apache.streams.datasift.Datasift;
+import org.apache.streams.datasift.interaction.Interaction;
+import org.apache.streams.pojo.json.*;
+
+import java.io.Serializable;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
+
+/**
+* Created with IntelliJ IDEA.
+* User: mdelaet
+* Date: 9/30/13
+* Time: 9:24 AM
+* To change this template use File | Settings | File Templates.
+*/
+public class DatasiftActivitySerializer implements ActivitySerializer<Datasift>, Serializable {
+
+ public static final String DATE_FORMAT = "EEE MMM dd HH:mm:ss Z yyyy";
+
+ ObjectMapper mapper = new ObjectMapper();
+
+ @Override
+ public String serializationFormat() {
+ return "application/json+datasift.com.v1.1";
+ }
+
+ @Override
+ public Datasift serialize(Activity deserialized) {
+ throw new UnsupportedOperationException("Cannot currently serialize to Datasift JSON");
+ }
+
+ @Override
+ public Activity deserialize(Datasift serialized) {
+
+ AnnotationIntrospector introspector = new JaxbAnnotationIntrospector(mapper.getTypeFactory());
+ mapper.setAnnotationIntrospector(introspector);
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
+ mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
+ mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
+ mapper.configure(DeserializationFeature.WRAP_EXCEPTIONS, Boolean.TRUE);
+
+ try {
+
+ Activity activity = convert(serialized);
+
+ return activity;
+
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Unable to deserialize", e);
+ }
+
+ }
+
+ @Override
+ public List<Activity> deserializeAll(List<Datasift> datasifts) {
+ List<Activity> activities = Lists.newArrayList();
+ for( Datasift datasift : datasifts ) {
+ activities.add(deserialize(datasift));
+ }
+ return activities;
+ }
+
+ public static Date parse(String str) {
+ Date date;
+ String dstr;
+ DateFormat fmt = new SimpleDateFormat(DATE_FORMAT);
+ DateFormat out = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
+ try {
+ date = fmt.parse(str);
+ dstr = out.format(date);
+ return out.parse(dstr);
+ } catch (ParseException e) {
+ throw new IllegalArgumentException("Invalid date format", e);
+ }
+ }
+
+ public static Generator buildGenerator(Interaction interaction) {
+ return null;
+ }
+
+ public static Icon getIcon(Interaction interaction) {
+ return null;
+ }
+
+ public static Provider buildProvider(Interaction interaction) {
+ Provider provider = new Provider();
+ provider.setId("id:providers:twitter");
+ return provider;
+ }
+
+ public static String getUrls(Interaction interaction) {
+ return null;
+ }
+
+ public static void addDatasiftExtension(Activity activity, Datasift datasift) {
+ Map<String, Object> extensions = org.apache.streams.data.util.ActivityUtil.ensureExtensions(activity);
+ extensions.put("datasift", datasift);
+ }
+
+ public static String formatId(String... idparts) {
+ return Joiner.on(":").join(Lists.asList("id:datasift", idparts));
+ }
+
+ public Activity convert(Datasift event) {
+
+ Activity activity = new Activity();
+ activity.setActor(buildActor(event.getInteraction()));
+ activity.setVerb("post");
+ activity.setObject(buildActivityObject(event.getInteraction()));
+ activity.setId(formatId(activity.getVerb(), event.getInteraction().getId()));
+ activity.setTarget(buildTarget(event.getInteraction()));
+ activity.setPublished(parse(event.getInteraction().getCreatedAt()));
+ activity.setGenerator(buildGenerator(event.getInteraction()));
+ activity.setIcon(getIcon(event.getInteraction()));
+ activity.setProvider(buildProvider(event.getInteraction()));
+ activity.setTitle(event.getInteraction().getTitle());
+ activity.setContent(event.getInteraction().getContent());
+ activity.setUrl(event.getInteraction().getLink());
+ activity.setLinks(getLinks(event.getInteraction()));
+ addDatasiftExtension(activity, event);
+ if( event.getInteraction().getGeo() != null) {
+ addLocationExtension(activity, event.getInteraction());
+ }
+ return activity;
+ }
+
+ public static Actor buildActor(Interaction interaction) {
+ Actor actor = new Actor();
+ actor.setId(formatId(interaction.getAuthor().getId().toString()));
+ actor.setDisplayName(interaction.getAuthor().getUsername());
+ Image image = new Image();
+ image.setUrl(interaction.getAuthor().getAvatar());
+ actor.setImage(image);
+ if (interaction.getAuthor().getLink()!=null){
+ actor.setUrl(interaction.getAuthor().getLink());
+ }
+ return actor;
+ }
+
+ public static ActivityObject buildActivityObject(Interaction interaction) {
+ ActivityObject actObj = new ActivityObject();
+ actObj.setObjectType(interaction.getContenttype());
+ return actObj;
+ }
+
+ public static List<Object> getLinks(Interaction interaction) {
+ List<Object> links = Lists.newArrayList();
+ return links;
+ }
+
+ public static ActivityObject buildTarget(Interaction interaction) {
+ return null;
+ }
+
+ public static void addLocationExtension(Activity activity, Interaction interaction) {
+ Map<String, Object> extensions = ensureExtensions(activity);
+ Map<String, Object> location = new HashMap<String, Object>();
+ Map<String, Double> coordinates = new HashMap<String, Double>();
+ coordinates.put("latitude", interaction.getGeo().getLatitude());
+ coordinates.put("longitude", interaction.getGeo().getLongitude());
+ location.put("coordinates", coordinates);
+ extensions.put("location", location);
+ }
+
+}
[13/13] git commit: dropping streams-pojo-extensions,
BC activity can be extended without it move streams cassandra under
contrib move streams-eip-routes under runtimes,
renaming streams-runtime-webapp
Posted by sb...@apache.org.
dropping streams-pojo-extensions, BC activity can be extended without it
move streams cassandra under contrib
move streams-eip-routes under runtimes, renaming streams-runtime-webapp
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/adb43b29
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/adb43b29
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/adb43b29
Branch: refs/heads/springcleaning
Commit: adb43b29567351e8849385be95806d4be2fcb7e5
Parents: 381d758
Author: Steve Blackmon <sb...@w2odigital.com>
Authored: Mon Mar 24 03:07:06 2014 -0500
Committer: Steve Blackmon <sb...@w2odigital.com>
Committed: Mon Mar 24 03:07:06 2014 -0500
----------------------------------------------------------------------
pom.xml | 8 +-
streams-cassandra/pom.xml | 142 -------------
.../configuration/CassandraConfiguration.java | 63 ------
.../model/CassandraActivityStreamsEntry.java | 45 ----
.../CassandraActivityStreamsRepository.java | 176 ----------------
.../repository/impl/CassandraKeyspace.java | 43 ----
.../impl/CassandraSubscriptionRepository.java | 69 ------
.../spring/streams-cassandra-context.xml | 25 ---
.../CassandraActivityStreamsRepositoryTest.java | 99 ---------
.../impl/CassandraActivitySubscriptionTest.java | 54 -----
streams-config-graph/pom.xml | 71 -------
.../config/graph/PipelineGraphConfigurator.java | 88 --------
.../streams-persist-cassandra/pom.xml | 142 +++++++++++++
.../configuration/CassandraConfiguration.java | 63 ++++++
.../model/CassandraActivityStreamsEntry.java | 43 ++++
.../CassandraActivityStreamsRepository.java | 176 ++++++++++++++++
.../repository/impl/CassandraKeyspace.java | 43 ++++
.../impl/CassandraSubscriptionRepository.java | 69 ++++++
.../spring/streams-cassandra-context.xml | 25 +++
.../CassandraActivityStreamsRepositoryTest.java | 99 +++++++++
.../impl/CassandraActivitySubscriptionTest.java | 54 +++++
streams-eip-routes/ReadMe.txt | 32 ---
streams-eip-routes/pom.xml | 209 -------------------
.../aggregation/ActivityAggregator.java | 59 ------
.../configuration/EipConfigurator.java | 201 ------------------
.../ActivityPublisherRegistrationProcessor.java | 73 -------
...yStreamsSubscriberRegistrationProcessor.java | 94 ---------
.../routers/ActivityConsumerRouteBuilder.java | 32 ---
.../ActivityStreamsSubscriberRouteBuilder.java | 32 ---
.../routers/impl/ActivityConsumerRouter.java | 144 -------------
.../impl/ActivityStreamsSubscriberRouter.java | 142 -------------
.../messaging/service/ActivityService.java | 31 ---
.../messaging/service/SubscriptionService.java | 29 ---
.../service/impl/CassandraActivityService.java | 96 ---------
.../impl/CassandraSubscriptionService.java | 43 ----
.../META-INF/spring/propertiesLoader.xml | 35 ----
.../spring/streams-eip-applicationContext.xml | 113 ----------
.../streams-eip-osgi-component-import.xml | 38 ----
.../META-INF/spring/streamsCamelContext.xml | 96 ---------
.../main/resources/META-INF/streams.properties | 41 ----
.../impl/CassandraActivityServiceTest.java | 87 --------
streams-pojo-extensions/pom.xml | 149 -------------
.../org/apache/streams/ActivityExtended.json | 17 --
.../pojo/test/ActivityExtendedSerDeTest.java | 108 ----------
.../test/resources/gnip_twitter_extended.json | 146 -------------
streams-runtimes/pom.xml | 1 +
.../streams-runtime-webapp/ReadMe.txt | 32 +++
streams-runtimes/streams-runtime-webapp/pom.xml | 209 +++++++++++++++++++
.../aggregation/ActivityAggregator.java | 59 ++++++
.../configuration/EipConfigurator.java | 201 ++++++++++++++++++
.../ActivityPublisherRegistrationProcessor.java | 72 +++++++
...yStreamsSubscriberRegistrationProcessor.java | 93 +++++++++
.../routers/ActivityConsumerRouteBuilder.java | 32 +++
.../ActivityStreamsSubscriberRouteBuilder.java | 32 +++
.../routers/impl/ActivityConsumerRouter.java | 144 +++++++++++++
.../impl/ActivityStreamsSubscriberRouter.java | 141 +++++++++++++
.../messaging/service/ActivityService.java | 31 +++
.../messaging/service/SubscriptionService.java | 29 +++
.../service/impl/CassandraActivityService.java | 96 +++++++++
.../impl/CassandraSubscriptionService.java | 43 ++++
.../META-INF/spring/propertiesLoader.xml | 35 ++++
.../spring/streams-eip-applicationContext.xml | 113 ++++++++++
.../streams-eip-osgi-component-import.xml | 38 ++++
.../META-INF/spring/streamsCamelContext.xml | 96 +++++++++
.../main/resources/META-INF/streams.properties | 41 ++++
.../impl/CassandraActivityServiceTest.java | 86 ++++++++
66 files changed, 2340 insertions(+), 2928 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c552cf1..a476c07 100644
--- a/pom.xml
+++ b/pom.xml
@@ -88,17 +88,13 @@
<module>poms</module>
<module>provision</module>
<module>streams-osgi-components</module>
- <module>streams-eip-routes</module>
- <module>streams-cassandra</module>
- <module>streams-web</module>
+ <module>streams-core</module>
<module>streams-config</module>
- <module>streams-config-graph</module>
<module>streams-pojo</module>
<module>streams-util</module>
- <module>streams-pojo-extensions</module>
<module>streams-contrib</module>
- <module>streams-core</module>
<module>streams-runtimes</module>
+ <module>streams-web</module>
</modules>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-cassandra/pom.xml
----------------------------------------------------------------------
diff --git a/streams-cassandra/pom.xml b/streams-cassandra/pom.xml
deleted file mode 100644
index 4837649..0000000
--- a/streams-cassandra/pom.xml
+++ /dev/null
@@ -1,142 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns="http://maven.apache.org/POM/4.0.0"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-project</artifactId>
- <version>0.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-cassandra</artifactId>
-
- <name>${bundle.symbolicName} [${bundle.namespace}]</name>
-
- <properties>
- <bundle.symbolicName>streams-cassandra</bundle.symbolicName>
- <bundle.namespace>org.apache.streams</bundle.namespace>
- <easymock.version>3.2</easymock.version>
- </properties>
-
- <packaging>bundle</packaging>
- <build>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
-
- <resource>
- <directory>.</directory>
- <includes>
- <include>plugin.xml</include>
- <include>plugin.properties</include>
- <include>icons/**</include>
- </includes>
- </resource>
- </resources>
- <plugins>
- <plugin>
- <groupId>org.ops4j</groupId>
- <artifactId>maven-pax-plugin</artifactId>
- <!--
- | enable improved OSGi compilation support for the bundle life-cycle.
- | to switch back to the standard bundle life-cycle, move this setting
- | down to the maven-bundle-plugin section
- -->
- <extensions>true</extensions>
- </plugin>
- <plugin>
- <groupId>org.apache.felix</groupId>
- <artifactId>maven-bundle-plugin</artifactId>
- <version>1.4.3</version>
- <!--
- | the following instructions build a simple set of public/private classes into an OSGi bundle
- -->
- <configuration>
- <instructions>
- <Bundle-SymbolicName>${bundle.symbolicName}</Bundle-SymbolicName>
- <Bundle-Version>${project.version}</Bundle-Version>
- <Export-Package>
- ${bundle.namespace};version="${project.version}",org.apache.streams.cassandra.repository.impl, org.apache.streams.cassandra.model, org.apache.streams.cassandra.configuration
- </Export-Package>
- <Private-Package>${bundle.namespace}.cassandra.repository.impl.*,${bundle.namespace}.cassandra.model, ${bundle.namespace}.cassandra.configuration </Private-Package>
- <Import-Package>
- org.apache.rave.model,org.apache.rave.portal.model.impl,
- com.datastax.driver.core, com.datastax.driver.core.exceptions, org.codehaus.jackson.map.annotate,
- javax.persistence, org.apache.commons.logging, com.google.common.collect, org.codehaus.jackson.map,
- org.apache.commons.lang,
- org.apache.streams.osgi.components.activitysubscriber,
- org.springframework.beans.factory.annotation, org.springframework.stereotype
- </Import-Package>
- </instructions>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.rave</groupId>
- <artifactId>rave-core-api</artifactId>
- <version>${rave.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.rave</groupId>
- <artifactId>rave-core</artifactId>
- <version>${rave.version}</version>
- </dependency>
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.8.2</version>
- </dependency>
-
- <dependency>
- <groupId>com.datastax.cassandra</groupId>
- <artifactId>cassandra-driver-core</artifactId>
- <version>${datastax.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- <version>3.2.9.Final</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams.osgi.components</groupId>
- <artifactId>activity-subscriber</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.easymock</groupId>
- <artifactId>easymock</artifactId>
- <version>${easymock.version}</version>
- <scope>test</scope>
- </dependency>
-
- </dependencies>
-
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-cassandra/src/main/java/org/apache/streams/cassandra/configuration/CassandraConfiguration.java
----------------------------------------------------------------------
diff --git a/streams-cassandra/src/main/java/org/apache/streams/cassandra/configuration/CassandraConfiguration.java b/streams-cassandra/src/main/java/org/apache/streams/cassandra/configuration/CassandraConfiguration.java
deleted file mode 100644
index 39e74a7..0000000
--- a/streams-cassandra/src/main/java/org/apache/streams/cassandra/configuration/CassandraConfiguration.java
+++ /dev/null
@@ -1,63 +0,0 @@
-package org.apache.streams.cassandra.configuration;
-
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-
-@Component
-public class CassandraConfiguration {
- @Value("${keyspaceName}")
- private String keyspaceName;
-
- @Value("${activitystreamsColumnFamilyName}")
- private String activitystreamsColumnFamilyName;
-
- @Value("${subscriptionColumnFamilyName}")
- private String subscriptionColumnFamilyName;
-
- @Value("${publisherColumnFamilyName}")
- private String publisherColumnFamilyName;
-
- @Value("${cassandraPort}")
- private String cassandraPort;
-
- public String getKeyspaceName() {
- return keyspaceName;
- }
-
- public void setKeyspaceName(String keyspaceName) {
- this.keyspaceName = keyspaceName;
- }
-
- public String getActivitystreamsColumnFamilyName() {
- return activitystreamsColumnFamilyName;
- }
-
- public void setActivitystreamsColumnFamilyName(String activitystreamsColumnFamilyName) {
- this.activitystreamsColumnFamilyName = activitystreamsColumnFamilyName;
- }
-
- public String getSubscriptionColumnFamilyName() {
- return subscriptionColumnFamilyName;
- }
-
- public void setSubscriptionColumnFamilyName(String subscriptionColumnFamilyName) {
- this.subscriptionColumnFamilyName = subscriptionColumnFamilyName;
- }
-
- public String getPublisherColumnFamilyName() {
- return publisherColumnFamilyName;
- }
-
- public void setPublisherColumnFamilyName(String publisherColumnFamilyName) {
- this.publisherColumnFamilyName = publisherColumnFamilyName;
- }
-
- public String getCassandraPort() {
- return cassandraPort;
- }
-
- public void setCassandraPort(String cassandraPort) {
- this.cassandraPort = cassandraPort;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-cassandra/src/main/java/org/apache/streams/cassandra/model/CassandraActivityStreamsEntry.java
----------------------------------------------------------------------
diff --git a/streams-cassandra/src/main/java/org/apache/streams/cassandra/model/CassandraActivityStreamsEntry.java b/streams-cassandra/src/main/java/org/apache/streams/cassandra/model/CassandraActivityStreamsEntry.java
deleted file mode 100644
index 2a2321a..0000000
--- a/streams-cassandra/src/main/java/org/apache/streams/cassandra/model/CassandraActivityStreamsEntry.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.cassandra.model;
-
-import org.apache.rave.model.ActivityStreamsObject;
-import org.apache.rave.portal.model.impl.ActivityStreamsEntryImpl;
-import org.apache.rave.portal.model.impl.ActivityStreamsObjectImpl;
-import org.codehaus.jackson.map.annotate.JsonDeserialize;
-
-import java.util.Date;
-
-public class CassandraActivityStreamsEntry extends ActivityStreamsEntryImpl implements Comparable<CassandraActivityStreamsEntry>{
-
- @JsonDeserialize(as=ActivityStreamsObjectImpl.class)
- private ActivityStreamsObject object;
-
- @JsonDeserialize(as=ActivityStreamsObjectImpl.class)
- private ActivityStreamsObject target;
-
- @JsonDeserialize(as=ActivityStreamsObjectImpl.class)
- private ActivityStreamsObject actor;
-
- @JsonDeserialize(as=ActivityStreamsObjectImpl.class)
- private ActivityStreamsObject provider;
-
- public int compareTo(CassandraActivityStreamsEntry entry){
- return (this.getPublished()).compareTo(entry.getPublished());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepository.java
----------------------------------------------------------------------
diff --git a/streams-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepository.java b/streams-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepository.java
deleted file mode 100644
index 56e5416..0000000
--- a/streams-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepository.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.cassandra.repository.impl;
-
-import com.datastax.driver.core.Cluster;
-import com.datastax.driver.core.ResultSet;
-import com.datastax.driver.core.Row;
-import com.datastax.driver.core.Session;
-import com.datastax.driver.core.exceptions.AlreadyExistsException;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.rave.model.ActivityStreamsEntry;
-import org.apache.rave.model.ActivityStreamsObject;
-import org.apache.rave.portal.model.impl.ActivityStreamsEntryImpl;
-import org.apache.rave.portal.model.impl.ActivityStreamsObjectImpl;
-import org.apache.streams.cassandra.configuration.CassandraConfiguration;
-import org.apache.streams.cassandra.model.CassandraActivityStreamsEntry;
-import org.springframework.beans.factory.annotation.Autowired;
-
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-
-public class CassandraActivityStreamsRepository {
-
- private static final Log LOG = LogFactory.getLog(CassandraActivityStreamsRepository.class);
-
- private CassandraKeyspace keyspace;
- private CassandraConfiguration configuration;
-
- @Autowired
- public CassandraActivityStreamsRepository(CassandraKeyspace keyspace, CassandraConfiguration configuration) {
- this.configuration = configuration;
- this.keyspace = keyspace;
-
- try {
- keyspace.getSession().execute("CREATE TABLE " + configuration.getActivitystreamsColumnFamilyName() + " (" +
- "id text, " +
- "published timestamp, " +
- "verb text, " +
- "tags text, " +
-
- "actor_displayname text, " +
- "actor_id text, " +
- "actor_url text, " +
- "actor_objecttype text, " +
-
- "target_displayname text, " +
- "target_id text, " +
- "target_url text, " +
-
- "provider_url text, " +
-
- "object_url text, " +
- "object_displayname text, " +
- "object_id text, " +
- "object_objecttype text, " +
-
- "PRIMARY KEY (id, tags, published));");
- } catch (AlreadyExistsException ignored) {
- }
- }
-
- public void save(ActivityStreamsEntry entry) {
- String sql = "INSERT INTO " + configuration.getActivitystreamsColumnFamilyName() + " (" +
- "id, published, verb, tags, " +
- "actor_displayname, actor_objecttype, actor_id, actor_url, " +
- "target_displayname, target_id, target_url, " +
- "provider_url, " +
- "object_displayname, object_objecttype, object_id, object_url) " +
- "VALUES ('" +
- entry.getId() + "','" +
- entry.getPublished().getTime() + "','" +
- entry.getVerb() + "','" +
- entry.getTags() + "','" +
-
- entry.getActor().getDisplayName() + "','" +
- entry.getActor().getObjectType() + "','" +
- entry.getActor().getId() + "','" +
- entry.getActor().getUrl() + "','" +
-
- entry.getTarget().getDisplayName() + "','" +
- entry.getTarget().getId() + "','" +
- entry.getTarget().getUrl() + "','" +
-
- entry.getProvider().getUrl() + "','" +
-
- entry.getObject().getDisplayName() + "','" +
- entry.getObject().getObjectType() + "','" +
- entry.getObject().getId() + "','" +
- entry.getObject().getUrl() +
-
- "')";
- keyspace.getSession().execute(sql);
- }
-
- public List<CassandraActivityStreamsEntry> getActivitiesForFilters(List<String> filters, Date lastUpdated) {
- List<CassandraActivityStreamsEntry> results = new ArrayList<CassandraActivityStreamsEntry>();
-
- for (String tag : filters) {
- String cql = "SELECT * FROM " + configuration.getActivitystreamsColumnFamilyName() + " WHERE ";
-
- //add filters
- cql = cql + " tags = '" + tag + "' AND ";
-
- //specify last modified
- cql = cql + "published > " + lastUpdated.getTime() + " ALLOW FILTERING";
-
- //execute the cql query and store the results
- ResultSet set = keyspace.getSession().execute(cql);
-
- //iterate through the results and create a new ActivityStreamsEntry for every result returned
-
- for (Row row : set) {
- CassandraActivityStreamsEntry entry = new CassandraActivityStreamsEntry();
- ActivityStreamsObject actor = new ActivityStreamsObjectImpl();
- ActivityStreamsObject target = new ActivityStreamsObjectImpl();
- ActivityStreamsObject object = new ActivityStreamsObjectImpl();
- ActivityStreamsObject provider = new ActivityStreamsObjectImpl();
-
- actor.setDisplayName(row.getString("actor_displayname"));
- actor.setId(row.getString("actor_id"));
- actor.setObjectType(row.getString("actor_objecttype"));
- actor.setUrl(row.getString("actor_url"));
-
- target.setDisplayName(row.getString("target_displayname"));
- target.setId(row.getString("target_id"));
- target.setUrl(row.getString("target_url"));
-
- object.setDisplayName(row.getString("object_displayname"));
- object.setObjectType(row.getString("object_objecttype"));
- object.setUrl(row.getString("object_url"));
- object.setId(row.getString("object_id"));
-
- provider.setUrl(row.getString("provider_url"));
-
- entry.setPublished(row.getDate("published"));
- entry.setVerb(row.getString("verb"));
- entry.setId(row.getString("id"));
- entry.setTags(row.getString("tags"));
- entry.setActor(actor);
- entry.setTarget(target);
- entry.setObject(object);
- entry.setProvider(provider);
-
- results.add(entry);
- }
- }
-
- return results;
- }
-
- public void dropTable(String table) {
- String cql = "DROP TABLE " + table;
- keyspace.getSession().execute(cql);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraKeyspace.java
----------------------------------------------------------------------
diff --git a/streams-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraKeyspace.java b/streams-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraKeyspace.java
deleted file mode 100644
index 0551bf2..0000000
--- a/streams-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraKeyspace.java
+++ /dev/null
@@ -1,43 +0,0 @@
-package org.apache.streams.cassandra.repository.impl;
-
-import com.datastax.driver.core.Cluster;
-import com.datastax.driver.core.Session;
-import com.datastax.driver.core.exceptions.AlreadyExistsException;
-import org.apache.streams.cassandra.configuration.CassandraConfiguration;
-import org.springframework.beans.factory.annotation.Autowired;
-
-public class CassandraKeyspace {
- private CassandraConfiguration configuration;
- private Cluster cluster;
- private Session session;
-
- @Autowired
- public CassandraKeyspace(CassandraConfiguration configuration){
- this.configuration = configuration;
-
- cluster = Cluster.builder().addContactPoint(configuration.getCassandraPort()).build();
- session = cluster.connect();
-
- //TODO: cassandra 2 will have support for CREATE KEYSPACE IF NOT EXISTS
- try {
- session.execute("CREATE KEYSPACE " + configuration.getKeyspaceName() + " WITH replication = { 'class': 'SimpleStrategy','replication_factor' : 1 };");
- } catch (AlreadyExistsException ignored) {
- }
-
- //connect to the keyspace
- session = cluster.connect(configuration.getKeyspaceName());
- }
-
- public Session getSession(){
- return session;
- }
-
- @Override
- protected void finalize() throws Throwable {
- try {
- cluster.shutdown();
- } finally {
- super.finalize();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraSubscriptionRepository.java
----------------------------------------------------------------------
diff --git a/streams-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraSubscriptionRepository.java b/streams-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraSubscriptionRepository.java
deleted file mode 100644
index f5fe471..0000000
--- a/streams-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraSubscriptionRepository.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.cassandra.repository.impl;
-
-import com.datastax.driver.core.ResultSet;
-import com.datastax.driver.core.exceptions.AlreadyExistsException;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.streams.cassandra.configuration.CassandraConfiguration;
-import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscription;
-import org.springframework.beans.factory.annotation.Autowired;
-
-public class CassandraSubscriptionRepository {
- private static final Log LOG = LogFactory.getLog(CassandraSubscriptionRepository.class);
-
- private CassandraKeyspace keyspace;
- private CassandraConfiguration configuration;
-
- @Autowired
- public CassandraSubscriptionRepository(CassandraKeyspace keyspace, CassandraConfiguration configuration) {
- this.keyspace = keyspace;
- this.configuration = configuration;
-
- try {
- keyspace.getSession().execute("CREATE TABLE " + configuration.getSubscriptionColumnFamilyName() + " (" +
- "id text, " +
- "filters text, " +
-
- "PRIMARY KEY (id));");
- } catch (AlreadyExistsException ignored) {
- }
- }
-
- public String getFilters(String id){
- String cql = "SELECT * FROM " + configuration.getSubscriptionColumnFamilyName() + " WHERE id = '" + id+"';";
-
- ResultSet set = keyspace.getSession().execute(cql);
-
- return set.one().getString("filters");
- }
-
- public void save(ActivityStreamsSubscription subscription){
- String cql = "INSERT INTO " + configuration.getSubscriptionColumnFamilyName() + " (" +
- "id, filters) " +
- "VALUES ('" +
- subscription.getAuthToken() + "','" +
- StringUtils.join(subscription.getFilters(), " ") +
-
- "')";
- keyspace.getSession().execute(cql);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-cassandra/src/main/resources/META_INF/spring/streams-cassandra-context.xml
----------------------------------------------------------------------
diff --git a/streams-cassandra/src/main/resources/META_INF/spring/streams-cassandra-context.xml b/streams-cassandra/src/main/resources/META_INF/spring/streams-cassandra-context.xml
deleted file mode 100644
index 842c918..0000000
--- a/streams-cassandra/src/main/resources/META_INF/spring/streams-cassandra-context.xml
+++ /dev/null
@@ -1,25 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<beans
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns="http://www.springframework.org/schema/beans"
- xmlns:context="http://www.springframework.org/schema/context"
- xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
- http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">
-
-</beans>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepositoryTest.java
----------------------------------------------------------------------
diff --git a/streams-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepositoryTest.java b/streams-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepositoryTest.java
deleted file mode 100644
index 978af10..0000000
--- a/streams-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepositoryTest.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.cassandra.repository.impl;
-
-import com.datastax.driver.core.ResultSet;
-import org.apache.rave.model.ActivityStreamsEntry;
-import org.apache.rave.model.ActivityStreamsObject;
-import org.apache.rave.portal.model.impl.ActivityStreamsEntryImpl;
-import org.apache.rave.portal.model.impl.ActivityStreamsObjectImpl;
-import org.apache.streams.cassandra.configuration.CassandraConfiguration;
-import org.apache.streams.cassandra.model.CassandraActivityStreamsEntry;
-import static org.easymock.EasyMock.*;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
-
-public class CassandraActivityStreamsRepositoryTest {
-
- private CassandraActivityStreamsRepository repository;
-
-
- @Before
- public void setup() {
- CassandraKeyspace keyspace = createMock(CassandraKeyspace.class);
- CassandraConfiguration configuration = createMock(CassandraConfiguration.class);
- repository = new CassandraActivityStreamsRepository(keyspace, configuration);
- }
-
- @Ignore
- @Test
- public void saveActivity() {
- ActivityStreamsEntry entry = new ActivityStreamsEntryImpl();
- ActivityStreamsObject actor = new ActivityStreamsObjectImpl();
- ActivityStreamsObject target = new ActivityStreamsObjectImpl();
- ActivityStreamsObject object = new ActivityStreamsObjectImpl();
- ActivityStreamsObject provider = new ActivityStreamsObjectImpl();
-
- actor.setId("actorid1");
- actor.setUrl("actorurl1");
- actor.setDisplayName("actorname1");
-
- target.setId("targetid1");
- target.setUrl("targeturl1");
- target.setDisplayName("r501");
-
- provider.setUrl("providerurl");
-
- object.setId("objectid1");
- object.setDisplayName("objectname1");
-
- entry.setId("dink");
- entry.setVerb("verb1");
- entry.setTags("r501");
- entry.setProvider(provider);
- Date d = new Date();
- entry.setPublished(d);
- entry.setActor(actor);
- entry.setObject(object);
- entry.setTarget(target);
-
- repository.save(entry);
- }
-
- @Ignore
- @Test
- public void getActivity() {
- String cql = "tags";
- String other = "r501";
- List<String> f = Arrays.asList(cql, other);
- Date d = new Date(0);
- List<CassandraActivityStreamsEntry> results = repository.getActivitiesForFilters(f,d);
- }
-
- @Ignore
- @Test
- public void dropTableTest(){
- repository.dropTable("coltest");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivitySubscriptionTest.java
----------------------------------------------------------------------
diff --git a/streams-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivitySubscriptionTest.java b/streams-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivitySubscriptionTest.java
deleted file mode 100644
index 2a90462..0000000
--- a/streams-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivitySubscriptionTest.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.cassandra.repository.impl;
-
-import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscription;
-import org.apache.streams.osgi.components.activitysubscriber.impl.ActivityStreamsSubscriptionImpl;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.util.Arrays;
-
-public class CassandraActivitySubscriptionTest {
-
- public CassandraSubscriptionRepository repository;
-
-
- @Before
- public void setup() {
-// repository = new CassandraSubscriptionRepository();
- }
-
- @Ignore
- @Test
- public void saveTest(){
- ActivityStreamsSubscription subscription = new ActivityStreamsSubscriptionImpl();
- subscription.setFilters(Arrays.asList("thisis", "atest"));
- subscription.setAuthToken("subid");
-
- repository.save(subscription);
- }
-
- @Ignore
- @Test
- public void getTest(){
- String filters = repository.getFilters("subid");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-config-graph/pom.xml
----------------------------------------------------------------------
diff --git a/streams-config-graph/pom.xml b/streams-config-graph/pom.xml
deleted file mode 100644
index d4a3dbb..0000000
--- a/streams-config-graph/pom.xml
+++ /dev/null
@@ -1,71 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-project</artifactId>
- <version>0.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-config-graph</artifactId>
-
- <dependencies>
- <dependency>
- <artifactId>gs-core</artifactId>
- <groupId>org.graphstream</groupId>
- <version>1.2</version>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- </build>
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-config-graph/src/main/java/org/apache/streams/config/graph/PipelineGraphConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-config-graph/src/main/java/org/apache/streams/config/graph/PipelineGraphConfigurator.java b/streams-config-graph/src/main/java/org/apache/streams/config/graph/PipelineGraphConfigurator.java
deleted file mode 100644
index 3d7b6dd..0000000
--- a/streams-config-graph/src/main/java/org/apache/streams/config/graph/PipelineGraphConfigurator.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.config.graph;
-
-import org.graphstream.graph.Graph;
-import org.graphstream.graph.implementations.SingleGraph;
-import org.graphstream.stream.file.FileSource;
-import org.graphstream.stream.file.FileSourceFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.Collections;
-import java.util.Enumeration;
-
-/**
- * Created with IntelliJ IDEA.
- * User: sblackmon
- * Date: 9/23/13
- * Time: 10:44 AM
- * To change this template use File | Settings | File Templates.
- */
-public class PipelineGraphConfigurator {
-
- public static Graph pipeline = loadPipeline();
-
- private static Graph loadPipeline() {
-
- Graph pipeline = new SingleGraph("pipelines");
-
- // this class looks for any pipelines specified with a graph definition
- // each is loaded into the execution graph
- // the application is responsible for launching each
- Enumeration<URL> pipelineFiles;
- try {
- pipelineFiles = PipelineGraphConfigurator.class.getClassLoader().getResources("*.dot");
-
- for( URL pipelineFile : Collections.list(pipelineFiles) ) {
- File file = new File(pipelineFile.toURI());
- String filePath = file.getAbsolutePath();
- FileSource fileSource = FileSourceFactory.sourceFor(filePath);
-
- fileSource.addSink(pipeline);
-
- try {
- fileSource.begin(filePath);
-
- while (fileSource.nextEvents()) {
- // Optionally some code here ...
- }
- } catch( IOException e) {
- e.printStackTrace();
- }
-
- try {
- fileSource.end();
- } catch( IOException e) {
- e.printStackTrace();
- } finally {
- fileSource.removeSink(pipeline);
- }
- }
- } catch (IOException e) {
- e.printStackTrace();
- } catch (URISyntaxException e) {
- e.printStackTrace();
- }
-
- return pipeline;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-contrib/streams-persist-cassandra/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-cassandra/pom.xml b/streams-contrib/streams-persist-cassandra/pom.xml
new file mode 100644
index 0000000..fd6711f
--- /dev/null
+++ b/streams-contrib/streams-persist-cassandra/pom.xml
@@ -0,0 +1,142 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://maven.apache.org/POM/4.0.0"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-contrib</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>streams-persist-cassandra</artifactId>
+
+ <name>${bundle.symbolicName} [${bundle.namespace}]</name>
+
+ <properties>
+ <bundle.symbolicName>streams-persist-cassandra</bundle.symbolicName>
+ <bundle.namespace>org.apache.streams</bundle.namespace>
+ <easymock.version>3.2</easymock.version>
+ </properties>
+
+ <packaging>bundle</packaging>
+ <build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+
+ <resource>
+ <directory>.</directory>
+ <includes>
+ <include>plugin.xml</include>
+ <include>plugin.properties</include>
+ <include>icons/**</include>
+ </includes>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.ops4j</groupId>
+ <artifactId>maven-pax-plugin</artifactId>
+ <!--
+ | enable improved OSGi compilation support for the bundle life-cycle.
+ | to switch back to the standard bundle life-cycle, move this setting
+ | down to the maven-bundle-plugin section
+ -->
+ <extensions>true</extensions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <version>1.4.3</version>
+ <!--
+ | the following instructions build a simple set of public/private classes into an OSGi bundle
+ -->
+ <configuration>
+ <instructions>
+ <Bundle-SymbolicName>${bundle.symbolicName}</Bundle-SymbolicName>
+ <Bundle-Version>${project.version}</Bundle-Version>
+ <Export-Package>
+ ${bundle.namespace};version="${project.version}",org.apache.streams.cassandra.repository.impl, org.apache.streams.cassandra.model, org.apache.streams.cassandra.configuration
+ </Export-Package>
+ <Private-Package>${bundle.namespace}.cassandra.repository.impl.*,${bundle.namespace}.cassandra.model, ${bundle.namespace}.cassandra.configuration </Private-Package>
+ <Import-Package>
+ org.apache.rave.model,org.apache.rave.portal.model.impl,
+ com.datastax.driver.core, com.datastax.driver.core.exceptions, org.codehaus.jackson.map.annotate,
+ javax.persistence, org.apache.commons.logging, com.google.common.collect, org.codehaus.jackson.map,
+ org.apache.commons.lang,
+ org.apache.streams.osgi.components.activitysubscriber,
+ org.springframework.beans.factory.annotation, org.springframework.stereotype
+ </Import-Package>
+ </instructions>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.rave</groupId>
+ <artifactId>rave-core-api</artifactId>
+ <version>${rave.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.rave</groupId>
+ <artifactId>rave-core</artifactId>
+ <version>${rave.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.2</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.datastax.cassandra</groupId>
+ <artifactId>cassandra-driver-core</artifactId>
+ <version>${datastax.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.jboss.netty</groupId>
+ <artifactId>netty</artifactId>
+ <version>3.2.9.Final</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams.osgi.components</groupId>
+ <artifactId>activity-subscriber</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
+ <version>${easymock.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/configuration/CassandraConfiguration.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/configuration/CassandraConfiguration.java b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/configuration/CassandraConfiguration.java
new file mode 100644
index 0000000..39e74a7
--- /dev/null
+++ b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/configuration/CassandraConfiguration.java
@@ -0,0 +1,63 @@
+package org.apache.streams.cassandra.configuration;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class CassandraConfiguration {
+ @Value("${keyspaceName}")
+ private String keyspaceName;
+
+ @Value("${activitystreamsColumnFamilyName}")
+ private String activitystreamsColumnFamilyName;
+
+ @Value("${subscriptionColumnFamilyName}")
+ private String subscriptionColumnFamilyName;
+
+ @Value("${publisherColumnFamilyName}")
+ private String publisherColumnFamilyName;
+
+ @Value("${cassandraPort}")
+ private String cassandraPort;
+
+ public String getKeyspaceName() {
+ return keyspaceName;
+ }
+
+ public void setKeyspaceName(String keyspaceName) {
+ this.keyspaceName = keyspaceName;
+ }
+
+ public String getActivitystreamsColumnFamilyName() {
+ return activitystreamsColumnFamilyName;
+ }
+
+ public void setActivitystreamsColumnFamilyName(String activitystreamsColumnFamilyName) {
+ this.activitystreamsColumnFamilyName = activitystreamsColumnFamilyName;
+ }
+
+ public String getSubscriptionColumnFamilyName() {
+ return subscriptionColumnFamilyName;
+ }
+
+ public void setSubscriptionColumnFamilyName(String subscriptionColumnFamilyName) {
+ this.subscriptionColumnFamilyName = subscriptionColumnFamilyName;
+ }
+
+ public String getPublisherColumnFamilyName() {
+ return publisherColumnFamilyName;
+ }
+
+ public void setPublisherColumnFamilyName(String publisherColumnFamilyName) {
+ this.publisherColumnFamilyName = publisherColumnFamilyName;
+ }
+
+ public String getCassandraPort() {
+ return cassandraPort;
+ }
+
+ public void setCassandraPort(String cassandraPort) {
+ this.cassandraPort = cassandraPort;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/model/CassandraActivityStreamsEntry.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/model/CassandraActivityStreamsEntry.java b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/model/CassandraActivityStreamsEntry.java
new file mode 100644
index 0000000..c97fb82
--- /dev/null
+++ b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/model/CassandraActivityStreamsEntry.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.cassandra.model;
+
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.ActivityObject;
+import org.apache.streams.pojo.json.Actor;
+import org.codehaus.jackson.map.annotate.JsonDeserialize;
+
+public class CassandraActivityStreamsEntry extends Activity implements Comparable<CassandraActivityStreamsEntry>{
+
+ @JsonDeserialize(as=ActivityObject.class)
+ private ActivityObject object;
+
+ @JsonDeserialize(as=ActivityObject.class)
+ private ActivityObject target;
+
+ @JsonDeserialize(as=Actor.class)
+ private Actor actor;
+
+ @JsonDeserialize(as=String.class)
+ private String provider;
+
+ public int compareTo(CassandraActivityStreamsEntry entry){
+ return (this.getPublished()).compareTo(entry.getPublished());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepository.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepository.java b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepository.java
new file mode 100644
index 0000000..56e5416
--- /dev/null
+++ b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepository.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.cassandra.repository.impl;
+
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.exceptions.AlreadyExistsException;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rave.model.ActivityStreamsEntry;
+import org.apache.rave.model.ActivityStreamsObject;
+import org.apache.rave.portal.model.impl.ActivityStreamsEntryImpl;
+import org.apache.rave.portal.model.impl.ActivityStreamsObjectImpl;
+import org.apache.streams.cassandra.configuration.CassandraConfiguration;
+import org.apache.streams.cassandra.model.CassandraActivityStreamsEntry;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+
+public class CassandraActivityStreamsRepository {
+
+ private static final Log LOG = LogFactory.getLog(CassandraActivityStreamsRepository.class);
+
+ private CassandraKeyspace keyspace;
+ private CassandraConfiguration configuration;
+
+ @Autowired
+ public CassandraActivityStreamsRepository(CassandraKeyspace keyspace, CassandraConfiguration configuration) {
+ this.configuration = configuration;
+ this.keyspace = keyspace;
+
+ try {
+ keyspace.getSession().execute("CREATE TABLE " + configuration.getActivitystreamsColumnFamilyName() + " (" +
+ "id text, " +
+ "published timestamp, " +
+ "verb text, " +
+ "tags text, " +
+
+ "actor_displayname text, " +
+ "actor_id text, " +
+ "actor_url text, " +
+ "actor_objecttype text, " +
+
+ "target_displayname text, " +
+ "target_id text, " +
+ "target_url text, " +
+
+ "provider_url text, " +
+
+ "object_url text, " +
+ "object_displayname text, " +
+ "object_id text, " +
+ "object_objecttype text, " +
+
+ "PRIMARY KEY (id, tags, published));");
+ } catch (AlreadyExistsException ignored) {
+ }
+ }
+
+ public void save(ActivityStreamsEntry entry) {
+ String sql = "INSERT INTO " + configuration.getActivitystreamsColumnFamilyName() + " (" +
+ "id, published, verb, tags, " +
+ "actor_displayname, actor_objecttype, actor_id, actor_url, " +
+ "target_displayname, target_id, target_url, " +
+ "provider_url, " +
+ "object_displayname, object_objecttype, object_id, object_url) " +
+ "VALUES ('" +
+ entry.getId() + "','" +
+ entry.getPublished().getTime() + "','" +
+ entry.getVerb() + "','" +
+ entry.getTags() + "','" +
+
+ entry.getActor().getDisplayName() + "','" +
+ entry.getActor().getObjectType() + "','" +
+ entry.getActor().getId() + "','" +
+ entry.getActor().getUrl() + "','" +
+
+ entry.getTarget().getDisplayName() + "','" +
+ entry.getTarget().getId() + "','" +
+ entry.getTarget().getUrl() + "','" +
+
+ entry.getProvider().getUrl() + "','" +
+
+ entry.getObject().getDisplayName() + "','" +
+ entry.getObject().getObjectType() + "','" +
+ entry.getObject().getId() + "','" +
+ entry.getObject().getUrl() +
+
+ "')";
+ keyspace.getSession().execute(sql);
+ }
+
+ public List<CassandraActivityStreamsEntry> getActivitiesForFilters(List<String> filters, Date lastUpdated) {
+ List<CassandraActivityStreamsEntry> results = new ArrayList<CassandraActivityStreamsEntry>();
+
+ for (String tag : filters) {
+ String cql = "SELECT * FROM " + configuration.getActivitystreamsColumnFamilyName() + " WHERE ";
+
+ //add filters
+ cql = cql + " tags = '" + tag + "' AND ";
+
+ //specify last modified
+ cql = cql + "published > " + lastUpdated.getTime() + " ALLOW FILTERING";
+
+ //execute the cql query and store the results
+ ResultSet set = keyspace.getSession().execute(cql);
+
+ //iterate through the results and create a new ActivityStreamsEntry for every result returned
+
+ for (Row row : set) {
+ CassandraActivityStreamsEntry entry = new CassandraActivityStreamsEntry();
+ ActivityStreamsObject actor = new ActivityStreamsObjectImpl();
+ ActivityStreamsObject target = new ActivityStreamsObjectImpl();
+ ActivityStreamsObject object = new ActivityStreamsObjectImpl();
+ ActivityStreamsObject provider = new ActivityStreamsObjectImpl();
+
+ actor.setDisplayName(row.getString("actor_displayname"));
+ actor.setId(row.getString("actor_id"));
+ actor.setObjectType(row.getString("actor_objecttype"));
+ actor.setUrl(row.getString("actor_url"));
+
+ target.setDisplayName(row.getString("target_displayname"));
+ target.setId(row.getString("target_id"));
+ target.setUrl(row.getString("target_url"));
+
+ object.setDisplayName(row.getString("object_displayname"));
+ object.setObjectType(row.getString("object_objecttype"));
+ object.setUrl(row.getString("object_url"));
+ object.setId(row.getString("object_id"));
+
+ provider.setUrl(row.getString("provider_url"));
+
+ entry.setPublished(row.getDate("published"));
+ entry.setVerb(row.getString("verb"));
+ entry.setId(row.getString("id"));
+ entry.setTags(row.getString("tags"));
+ entry.setActor(actor);
+ entry.setTarget(target);
+ entry.setObject(object);
+ entry.setProvider(provider);
+
+ results.add(entry);
+ }
+ }
+
+ return results;
+ }
+
+ public void dropTable(String table) {
+ String cql = "DROP TABLE " + table;
+ keyspace.getSession().execute(cql);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraKeyspace.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraKeyspace.java b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraKeyspace.java
new file mode 100644
index 0000000..0551bf2
--- /dev/null
+++ b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraKeyspace.java
@@ -0,0 +1,43 @@
+package org.apache.streams.cassandra.repository.impl;
+
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.exceptions.AlreadyExistsException;
+import org.apache.streams.cassandra.configuration.CassandraConfiguration;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class CassandraKeyspace {
+ private CassandraConfiguration configuration;
+ private Cluster cluster;
+ private Session session;
+
+ @Autowired
+ public CassandraKeyspace(CassandraConfiguration configuration){
+ this.configuration = configuration;
+
+ cluster = Cluster.builder().addContactPoint(configuration.getCassandraPort()).build();
+ session = cluster.connect();
+
+ //TODO: cassandra 2 will have support for CREATE KEYSPACE IF NOT EXISTS
+ try {
+ session.execute("CREATE KEYSPACE " + configuration.getKeyspaceName() + " WITH replication = { 'class': 'SimpleStrategy','replication_factor' : 1 };");
+ } catch (AlreadyExistsException ignored) {
+ }
+
+ //connect to the keyspace
+ session = cluster.connect(configuration.getKeyspaceName());
+ }
+
+ public Session getSession(){
+ return session;
+ }
+
+ @Override
+ protected void finalize() throws Throwable {
+ try {
+ cluster.shutdown();
+ } finally {
+ super.finalize();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraSubscriptionRepository.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraSubscriptionRepository.java b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraSubscriptionRepository.java
new file mode 100644
index 0000000..f5fe471
--- /dev/null
+++ b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraSubscriptionRepository.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.cassandra.repository.impl;
+
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.exceptions.AlreadyExistsException;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.streams.cassandra.configuration.CassandraConfiguration;
+import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscription;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class CassandraSubscriptionRepository {
+ private static final Log LOG = LogFactory.getLog(CassandraSubscriptionRepository.class);
+
+ private CassandraKeyspace keyspace;
+ private CassandraConfiguration configuration;
+
+ @Autowired
+ public CassandraSubscriptionRepository(CassandraKeyspace keyspace, CassandraConfiguration configuration) {
+ this.keyspace = keyspace;
+ this.configuration = configuration;
+
+ try {
+ keyspace.getSession().execute("CREATE TABLE " + configuration.getSubscriptionColumnFamilyName() + " (" +
+ "id text, " +
+ "filters text, " +
+
+ "PRIMARY KEY (id));");
+ } catch (AlreadyExistsException ignored) {
+ }
+ }
+
+ public String getFilters(String id){
+ String cql = "SELECT * FROM " + configuration.getSubscriptionColumnFamilyName() + " WHERE id = '" + id+"';";
+
+ ResultSet set = keyspace.getSession().execute(cql);
+
+ return set.one().getString("filters");
+ }
+
+ public void save(ActivityStreamsSubscription subscription){
+ String cql = "INSERT INTO " + configuration.getSubscriptionColumnFamilyName() + " (" +
+ "id, filters) " +
+ "VALUES ('" +
+ subscription.getAuthToken() + "','" +
+ StringUtils.join(subscription.getFilters(), " ") +
+
+ "')";
+ keyspace.getSession().execute(cql);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-contrib/streams-persist-cassandra/src/main/resources/META_INF/spring/streams-cassandra-context.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-cassandra/src/main/resources/META_INF/spring/streams-cassandra-context.xml b/streams-contrib/streams-persist-cassandra/src/main/resources/META_INF/spring/streams-cassandra-context.xml
new file mode 100644
index 0000000..842c918
--- /dev/null
+++ b/streams-contrib/streams-persist-cassandra/src/main/resources/META_INF/spring/streams-cassandra-context.xml
@@ -0,0 +1,25 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<beans
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://www.springframework.org/schema/beans"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+ http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">
+
+</beans>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-contrib/streams-persist-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepositoryTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepositoryTest.java b/streams-contrib/streams-persist-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepositoryTest.java
new file mode 100644
index 0000000..978af10
--- /dev/null
+++ b/streams-contrib/streams-persist-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepositoryTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.cassandra.repository.impl;
+
+import com.datastax.driver.core.ResultSet;
+import org.apache.rave.model.ActivityStreamsEntry;
+import org.apache.rave.model.ActivityStreamsObject;
+import org.apache.rave.portal.model.impl.ActivityStreamsEntryImpl;
+import org.apache.rave.portal.model.impl.ActivityStreamsObjectImpl;
+import org.apache.streams.cassandra.configuration.CassandraConfiguration;
+import org.apache.streams.cassandra.model.CassandraActivityStreamsEntry;
+import static org.easymock.EasyMock.*;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+
+public class CassandraActivityStreamsRepositoryTest {
+
+ private CassandraActivityStreamsRepository repository;
+
+
+ @Before
+ public void setup() {
+ CassandraKeyspace keyspace = createMock(CassandraKeyspace.class);
+ CassandraConfiguration configuration = createMock(CassandraConfiguration.class);
+ repository = new CassandraActivityStreamsRepository(keyspace, configuration);
+ }
+
+ @Ignore
+ @Test
+ public void saveActivity() {
+ ActivityStreamsEntry entry = new ActivityStreamsEntryImpl();
+ ActivityStreamsObject actor = new ActivityStreamsObjectImpl();
+ ActivityStreamsObject target = new ActivityStreamsObjectImpl();
+ ActivityStreamsObject object = new ActivityStreamsObjectImpl();
+ ActivityStreamsObject provider = new ActivityStreamsObjectImpl();
+
+ actor.setId("actorid1");
+ actor.setUrl("actorurl1");
+ actor.setDisplayName("actorname1");
+
+ target.setId("targetid1");
+ target.setUrl("targeturl1");
+ target.setDisplayName("r501");
+
+ provider.setUrl("providerurl");
+
+ object.setId("objectid1");
+ object.setDisplayName("objectname1");
+
+ entry.setId("dink");
+ entry.setVerb("verb1");
+ entry.setTags("r501");
+ entry.setProvider(provider);
+ Date d = new Date();
+ entry.setPublished(d);
+ entry.setActor(actor);
+ entry.setObject(object);
+ entry.setTarget(target);
+
+ repository.save(entry);
+ }
+
+ @Ignore
+ @Test
+ public void getActivity() {
+ String cql = "tags";
+ String other = "r501";
+ List<String> f = Arrays.asList(cql, other);
+ Date d = new Date(0);
+ List<CassandraActivityStreamsEntry> results = repository.getActivitiesForFilters(f,d);
+ }
+
+ @Ignore
+ @Test
+ public void dropTableTest(){
+ repository.dropTable("coltest");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-contrib/streams-persist-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivitySubscriptionTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivitySubscriptionTest.java b/streams-contrib/streams-persist-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivitySubscriptionTest.java
new file mode 100644
index 0000000..2a90462
--- /dev/null
+++ b/streams-contrib/streams-persist-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivitySubscriptionTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.cassandra.repository.impl;
+
+import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscription;
+import org.apache.streams.osgi.components.activitysubscriber.impl.ActivityStreamsSubscriptionImpl;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+public class CassandraActivitySubscriptionTest {
+
+ public CassandraSubscriptionRepository repository;
+
+
+ @Before
+ public void setup() {
+// repository = new CassandraSubscriptionRepository();
+ }
+
+ @Ignore
+ @Test
+ public void saveTest(){
+ ActivityStreamsSubscription subscription = new ActivityStreamsSubscriptionImpl();
+ subscription.setFilters(Arrays.asList("thisis", "atest"));
+ subscription.setAuthToken("subid");
+
+ repository.save(subscription);
+ }
+
+ @Ignore
+ @Test
+ public void getTest(){
+ String filters = repository.getFilters("subid");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/ReadMe.txt
----------------------------------------------------------------------
diff --git a/streams-eip-routes/ReadMe.txt b/streams-eip-routes/ReadMe.txt
deleted file mode 100644
index 19a1d19..0000000
--- a/streams-eip-routes/ReadMe.txt
+++ /dev/null
@@ -1,32 +0,0 @@
-Camel Router WAR Project with Web Console and REST Support
-==========================================================
-
-This project bundles the Camel Web Console, REST API, and some
-sample routes as a WAR. You can build the WAR by running
-
-mvn install
-
-You can then run the project by dropping the WAR into your
-favorite web container or just run
-
-mvn jetty:run
-
-to start up and deploy to Jetty.
-
-
-Web Console
-===========
-
-You can view the Web Console by pointing your browser to http://localhost:8080/
-
-You should be able to do things like
-
- * browse the available endpoints
- * browse the messages on an endpoint if it is a BrowsableEndpoint
- * send a message to an endpoint
- * create new endpoints
-
-For more help see the Apache Camel documentation
-
- http://camel.apache.org/
-
[12/13] dropping streams-pojo-extensions,
BC activity can be extended without it move streams cassandra under
contrib move streams-eip-routes under runtimes,
renaming streams-runtime-webapp
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/pom.xml
----------------------------------------------------------------------
diff --git a/streams-eip-routes/pom.xml b/streams-eip-routes/pom.xml
deleted file mode 100644
index 78d0767..0000000
--- a/streams-eip-routes/pom.xml
+++ /dev/null
@@ -1,209 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns="http://maven.apache.org/POM/4.0.0"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-project</artifactId>
- <version>0.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-eip-routes</artifactId>
-
- <name>${bundle.symbolicName} [${bundle.namespace}]</name>
-
- <properties>
- <bundle.symbolicName>streams-eip-routes</bundle.symbolicName>
- <bundle.namespace>org.apache.streams</bundle.namespace>
- <jackson.old.version>1.9.11</jackson.old.version>
- </properties>
-
- <packaging>bundle</packaging>
-
- <repositories>
- <repository>
- <id>clojars.org</id>
- <url>http://clojars.org/repo</url>
- </repository>
- </repositories>
-
- <build>
-
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- <!--
- | example additional resource entries, useful when building Eclipse RCP applications
- -->
- <resource>
- <directory>.</directory>
- <includes>
- <include>plugin.xml</include>
- <include>plugin.properties</include>
- <include>icons/**</include>
- <include>META-INF/*</include>
- </includes>
- </resource>
- </resources>
- <plugins>
- <plugin>
- <groupId>org.ops4j</groupId>
- <artifactId>maven-pax-plugin</artifactId>
- <!--
- | enable improved OSGi compilation support for the bundle life-cycle.
- | to switch back to the standard bundle life-cycle, move this setting
- | down to the maven-bundle-plugin section
- -->
- <extensions>true</extensions>
- </plugin>
- <plugin>
- <groupId>org.apache.felix</groupId>
- <artifactId>maven-bundle-plugin</artifactId>
- <version>1.4.3</version>
- <!--
- | the following instructions build a simple set of public/private classes into an OSGi bundle
- -->
- <configuration>
- <instructions>
- <Bundle-SymbolicName>${bundle.symbolicName}</Bundle-SymbolicName>
- <Bundle-Version>${project.version}</Bundle-Version>
- <Export-Package>${bundle.namespace};version="${project.version}",org.apache.streams.messaging.configuration,org.apache.streams.messaging.routers,org.apache.streams.messaging.rules,org.apache.streams.messaging.processors,org.apache.streams.messaging.aggregation,org.apache.streams.messaging.service, org.apache.streams.messaging.storm,org.apache.activemq,org.codehaus.jackson.*;version="${jackson.version}"</Export-Package>
- <Private-Package>${bundle.namespace}.messaging.routers.impl.*,${bundle.namespace}.messaging.rules.impl.*, ${bundle.namespace}.messaging.service.impl.*</Private-Package>
- <Import-Package>org.apache.camel.*;version="2.8.5",org.apache.streams.messaging.configuration,org.apache.activemq.camel.component,org.apache.activemq,org.apache.activemq.pool,org.apache.camel.component.jms,org.springframework.*;version="3.0.6.RELEASE",org.apache.commons.logging,org.apache.streams.*,org.apache.streams.osgi.components,org.apache.streams.osgi.components.activityconsumer,org.apache.streams.osgi.components.activityconsumer.impl,org.apache.streams.osgi.components.activitysubscriber,org.apache.streams.osgi.components.activitysubscriber.impl,org.apache.streams.messaging.processors,org.apache.streams.messaging.aggregation,javax.jms, javax.net.ssl, javax.transaction.xa, org.apache.activemq.advisory, org.apache.activemq.blob, org.apache.activemq.broker, org.apache.activemq.broker.region, org.apache.activemq.command, org.apache.activemq.filter, org.apache.activemq.jndi, org.apache.activemq.management, org.apache.activemq.selector, org.apache.activemq.sta
te, org.apache.activemq.thread, org.apache.activemq.transaction, org.apache.activemq.transport, org.apache.activemq.transport.failover, org.apache.activemq.transport.tcp, org.apache.activemq.usage, org.apache.activemq.util, org.slf4j,org.codehaus.jackson;version="${jackson.version}",javax.xml.datatype, javax.xml.namespace, javax.xml.parsers, org.joda.time, org.joda.time.format, org.w3c.dom, org.w3c.dom.bootstrap, org.w3c.dom.ls, org.xml.sax, org.apache.rave.model, org.apache.rave.portal.model.impl, backtype.storm, backtype.storm.coordination, backtype.storm.generated, backtype.storm.spout, backtype.storm.task, backtype.storm.topology, backtype.storm.topology.base, backtype.storm.tuple, javax.annotation, backtype.storm.utils</Import-Package>
- </instructions>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.camel</groupId>
- <artifactId>camel-core</artifactId>
- <version>2.9.0</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.activemq</groupId>
- <artifactId>activemq-camel</artifactId>
- <version>5.5.1</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.activemq</groupId>
- <artifactId>activemq-pool</artifactId>
- <version>5.5.1</version>
- </dependency>
-
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-core</artifactId>
- <version>3.0.6.RELEASE</version>
- </dependency>
-
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-context</artifactId>
- <version>3.0.6.RELEASE</version>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mrbean</artifactId>
- <version>${jackson.old.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- <version>${jackson.old.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.osgi</groupId>
- <artifactId>osgi_R4_core</artifactId>
- <version>1.0</version>
- <scope>provided</scope>
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>org.osgi</groupId>
- <artifactId>osgi_R4_compendium</artifactId>
- <version>1.0</version>
- <scope>provided</scope>
- <optional>true</optional>
- </dependency>
-
-
- <dependency>
- <groupId>org.apache.streams.osgi.components</groupId>
- <artifactId>activity-registration</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams.osgi.components</groupId>
- <artifactId>activity-consumer</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams.osgi.components</groupId>
- <artifactId>activity-subscriber</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-cassandra</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.rave</groupId>
- <artifactId>rave-core</artifactId>
- <version>${rave.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.rave</groupId>
- <artifactId>rave-core-api</artifactId>
- <version>${rave.version}</version>
- </dependency>
-
- <dependency>
- <groupId>storm</groupId>
- <artifactId>storm</artifactId>
- <version>0.8.2</version>
- </dependency>
-
- <dependency>
- <groupId>org.easymock</groupId>
- <artifactId>easymock</artifactId>
- <version>3.1</version>
- </dependency>
- </dependencies>
-
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/src/main/java/org/apache/streams/messaging/aggregation/ActivityAggregator.java
----------------------------------------------------------------------
diff --git a/streams-eip-routes/src/main/java/org/apache/streams/messaging/aggregation/ActivityAggregator.java b/streams-eip-routes/src/main/java/org/apache/streams/messaging/aggregation/ActivityAggregator.java
deleted file mode 100644
index dc7ba0c..0000000
--- a/streams-eip-routes/src/main/java/org/apache/streams/messaging/aggregation/ActivityAggregator.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.messaging.aggregation;
-
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.streams.messaging.service.impl.CassandraActivityService;
-import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscriber;
-import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscriberWarehouse;
-import org.springframework.scheduling.annotation.Scheduled;
-
-import java.util.*;
-
-public class ActivityAggregator {
-
- private ActivityStreamsSubscriberWarehouse activityStreamsSubscriberWarehouse;
- private CassandraActivityService activityService;
- private static final transient Log LOG = LogFactory.getLog(ActivityAggregator.class);
-
- public void setActivityStreamsSubscriberWarehouse(ActivityStreamsSubscriberWarehouse activityStreamsSubscriberWarehouse) {
- this.activityStreamsSubscriberWarehouse = activityStreamsSubscriberWarehouse;
- }
-
- public void setActivityService(CassandraActivityService activityService) {
- this.activityService = activityService;
- }
-
- @Scheduled(fixedRate=30000)
- public void distributeToSubscribers() {
- for (ActivityStreamsSubscriber subscriber : activityStreamsSubscriberWarehouse.getAllSubscribers()) {
- updateSubscriber(subscriber);
- }
- }
-
- public void updateSubscriber(ActivityStreamsSubscriber subscriber){
- Set<String> activities = new TreeSet<String>();
- activities.addAll(activityService.getActivitiesForFilters(subscriber.getActivityStreamsSubscriberConfiguration().getFilters(), subscriber.getLastUpdated()));
- //TODO: an activity posted in between the cql query and setting the lastUpdated field will be lost
- subscriber.setLastUpdated(new Date());
- subscriber.receive(new ArrayList<String>(activities));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/src/main/java/org/apache/streams/messaging/configuration/EipConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-eip-routes/src/main/java/org/apache/streams/messaging/configuration/EipConfigurator.java b/streams-eip-routes/src/main/java/org/apache/streams/messaging/configuration/EipConfigurator.java
deleted file mode 100644
index 460c43a..0000000
--- a/streams-eip-routes/src/main/java/org/apache/streams/messaging/configuration/EipConfigurator.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.messaging.configuration;
-
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-
-@Component
-public class EipConfigurator {
-
-
-
- @Value("${consumer.inRouteHost}")
- private String consumerInRouteHost;
-
- @Value("${consumer.inRoutePort}")
- private String consumerInRoutePort;
-
- @Value("${subscriber.inRouteHost}")
- private String subscriberInRouteHost;
-
- @Value("${subscriber.inRoutePort}")
- private String subscriberInRoutePort;
-
-
- @Value("${consumer.activityQUri}")
- private String consumerActivityQUri;
-
- @Value("${consumer.publisherEndpointProtocol}")
- private String publisherEndpointProtocol;
-
- @Value("${consumer.publisherEndpointUrlResource}")
- private String publisherEndpointUrlResource;
-
- @Value("${consumer.receiveMethod}")
- private String consumerReceiveMethod;
-
- @Value("${consumer.splitMethod}")
- private String consumerSplitMethod;
-
- @Value("${subscriber.subscriberEndpointProtocol}")
- private String subscriberEndpointProtocol;
-
- @Value("${subscriber.subscriberEndpointUrlResource}")
- private String subscriberEndpointUrlResource;
-
- @Value("${subscriber.receiveMethod}")
- private String subscriberReceiveMethod;
-
- @Value("${subscriber.postMethod}")
- private String subscriberPostMethod;
-
- @Value("${subscriber.getMethod}")
- private String subscriberGetMethod;
-
-
- @Value("${servlet.baseUrlPath}")
- private String baseUrlPath;
-
-
- public static String ENDPOINT_PROTOCOL_JETTY="jetty:http://";
- public static String ENDPOINT_PROTOCOL_SERVLET="servlet:///";
-
- public String getConsumerInRouteHost() {
- return consumerInRouteHost;
- }
-
- public String getConsumerInRoutePort() {
- return consumerInRoutePort;
- }
-
- public String getConsumerActivityQUri() {
- return consumerActivityQUri;
- }
-
- public void setConsumerActivityQUri(String consumerActivityQUri) {
- this.consumerActivityQUri = consumerActivityQUri;
- }
-
- public void setConsumerInRoutePort(String consumerInRoutePort) {
- this.consumerInRoutePort = consumerInRoutePort;
- }
-
- public void setConsumerInRouteHost(String consumerInRouteHost) {
- this.consumerInRouteHost = consumerInRouteHost;
- }
-
- public String getSubscriberInRouteHost() {
- return subscriberInRouteHost;
- }
-
- public void setSubscriberInRouteHost(String subscriberInRouteHost) {
- this.subscriberInRouteHost = subscriberInRouteHost;
- }
-
- public String getSubscriberInRoutePort() {
- return subscriberInRoutePort;
- }
-
- public void setSubscriberInRoutePort(String subscriberInRoutePort) {
- this.subscriberInRoutePort = subscriberInRoutePort;
- }
-
- public String getPublisherEndpointProtocol() {
- return publisherEndpointProtocol;
- }
-
- public void setPublisherEndpointProtocol(String publisherEndpointProtocol) {
- this.publisherEndpointProtocol = publisherEndpointProtocol;
- }
-
- public String getPublisherEndpointUrlResource() {
- return publisherEndpointUrlResource;
- }
-
- public void setPublisherEndpointUrlResource(String publisherEndpointUrlResource) {
- this.publisherEndpointUrlResource = publisherEndpointUrlResource;
- }
-
- public String getConsumerReceiveMethod() {
- return consumerReceiveMethod;
- }
-
- public void setConsumerReceiveMethod(String consumerReceiveMethod) {
- this.consumerReceiveMethod = consumerReceiveMethod;
- }
-
- public String getConsumerSplitMethod() {
- return consumerSplitMethod;
- }
-
- public void setConsumerSplitMethod(String consumerSplitMethod) {
- this.consumerSplitMethod = consumerSplitMethod;
- }
-
- public String getSubscriberEndpointProtocol() {
- return subscriberEndpointProtocol;
- }
-
- public void setSubscriberEndpointProtocol(String subscriberEndpointProtocol) {
- this.subscriberEndpointProtocol = subscriberEndpointProtocol;
- }
-
- public String getSubscriberEndpointUrlResource() {
- return subscriberEndpointUrlResource;
- }
-
- public void setSubscriberEndpointUrlResource(String subscriberEndpointUrlResource) {
- this.subscriberEndpointUrlResource = subscriberEndpointUrlResource;
- }
-
- public String getSubscriberReceiveMethod() {
- return subscriberReceiveMethod;
- }
-
- public void setSubscriberReceiveMethod(String subscriberReceiveMethod) {
- this.subscriberReceiveMethod = subscriberReceiveMethod;
- }
-
- public String getSubscriberPostMethod() {
- return subscriberPostMethod;
- }
-
- public void setSubscriberPostMethod(String subscriberPostMethod) {
- this.subscriberPostMethod = subscriberPostMethod;
- }
-
- public String getSubscriberGetMethod() {
- return subscriberGetMethod;
- }
-
- public void setSubscriberGetMethod(String subscriberGetMethod) {
- this.subscriberGetMethod = subscriberGetMethod;
- }
-
- public String getBaseUrlPath() {
- return baseUrlPath;
- }
-
- public void setBaseUrlPath(String baseUrlPath) {
- this.baseUrlPath = baseUrlPath;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/src/main/java/org/apache/streams/messaging/processors/ActivityPublisherRegistrationProcessor.java
----------------------------------------------------------------------
diff --git a/streams-eip-routes/src/main/java/org/apache/streams/messaging/processors/ActivityPublisherRegistrationProcessor.java b/streams-eip-routes/src/main/java/org/apache/streams/messaging/processors/ActivityPublisherRegistrationProcessor.java
deleted file mode 100644
index 0c60349..0000000
--- a/streams-eip-routes/src/main/java/org/apache/streams/messaging/processors/ActivityPublisherRegistrationProcessor.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.messaging.processors;
-
-import org.apache.camel.Exchange;
-import org.apache.camel.Processor;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.streams.osgi.components.activityconsumer.ActivityConsumer;
-import org.apache.streams.osgi.components.activityconsumer.impl.PushActivityConsumer;
-import org.codehaus.jackson.map.DeserializationConfig;
-import org.codehaus.jackson.map.ObjectMapper;
-
-
-public class ActivityPublisherRegistrationProcessor implements Processor{
- private static final transient Log LOG = LogFactory.getLog(ActivityStreamsSubscriberRegistrationProcessor.class);
- public void process(Exchange exchange){
- //add the necessary headers to the message so that the activity registration component
- //can do a lookup to either make a new processor and endpoint, or pass the message to the right one
- String httpMethod = exchange.getIn().getHeader("CamelHttpMethod").toString();
-
- if (!httpMethod.equals("POST")){
- //reject anything that isn't a post...Camel 2.10 solves needing this check, however, SM 4.4 doesn't have the latest
- exchange.getOut().setFault(true);
- exchange.getOut().setHeader(Exchange.HTTP_RESPONSE_CODE,405);
- } else {
-
- //for now...just expect a post with a uri in the body...should have some checking here with http response codes
- // authentication, all that good stuff...happens in the registration module
-
- String body = exchange.getIn().getBody(String.class);
- ObjectMapper mapper = new ObjectMapper();
- mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES,false);
-
- try {
-
- // read from file, convert it to user class
- ActivityConsumer configuration = mapper.readValue(body, ActivityConsumer.class);
- if (configuration.getSrc()==null){
- LOG.info("configuration src is null");
- throw new Exception();
- }
-
- exchange.getOut().setBody(configuration);
-
- } catch (Exception e) {
- LOG.info("error: " + e);
- exchange.getOut().setFault(true);
- exchange.getOut().setHeader(Exchange.HTTP_RESPONSE_CODE,400);
- exchange.getOut().setBody("POST should contain a valid JSON configuration for registering as an Activity Publisher (check that src element is valid).");
- }
- }
-
-
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/src/main/java/org/apache/streams/messaging/processors/ActivityStreamsSubscriberRegistrationProcessor.java
----------------------------------------------------------------------
diff --git a/streams-eip-routes/src/main/java/org/apache/streams/messaging/processors/ActivityStreamsSubscriberRegistrationProcessor.java b/streams-eip-routes/src/main/java/org/apache/streams/messaging/processors/ActivityStreamsSubscriberRegistrationProcessor.java
deleted file mode 100644
index 9306aa3..0000000
--- a/streams-eip-routes/src/main/java/org/apache/streams/messaging/processors/ActivityStreamsSubscriberRegistrationProcessor.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.messaging.processors;
-
-import org.apache.camel.Exchange;
-import org.apache.camel.Processor;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.streams.messaging.service.SubscriptionService;
-import org.apache.streams.messaging.service.impl.CassandraSubscriptionService;
-import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscription;
-import org.codehaus.jackson.map.DeserializationConfig;
-import org.codehaus.jackson.map.ObjectMapper;
-
-
-public class ActivityStreamsSubscriberRegistrationProcessor implements Processor{
- private static final transient Log LOG = LogFactory.getLog(ActivityStreamsSubscriberRegistrationProcessor.class);
- private SubscriptionService subscriptionService;
-
- public ActivityStreamsSubscriberRegistrationProcessor(SubscriptionService subscriptionService){
- this.subscriptionService = subscriptionService;
- }
-
- public void process(Exchange exchange){
- LOG.info("processing the subscriber...");
- //add the necessary headers to the message so that the activity registration component
- //can do a lookup to either make a new processor and endpoint, or pass the message to the right one
- String httpMethod = exchange.getIn().getHeader("CamelHttpMethod").toString();
-
- if (!httpMethod.equals("POST")){
- //reject anything that isn't a post...Camel 2.10 solves needing this check, however, SM 4.4 doesn't have the latest
- exchange.getOut().setFault(true);
- exchange.getOut().setHeader(Exchange.HTTP_RESPONSE_CODE,405);
- } else {
-
- //for now...just expect a post with a uri in the body...should have some checking here with http response codes
- // authentication, all that good stuff...happens in the registration module
-
-
- String body = exchange.getIn().getBody(String.class);
-
- LOG.info("receiving the subscriber: "+body);
- //OAuth token? What does subscriber post to init a subscription URL?
- //maybe its a list of URLs to subscribe to subscriptions=1,2,3,4&auth_token=XXXX
-
- ObjectMapper mapper = new ObjectMapper();
- mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES,false);
-
- try {
-
- // read from file, convert it to user class
- ActivityStreamsSubscription configuration = mapper.readValue(body, ActivityStreamsSubscription.class);
- if(configuration.getFilters() == null){
- configuration.setFilters(subscriptionService.getFilters(configuration.getAuthToken()));
- }else{
- subscriptionService.saveFilters(configuration);
- }
- exchange.getOut().setBody(configuration);
-
- } catch (Exception e) {
- LOG.info("exception" + e);
- exchange.getOut().setFault(true);
- exchange.getOut().setHeader(Exchange.HTTP_RESPONSE_CODE,400);
- exchange.getOut().setBody("POST should contain a valid Subscription configuration object.");
- }
-
-
-
- //just pass this on to the route creator, body will be the dedicated URL for this subscriber
-
- }
-
-
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/src/main/java/org/apache/streams/messaging/routers/ActivityConsumerRouteBuilder.java
----------------------------------------------------------------------
diff --git a/streams-eip-routes/src/main/java/org/apache/streams/messaging/routers/ActivityConsumerRouteBuilder.java b/streams-eip-routes/src/main/java/org/apache/streams/messaging/routers/ActivityConsumerRouteBuilder.java
deleted file mode 100644
index dea8781..0000000
--- a/streams-eip-routes/src/main/java/org/apache/streams/messaging/routers/ActivityConsumerRouteBuilder.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.messaging.routers;
-
-
-
-import org.apache.camel.Exchange;
-import org.apache.streams.osgi.components.activityconsumer.ActivityConsumer;
-
-
-public interface ActivityConsumerRouteBuilder {
-
-
- void createNewRouteForConsumer(Exchange exchange, ActivityConsumer activityConsumer);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/src/main/java/org/apache/streams/messaging/routers/ActivityStreamsSubscriberRouteBuilder.java
----------------------------------------------------------------------
diff --git a/streams-eip-routes/src/main/java/org/apache/streams/messaging/routers/ActivityStreamsSubscriberRouteBuilder.java b/streams-eip-routes/src/main/java/org/apache/streams/messaging/routers/ActivityStreamsSubscriberRouteBuilder.java
deleted file mode 100644
index 6947722..0000000
--- a/streams-eip-routes/src/main/java/org/apache/streams/messaging/routers/ActivityStreamsSubscriberRouteBuilder.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.messaging.routers;
-
-
-
-import org.apache.camel.Exchange;
-import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscriber;
-
-
-public interface ActivityStreamsSubscriberRouteBuilder {
-
-
- void createNewRouteForSubscriber(Exchange exchange, ActivityStreamsSubscriber activityStreamsSubscriber);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/src/main/java/org/apache/streams/messaging/routers/impl/ActivityConsumerRouter.java
----------------------------------------------------------------------
diff --git a/streams-eip-routes/src/main/java/org/apache/streams/messaging/routers/impl/ActivityConsumerRouter.java b/streams-eip-routes/src/main/java/org/apache/streams/messaging/routers/impl/ActivityConsumerRouter.java
deleted file mode 100644
index 20b8246..0000000
--- a/streams-eip-routes/src/main/java/org/apache/streams/messaging/routers/impl/ActivityConsumerRouter.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.messaging.routers.impl;
-
-
-import org.apache.camel.builder.RouteBuilder;
-import org.apache.streams.messaging.routers.ActivityConsumerRouteBuilder;
-
-
-import org.apache.streams.osgi.components.activityconsumer.ActivityConsumerWarehouse;
-import org.apache.streams.osgi.components.activityconsumer.ActivityConsumer;
-import org.apache.streams.messaging.configuration.EipConfigurator;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.apache.camel.Exchange;
-import org.apache.camel.CamelContext;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import java.util.UUID;
-
-
-public class ActivityConsumerRouter extends RouteBuilder implements ActivityConsumerRouteBuilder {
- private static final transient Log LOG = LogFactory.getLog(ActivityConsumerRouter.class);
-
- @Autowired
- private EipConfigurator configuration;
-
- protected CamelContext camelContext;
-
- private ActivityConsumerWarehouse activityConsumerWarehouse;
-
- public void setCamelContext(CamelContext camelContext) {
- this.camelContext = camelContext;
- }
-
- public void setActivityConsumerWarehouse(ActivityConsumerWarehouse activityConsumerWarehouse) {
- this.activityConsumerWarehouse = activityConsumerWarehouse;
- }
-
-
- public void createNewRouteForConsumer(Exchange exchange, ActivityConsumer activityConsumer){
-
- //todo: add some better scheme then getCount for URL...
- //todo: make the route again if consumer exists...and context doesn't have route
- if (activityConsumer.isAuthenticated()){
- ActivityConsumer existingConsumer = activityConsumerWarehouse.findConsumerBySrc(activityConsumer.getSrc().toASCIIString());
-
- if (existingConsumer==null){
-
- try{
-
- if (configuration.getPublisherEndpointProtocol().equals(EipConfigurator.ENDPOINT_PROTOCOL_JETTY)){
- activityConsumer.setInRoute(configuration.getConsumerInRouteHost()+ ":" + configuration.getConsumerInRoutePort() +"/" + configuration.getPublisherEndpointUrlResource() + "/" + UUID.randomUUID());
- //set the body to the url the producer should post to
- exchange.getOut().setBody("http://" + activityConsumer.getInRoute());
- }else if (configuration.getPublisherEndpointProtocol().equals(EipConfigurator.ENDPOINT_PROTOCOL_SERVLET)){
- activityConsumer.setInRoute( configuration.getPublisherEndpointUrlResource() + "/" + UUID.randomUUID());
- //set the body to the url the producer should post to
- exchange.getOut().setBody(configuration.getBaseUrlPath() + activityConsumer.getInRoute());
- } else{
- throw new Exception("No supported endpoint protocol is configured.");
- }
-
-
- //setup a message queue for this consumer.getInRoute()
- camelContext.addRoutes(new DynamicConsumerRouteBuilder(configuration,camelContext, configuration.getPublisherEndpointProtocol() + activityConsumer.getInRoute(), activityConsumer));
-
-
- LOG.info("all messages sent from " + activityConsumer.getSrc() + " must be posted to " + activityConsumer.getInRoute());
- //only add the route to the warehouse after its been created in messaging system...
- activityConsumerWarehouse.register(activityConsumer);
- }catch (Exception e){
- exchange.getOut().setHeader(Exchange.HTTP_RESPONSE_CODE,500);
- exchange.getOut().setBody("error creating route: " + e);
- LOG.error("error creating route: " + e);
- }
-
- } else{
-
- exchange.getOut().setBody(configuration.getBaseUrlPath() + existingConsumer.getInRoute());
- }
-
- }else{
- exchange.getOut().setFault(true);
- exchange.getOut().setHeader(Exchange.HTTP_RESPONSE_CODE,401);
- exchange.getOut().setBody("Authentication failed.");
- }
-
- }
-
-
- public void configure() throws java.lang.Exception{
- //nothing...set the context?
-
- }
-
- /**
- * This route builder is a skeleton to add new routes at runtime
- */
- private static final class DynamicConsumerRouteBuilder extends RouteBuilder {
- private final String from;
- private ActivityConsumer activityConsumer;
-
-
- private EipConfigurator configuration;
-
- private DynamicConsumerRouteBuilder(EipConfigurator configuration, CamelContext context, String from, ActivityConsumer activityConsumer) {
- super(context);
- this.from = from;
- this.activityConsumer = activityConsumer;
- this.configuration = configuration;
- }
-
- @Override
- public void configure() throws Exception {
-
-
- from(from)
- .bean(activityConsumer, configuration.getConsumerReceiveMethod()).setBody(body())
- .split()
- .method(activityConsumer, configuration.getConsumerSplitMethod())
- .to(configuration.getConsumerActivityQUri());
-
-
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/src/main/java/org/apache/streams/messaging/routers/impl/ActivityStreamsSubscriberRouter.java
----------------------------------------------------------------------
diff --git a/streams-eip-routes/src/main/java/org/apache/streams/messaging/routers/impl/ActivityStreamsSubscriberRouter.java b/streams-eip-routes/src/main/java/org/apache/streams/messaging/routers/impl/ActivityStreamsSubscriberRouter.java
deleted file mode 100644
index ad4b779..0000000
--- a/streams-eip-routes/src/main/java/org/apache/streams/messaging/routers/impl/ActivityStreamsSubscriberRouter.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.messaging.routers.impl;
-
-
-import org.apache.camel.CamelContext;
-import org.apache.camel.Exchange;
-import org.apache.camel.builder.RouteBuilder;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.streams.messaging.aggregation.ActivityAggregator;
-import org.apache.streams.messaging.configuration.EipConfigurator;
-import org.apache.streams.messaging.routers.ActivityStreamsSubscriberRouteBuilder;
-import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscriber;
-import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscriberWarehouse;
-import org.springframework.beans.factory.annotation.Autowired;
-
-import java.util.HashMap;
-import java.util.UUID;
-
-
-public class ActivityStreamsSubscriberRouter extends RouteBuilder implements ActivityStreamsSubscriberRouteBuilder {
- private static final transient Log LOG = LogFactory.getLog(ActivityStreamsSubscriberRouter.class);
-
- @Autowired
- private EipConfigurator configuration;
-
- protected CamelContext camelContext;
-
- @Autowired
- private ActivityAggregator activityAggregator;
-
- private ActivityStreamsSubscriberWarehouse activityStreamsSubscriberWarehouse;
-
- public void setCamelContext(CamelContext camelContext) {
- this.camelContext = camelContext;
- }
-
- public void setActivityStreamsSubscriberWarehouse(ActivityStreamsSubscriberWarehouse activityStreamsSubscriberWarehouse) {
- this.activityStreamsSubscriberWarehouse = activityStreamsSubscriberWarehouse;
- }
-
-
- public void createNewRouteForSubscriber(Exchange exchange, ActivityStreamsSubscriber activityStreamsSubscriber){
-
- //todo: add some better scheme then getCount for URL...
- //todo: make the route again if subscriber exists...and context doesn't have route
- if (activityStreamsSubscriber.isAuthenticated()){
-
- try{
-
- if (configuration.getSubscriberEndpointProtocol().equals(EipConfigurator.ENDPOINT_PROTOCOL_JETTY)){
- activityStreamsSubscriber.setInRoute(configuration.getSubscriberInRouteHost()+ ":" + configuration.getSubscriberInRoutePort() +"/" + configuration.getSubscriberEndpointUrlResource() + "/" + UUID.randomUUID());
- //set the body to the url the producer should post to
- exchange.getOut().setBody("http://" + activityStreamsSubscriber.getInRoute());
- }else if (configuration.getSubscriberEndpointProtocol().equals(EipConfigurator.ENDPOINT_PROTOCOL_SERVLET)){
- activityStreamsSubscriber.setInRoute( configuration.getSubscriberEndpointUrlResource() + "/" + UUID.randomUUID());
- //set the body to the url the producer should post to
- exchange.getOut().setBody(configuration.getBaseUrlPath() + activityStreamsSubscriber.getInRoute());
- } else{
- throw new Exception("No supported endpoint protocol is configured.");
- }
-
- //setup a message queue for this consumer.getInRoute()
- camelContext.addRoutes(new DynamicSubscriberRouteBuilder(configuration,camelContext, configuration.getSubscriberEndpointProtocol() + activityStreamsSubscriber.getInRoute(), activityStreamsSubscriber));
-
- activityAggregator.updateSubscriber(activityStreamsSubscriber);
- activityStreamsSubscriberWarehouse.register(activityStreamsSubscriber);
- }catch (Exception e){
- exchange.getOut().setHeader(Exchange.HTTP_RESPONSE_CODE,500);
- exchange.getOut().setBody("error creating route: " + e);
- LOG.error("error creating route: " + e);
- }
-
- }else{
- exchange.getOut().setFault(true);
- exchange.getOut().setHeader(Exchange.HTTP_RESPONSE_CODE,401);
- exchange.getOut().setBody("Authentication failed.");
- }
-
- }
-
-
-
-
- public void configure() throws Exception{
- //nothing...set the context?
-
- }
-
- /**
- * This route builder is a skeleton to add new routes at runtime
- */
- private static final class DynamicSubscriberRouteBuilder extends RouteBuilder {
- private final String from;
- private ActivityStreamsSubscriber activityStreamsSubscriber;
-
-
- private EipConfigurator configuration;
-
- private DynamicSubscriberRouteBuilder(EipConfigurator configuration, CamelContext context, String from, ActivityStreamsSubscriber activityStreamsSubscriber) {
- super(context);
- this.from = from;
- this.activityStreamsSubscriber = activityStreamsSubscriber;
- this.configuration = configuration;
- }
-
- @Override
- public void configure() throws Exception {
-
-
- from(from)
- .choice()
- .when(header("CamelHttpMethod").isEqualTo("POST"))
- //when its a post...it goes to adding a new src
- .bean(activityStreamsSubscriber, configuration.getSubscriberPostMethod()).setBody(body())
- .when(header("CamelHttpMethod").isEqualTo("GET"))
- // when its a GET it goes to getStream()
- .bean(activityStreamsSubscriber, configuration.getSubscriberGetMethod()) ;
-
-
-
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/src/main/java/org/apache/streams/messaging/service/ActivityService.java
----------------------------------------------------------------------
diff --git a/streams-eip-routes/src/main/java/org/apache/streams/messaging/service/ActivityService.java b/streams-eip-routes/src/main/java/org/apache/streams/messaging/service/ActivityService.java
deleted file mode 100644
index 0c85974..0000000
--- a/streams-eip-routes/src/main/java/org/apache/streams/messaging/service/ActivityService.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.messaging.service;
-
-import org.apache.camel.Exchange;
-
-import java.util.Date;
-import java.util.List;
-
-public interface ActivityService {
-
- void receiveExchange(Exchange exchange);
-
- List<String> getActivitiesForFilters(List<String> filters, Date lastUpdated);
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/src/main/java/org/apache/streams/messaging/service/SubscriptionService.java
----------------------------------------------------------------------
diff --git a/streams-eip-routes/src/main/java/org/apache/streams/messaging/service/SubscriptionService.java b/streams-eip-routes/src/main/java/org/apache/streams/messaging/service/SubscriptionService.java
deleted file mode 100644
index 98f585d..0000000
--- a/streams-eip-routes/src/main/java/org/apache/streams/messaging/service/SubscriptionService.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.messaging.service;
-
-import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscription;
-
-import java.util.List;
-
-public interface SubscriptionService {
-
- List<String> getFilters(String authToken);
- void saveFilters(ActivityStreamsSubscription subscription);
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/src/main/java/org/apache/streams/messaging/service/impl/CassandraActivityService.java
----------------------------------------------------------------------
diff --git a/streams-eip-routes/src/main/java/org/apache/streams/messaging/service/impl/CassandraActivityService.java b/streams-eip-routes/src/main/java/org/apache/streams/messaging/service/impl/CassandraActivityService.java
deleted file mode 100644
index 89f71ab..0000000
--- a/streams-eip-routes/src/main/java/org/apache/streams/messaging/service/impl/CassandraActivityService.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.messaging.service.impl;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.camel.Exchange;
-import org.apache.rave.model.ActivityStreamsEntry;
-import org.apache.streams.cassandra.model.CassandraActivityStreamsEntry;
-import org.apache.streams.cassandra.repository.impl.CassandraActivityStreamsRepository;
-import org.apache.streams.messaging.service.ActivityService;
-import org.codehaus.jackson.map.DeserializationConfig;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.springframework.beans.factory.annotation.Autowired;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.List;
-
-public class CassandraActivityService implements ActivityService {
-
- private static final transient Log LOG = LogFactory.getLog(CassandraActivityService.class);
-
- private CassandraActivityStreamsRepository cassandraActivityStreamsRepository;
- private ObjectMapper mapper;
-
- @Autowired
- public CassandraActivityService(CassandraActivityStreamsRepository cassandraActivityStreamsRepository, ObjectMapper mapper) {
- this.cassandraActivityStreamsRepository = cassandraActivityStreamsRepository;
- this.mapper = mapper;
- mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- @Override
- public void receiveExchange(Exchange exchange) {
-
- //receive the exchange as a list
- List<Exchange> grouped = exchange.getProperty(Exchange.GROUPED_EXCHANGE, List.class);
-
- for (Exchange e : grouped) {
- //get activity off of exchange
- LOG.info("Exchange: " + e);
-
- //extract the ActivityStreamsEntry object and save it in the database
- LOG.info("About to preform the translation to JSON Object");
- String activityJson = e.getIn().getBody(String.class);
-
- try {
- ActivityStreamsEntry streamsEntry = mapper.readValue(activityJson, CassandraActivityStreamsEntry.class);
- streamsEntry.setPublished(new Date());
- cassandraActivityStreamsRepository.save(streamsEntry);
- } catch (IOException err) {
- LOG.error("there was an error while converting the json string to an object and saving to the database", err);
- }
-
- }
- }
-
- @Override
- public List<String> getActivitiesForFilters(List<String> filters, Date lastUpdated) {
- List<CassandraActivityStreamsEntry> activityObjects = cassandraActivityStreamsRepository.getActivitiesForFilters(filters, lastUpdated);
- Collections.sort(activityObjects, Collections.reverseOrder());
- //TODO: make the number of streams returned configurable
- return getJsonList(activityObjects.subList(0,Math.min(activityObjects.size(),10)));
- }
-
- private List<String> getJsonList(List<CassandraActivityStreamsEntry> activities) {
- List<String> jsonList = new ArrayList<String>();
- for (ActivityStreamsEntry entry : activities) {
- try {
- jsonList.add(mapper.writeValueAsString(entry));
- } catch (IOException e) {
- LOG.error("There was an error while trying to convert the java object to a string: " + entry, e);
- }
- }
- return jsonList;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/src/main/java/org/apache/streams/messaging/service/impl/CassandraSubscriptionService.java
----------------------------------------------------------------------
diff --git a/streams-eip-routes/src/main/java/org/apache/streams/messaging/service/impl/CassandraSubscriptionService.java b/streams-eip-routes/src/main/java/org/apache/streams/messaging/service/impl/CassandraSubscriptionService.java
deleted file mode 100644
index 8972d1e..0000000
--- a/streams-eip-routes/src/main/java/org/apache/streams/messaging/service/impl/CassandraSubscriptionService.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.messaging.service.impl;
-
-import org.apache.streams.cassandra.repository.impl.CassandraSubscriptionRepository;
-import org.apache.streams.messaging.service.SubscriptionService;
-import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscription;
-
-import java.util.Arrays;
-import java.util.List;
-
-public class CassandraSubscriptionService implements SubscriptionService {
-
- private CassandraSubscriptionRepository repository;
-
- public CassandraSubscriptionService(CassandraSubscriptionRepository repository){
- this.repository = repository;
- }
-
- public List<String> getFilters(String authToken){
- return Arrays.asList(repository.getFilters(authToken).split(" "));
- }
-
- public void saveFilters(ActivityStreamsSubscription subscription){
- repository.save(subscription);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/src/main/resources/META-INF/spring/propertiesLoader.xml
----------------------------------------------------------------------
diff --git a/streams-eip-routes/src/main/resources/META-INF/spring/propertiesLoader.xml b/streams-eip-routes/src/main/resources/META-INF/spring/propertiesLoader.xml
deleted file mode 100644
index 60a3f1f..0000000
--- a/streams-eip-routes/src/main/resources/META-INF/spring/propertiesLoader.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<!--<beans-->
- <!--xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"-->
- <!--xmlns="http://www.springframework.org/schema/beans"-->
- <!--xmlns:context="http://www.springframework.org/schema/context"-->
- <!--xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd-->
- <!--http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">-->
-
-
-
- <!--<context:component-scan base-package="org.apache.streams.messaging" annotation-config="true"/>-->
-
- <!--<context:property-placeholder location="/META-INF/streams.properties"/>-->
-
- <!--<bean id="configuration" class="org.apache.streams.messaging.configuration.EipConfigurator" />-->
-
-
-<!--</beans>-->
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/src/main/resources/META-INF/spring/streams-eip-applicationContext.xml
----------------------------------------------------------------------
diff --git a/streams-eip-routes/src/main/resources/META-INF/spring/streams-eip-applicationContext.xml b/streams-eip-routes/src/main/resources/META-INF/spring/streams-eip-applicationContext.xml
deleted file mode 100644
index a9b97a7..0000000
--- a/streams-eip-routes/src/main/resources/META-INF/spring/streams-eip-applicationContext.xml
+++ /dev/null
@@ -1,113 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<beans
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns="http://www.springframework.org/schema/beans"
- xmlns:context="http://www.springframework.org/schema/context"
- xmlns:task="http://www.springframework.org/schema/task"
- xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
- http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task.xsd">
-
-
- <bean id="activityConsumerRouter" class="org.apache.streams.messaging.routers.impl.ActivityConsumerRouter">
- <property name="activityConsumerWarehouse" ref="activityConsumerWarehouse"/>
- <property name="camelContext" ref="context"/>
- </bean>
-
- <bean id="activityRegistrationProcessor"
- class="org.apache.streams.messaging.processors.ActivityPublisherRegistrationProcessor"/>
-
-
- <bean id="activityStreamsSubscriberRouter"
- class="org.apache.streams.messaging.routers.impl.ActivityStreamsSubscriberRouter">
- <property name="activityStreamsSubscriberWarehouse" ref="activityStreamsSubscriberWarehouse"/>
- <property name="camelContext" ref="context"/>
- </bean>
-
- <bean id="subscriberRegistrationProcessor"
- class="org.apache.streams.messaging.processors.ActivityStreamsSubscriberRegistrationProcessor">
- <constructor-arg ref="subscriptionService"/>
- </bean>
-
- <bean id="cassandraKeyspace" class="org.apache.streams.cassandra.repository.impl.CassandraKeyspace">
- <constructor-arg ref="cassandraConfig"/>
- </bean>
-
- <bean id="cassandraActivityStreamsRepository"
- class="org.apache.streams.cassandra.repository.impl.CassandraActivityStreamsRepository">
- <constructor-arg ref="cassandraKeyspace"/>
- <constructor-arg ref="cassandraConfig"/>
- </bean>
-
- <bean id="cassandraSubscriptionRepository"
- class="org.apache.streams.cassandra.repository.impl.CassandraSubscriptionRepository">
- <constructor-arg ref="cassandraKeyspace"/>
- <constructor-arg ref="cassandraConfig"/>
- </bean>
-
- <bean id="objectMapper" class="org.codehaus.jackson.map.ObjectMapper"/>
-
- <bean id="subscriptionService" class="org.apache.streams.messaging.service.impl.CassandraSubscriptionService">
- <constructor-arg ref="cassandraSubscriptionRepository"/>
- </bean>
-
- <bean id="activityService" class="org.apache.streams.messaging.service.impl.CassandraActivityService">
- <constructor-arg ref="cassandraActivityStreamsRepository"/>
- <constructor-arg ref="objectMapper"/>
- </bean>
-
- <!--<bean id="stromActivityAggregator" class="org.apache.streams.messaging.storm.StormActivityAggregator">-->
- <!--<constructor-arg ref="bolt"/>-->
- <!--<constructor-arg ref="spout"/>-->
- <!--</bean>-->
-
- <!--<bean id="bolt" class="org.apache.streams.messaging.storm.StormSubscriberBolt"/>-->
- <!--<bean id="spout" class="org.apache.streams.messaging.storm.StormSubscriberSpout"/>-->
-
- <bean id="activityAggregator" class="org.apache.streams.messaging.aggregation.ActivityAggregator">
- <property name="activityService" ref="activityService"/>
- <property name="activityStreamsSubscriberWarehouse" ref="activityStreamsSubscriberWarehouse"/>
- </bean>
-
- <task:annotation-driven/>
-
- <bean id="jmsConnectionFactory"
- class="org.apache.activemq.ActiveMQConnectionFactory">
- <property name="brokerURL" value="${activemq.jmsConnectionFactoryUrl}"/>
- </bean>
-
- <bean id="pooledConnectionFactory"
- class="org.apache.activemq.pool.PooledConnectionFactory" init-method="start" destroy-method="stop">
- <property name="maxConnections" value="8"/>
- <property name="connectionFactory" ref="jmsConnectionFactory"/>
- </bean>
-
- <bean id="jmsConfig"
- class="org.apache.camel.component.jms.JmsConfiguration">
- <property name="connectionFactory" ref="pooledConnectionFactory"/>
- <property name="concurrentConsumers" value="10"/>
- </bean>
-
- <bean id="activemq"
- class="org.apache.activemq.camel.component.ActiveMQComponent">
- <property name="configuration" ref="jmsConfig"/>
- </bean>
-
-
-</beans>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/src/main/resources/META-INF/spring/streams-eip-osgi-component-import.xml
----------------------------------------------------------------------
diff --git a/streams-eip-routes/src/main/resources/META-INF/spring/streams-eip-osgi-component-import.xml b/streams-eip-routes/src/main/resources/META-INF/spring/streams-eip-osgi-component-import.xml
deleted file mode 100644
index 9066206..0000000
--- a/streams-eip-routes/src/main/resources/META-INF/spring/streams-eip-osgi-component-import.xml
+++ /dev/null
@@ -1,38 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<beans xmlns="http://www.springframework.org/schema/beans"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns:osgi="http://www.springframework.org/schema/osgi"
- xsi:schemaLocation="http://www.springframework.org/schema/beans
- http://www.springframework.org/schema/beans/spring-beans.xsd
- http://www.springframework.org/schema/osgi
- http://www.springframework.org/schema/osgi/spring-osgi.xsd">
-
-
- <osgi:reference id="activityPublisherRegistration" interface="org.apache.streams.osgi.components.ActivityPublisherRegistration" />
- <osgi:reference id="activityConsumerWarehouse" interface="org.apache.streams.osgi.components.activityconsumer.ActivityConsumerWarehouse" />
-
- <osgi:reference id="activityStreamsSubscriberRegistration" interface="org.apache.streams.osgi.components.ActivityStreamsSubscriberRegistration" />
- <osgi:reference id="activityStreamsSubscriberWarehouse" interface="org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscriberWarehouse" />
-
-
-
-
-</beans>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/src/main/resources/META-INF/spring/streamsCamelContext.xml
----------------------------------------------------------------------
diff --git a/streams-eip-routes/src/main/resources/META-INF/spring/streamsCamelContext.xml b/streams-eip-routes/src/main/resources/META-INF/spring/streamsCamelContext.xml
deleted file mode 100644
index 4360f39..0000000
--- a/streams-eip-routes/src/main/resources/META-INF/spring/streamsCamelContext.xml
+++ /dev/null
@@ -1,96 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<beans xmlns="http://www.springframework.org/schema/beans"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns:osgi="http://www.springframework.org/schema/osgi"
- xsi:schemaLocation="
- http://www.springframework.org/schema/beans
- http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
- http://camel.apache.org/schema/spring
- http://camel.apache.org/schema/spring/camel-spring-2.0.0.xsd
- http://www.springframework.org/schema/osgi
- http://www.springframework.org/schema/osgi/spring-osgi.xsd">
-
-
-
-
- <camelContext id="context" xmlns="http://camel.apache.org/schema/spring">
-
- <endpoint id="consumerRegistrationEndpoint" uri="${consumer.registrationEndpoint}"/>
- <endpoint id="subscriberRegistrationEndpoint" uri="${subscriber.registrationEndpoint}"/>
- <!--publisher registration route setup -->
- <route>
- <from uri="ref:consumerRegistrationEndpoint"/>
- <bean ref="activityRegistrationProcessor" />
- <to uri="direct:publisher-register"/>
- </route>
-
- <route>
- <from uri="direct:publisher-register"/>
- <bean ref="activityPublisherRegistration" method="register"/>
- <to uri="direct:add-publisher-route"/>
- </route>
-
- <route>
- <from uri="direct:add-publisher-route"/>
- <bean ref="activityConsumerRouter" method="createNewRouteForConsumer"/>
- <to uri="log:ExampleLog"/>
- </route>
-
- <!--split activities on Q, waiting for aggregation -->
- <route>
- <from uri="direct:activityQ"/>
- <inOnly uri="activemq:queue:activities"/>
- </route>
-
- <route>
- <from uri="activemq:queue:activities"/>
- <aggregate completionInterval="500" groupExchanges="true">
- <correlationExpression>
- <constant>true</constant>
- </correlationExpression>
- <bean ref="activityService" method="receiveExchange"/>
- </aggregate>
- </route>
-
-
- <!-- register as a subscriber - returned the endpoint to poll and add to subscription sources - GET/POST -->
- <route>
- <from uri="ref:subscriberRegistrationEndpoint"/>
- <bean ref="subscriberRegistrationProcessor" />
- <to uri="direct:subscriber-register"/>
- </route>
-
- <route>
- <from uri="direct:subscriber-register"/>
- <bean ref="activityStreamsSubscriberRegistration" method="register"/>
- <to uri="direct:add-subscriber-route"/>
- </route>
-
- <route>
- <from uri="direct:add-subscriber-route"/>
- <bean ref="activityStreamsSubscriberRouter" method="createNewRouteForSubscriber"/>
- <to uri="log:ExampleLog"/>
- </route>
-
-
- </camelContext>
-
-</beans>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/src/main/resources/META-INF/streams.properties
----------------------------------------------------------------------
diff --git a/streams-eip-routes/src/main/resources/META-INF/streams.properties b/streams-eip-routes/src/main/resources/META-INF/streams.properties
deleted file mode 100644
index b7bbfce..0000000
--- a/streams-eip-routes/src/main/resources/META-INF/streams.properties
+++ /dev/null
@@ -1,41 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-servlet.baseUrlPath=http://localhost:8080/streams-web/
-
-consumer.inRouteHost=localhost
-consumer.inRoutePort=8000
-consumer.activityQUri = direct:activityQ
-
-consumer.publisherEndpointProtocol=jetty:http://
-consumer.publisherEndpointUrlResource=streams/publish
-consumer.receiveMethod=receive
-consumer.splitMethod=split
-
-consumer.registrationEndpoint=jetty:http://localhost:8000/streams/publisher/register
-
-subscriber.inRouteHost=localhost
-subscriber.inRoutePort=8000
-subscriber.subscriberEndpointUrlResource=streams/subscriber
-subscriber.receiveMethod=receive
-subscriber.postMethod=updateActivityStreamsSubscriberConfiguration
-subscriber.getMethod=getStream
-subscriber.registrationEndpoint=jetty:http://localhost:8000/streams/subscriber/register
-subscriber.subscriberEndpointProtocol=jetty:http://
-
-activemq.jmsConnectionFactoryUrl=tcp://localhost:61616
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-eip-routes/src/test/java/org/apache/streams/messaging/service/impl/CassandraActivityServiceTest.java
----------------------------------------------------------------------
diff --git a/streams-eip-routes/src/test/java/org/apache/streams/messaging/service/impl/CassandraActivityServiceTest.java b/streams-eip-routes/src/test/java/org/apache/streams/messaging/service/impl/CassandraActivityServiceTest.java
deleted file mode 100644
index d22826f..0000000
--- a/streams-eip-routes/src/test/java/org/apache/streams/messaging/service/impl/CassandraActivityServiceTest.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.messaging.service.impl;
-
-import org.apache.camel.Exchange;
-import org.apache.camel.Message;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-
-import static org.easymock.EasyMock.*;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
-
-public class CassandraActivityServiceTest {
-
- private CassandraActivityService cassandraActivityService;
-
- @Before
- public void setup(){
-// cassandraActivityService = new CassandraActivityService();
- }
-
- @Ignore
- @Test
- public void getActivititiesForFilterTest(){
- List<String> activities = cassandraActivityService.getActivitiesForFilters(Arrays.asList("r501"), new Date(0));
- }
-
- @Ignore
- @Test
- public void receiveExchangeTest(){
- Exchange e = createMock(Exchange.class);
- List<Exchange> grouped = new ArrayList<Exchange>();
- Exchange e2 = createMock(Exchange.class);
- grouped.add(e2);
- Message m = createMock(Message.class);
-
- String activityJson = "{\n" +
- "\"id\":\"id2\",\n" +
- "\"verb\":\"verb2\",\n" +
- "\"displayName\":\"displayname2\",\n" +
- "\"target\":{\n" +
- "\t\"id\":\"targetid2\",\n" +
- "\t\"displayName\":\"targetname2\"\n" +
- "\t},\n" +
- "\t\"object\":{\n" +
- "\t\"id\":\"objectid2\",\n" +
- "\t\"displayName\":\"objectname2\"\n" +
- "\t},\n" +
- "\t\"actor\":{\n" +
- "\t\"id\":\"actorid2\",\n" +
- "\t\"displayName\":\"actorname2\"\n" +
- "\t}\n" +
- "\t\n" +
- "\t}";
-
- expect(e.getProperty(Exchange.GROUPED_EXCHANGE, List.class)).andReturn(grouped);
- expect(e2.getIn()).andReturn(m);
- expect(m.getBody(String.class)).andReturn(activityJson);
-
- replay(e, e2, m);
-
- cassandraActivityService.receiveExchange(e);
- //List<String> myTest = cassandraActivityService.getActivitiesForQuery("select * from coltest");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-pojo-extensions/pom.xml
----------------------------------------------------------------------
diff --git a/streams-pojo-extensions/pom.xml b/streams-pojo-extensions/pom.xml
deleted file mode 100644
index 85796dd..0000000
--- a/streams-pojo-extensions/pom.xml
+++ /dev/null
@@ -1,149 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-project</artifactId>
- <version>0.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-pojo-extensions</artifactId>
-
- <properties>
- <streams.version>0.1-SNAPSHOT</streams.version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${streams.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <!--<plugin>-->
- <!--<artifactId>maven-remote-resources-plugin</artifactId>-->
- <!--<configuration>-->
- <!--<resourceBundles>-->
- <!--<resourceBundle>org.apache.streams:shared-resources:${project.version}</resourceBundle>-->
- <!--</resourceBundles>-->
- <!--</configuration>-->
- <!--<executions>-->
- <!--<execution>-->
- <!--<id>process-remote-resources</id>-->
- <!--<goals>-->
- <!--<goal>process</goal>-->
- <!--</goals>-->
- <!--</execution>-->
- <!--</executions>-->
- <!--<configuration>-->
- <!--<includes>-->
- <!--<include>**/*.json</include>-->
- <!--</includes>-->
- <!--</configuration>-->
- <!--</plugin>-->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.8</version>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/jsonschema2pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-maven-plugin</artifactId>
- <configuration>
- <addCompileSourceRoot>true</addCompileSourceRoot>
- <generateBuilders>true</generateBuilders>
- <sourcePaths>
- <sourcePath>src/main/jsonschema/org/apache/streams/ActivityExtended.json</sourcePath>
- </sourcePaths>
- <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
- <targetPackage>org.apache.streams</targetPackage>
- <useLongIntegers>true</useLongIntegers>
- <useJodaDates>false</useJodaDates>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-pojo-extensions/src/main/jsonschema/org/apache/streams/ActivityExtended.json
----------------------------------------------------------------------
diff --git a/streams-pojo-extensions/src/main/jsonschema/org/apache/streams/ActivityExtended.json b/streams-pojo-extensions/src/main/jsonschema/org/apache/streams/ActivityExtended.json
deleted file mode 100644
index c900c05..0000000
--- a/streams-pojo-extensions/src/main/jsonschema/org/apache/streams/ActivityExtended.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "type": "object",
- "title" : "activity",
- "$schema": "http://json-schema.org/draft-03/schema",
- "id": "#",
- "javaType" : "org.apache.streams.pojo.json.ActivityExtended",
- "properties": {
- "extensions": {
- "type": "object",
- "dynamic": "true",
- "required": false,
- "description": "Extension fields go here.",
- "javaType" : "org.apache.streams.pojo.json.Extensions"
- }
- },
- "extends": "file://../../../../../../../streams-pojo/src/main/jsonschema/org/apache/streams/pojo/activity.json"
-}
\ No newline at end of file
[03/13] adding some uncommitted modules reorganizing pom improvements
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/test/Top100FeedsTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/test/Top100FeedsTest.java b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/test/Top100FeedsTest.java
index 1277553..780aa3b 100644
--- a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/test/Top100FeedsTest.java
+++ b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/test/Top100FeedsTest.java
@@ -2,7 +2,6 @@ package org.apache.streams.rss.test;
import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
-import org.apache.streams.core.tasks.StreamsProviderTask;
import org.apache.streams.pojo.json.Activity;
import org.apache.streams.rss.FeedDetails;
import org.apache.streams.rss.RssStreamConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/main/java/org/apache/streams/core/StreamBuilder.java
----------------------------------------------------------------------
diff --git a/streams-core/src/main/java/org/apache/streams/core/StreamBuilder.java b/streams-core/src/main/java/org/apache/streams/core/StreamBuilder.java
new file mode 100644
index 0000000..476d8c1
--- /dev/null
+++ b/streams-core/src/main/java/org/apache/streams/core/StreamBuilder.java
@@ -0,0 +1,104 @@
+package org.apache.streams.core;
+
+import org.joda.time.DateTime;
+
+import java.math.BigInteger;
+
+/**
+ * Interface for building data streams.
+ *
+ * <pre>
+ * StreamBuilder builder = ...
+ * builder.newReadCurrentStream(. . .)
+ * .addStreamsProcessor(. . .)
+ * ...
+ * .addStreamsPersistWriter(. . .)
+ * builder.run();
+ * </pre>
+ *
+ */
+public interface StreamBuilder {
+
+
+ /**
+ * Add a {@link org.apache.streams.core.StreamsProcessor} to the data processing stream.
+ * @param processorId unique id for this processor - must be unique across the entire stream
+ * @param processor the processor to execute
+ * @param numTasks the number of instances of this processor to run concurrently
+ * @param connectToIds the ids of the {@link org.apache.streams.core.StreamsOperation} that this process will
+ * receive data from.
+ * @return this
+ */
+ public StreamBuilder addStreamsProcessor(String processorId, StreamsProcessor processor, int numTasks, String... connectToIds);
+
+ /**
+ * Add a {@link org.apache.streams.core.StreamsPersistWriter} to the data processing stream.
+ * @param persistWriterId unique id for this processor - must be unique across the entire stream
+ * @param writer the writer to execute
+ * @param numTasks the number of instances of this writer to run concurrently
+ * @param connectToIds the ids of the {@link org.apache.streams.core.StreamsOperation} that this process will
+ * receive data from.
+ * @return this
+ */
+ public StreamBuilder addStreamsPersistWriter(String persistWriterId, StreamsPersistWriter writer, int numTasks, String... connectToIds);
+
+ /**
+ * Add a {@link org.apache.streams.core.StreamsProvider} to the data processing stream. The provider will execute
+ * {@link org.apache.streams.core.StreamsProvider:readCurrent()} to produce data.
+ * @param streamId unique if for this provider - must be unique across the entire stream.
+ * @param provider provider to execute
+ * @return this
+ */
+ public StreamBuilder newPerpetualStream(String streamId, StreamsProvider provider);
+
+ /**
+ * Add a {@link org.apache.streams.core.StreamsProvider} to the data processing stream. The provider will execute
+ * {@link org.apache.streams.core.StreamsProvider:readCurrent()} to produce data.
+ * @param streamId unique if for this provider - must be unique across the entire stream.
+ * @param provider provider to execute
+ * @return this
+ */
+ public StreamBuilder newReadCurrentStream(String streamId, StreamsProvider provider);
+
+ /**
+ * Add a {@link org.apache.streams.core.StreamsProvider} to the data processing stream. The provider will execute
+ * {@link org.apache.streams.core.StreamsProvider:readNext(BigInteger)} to produce data.
+ * @param streamId unique if for this provider - must be unique across the entire stream.
+ * @param provider provider to execute
+ * @param sequence sequence to pass to {@link org.apache.streams.core.StreamsProvider:readNext(BigInteger)} method
+ * @return this
+ */
+ public StreamBuilder newReadNewStream(String streamId, StreamsProvider provider, BigInteger sequence);
+
+ /**
+ * Add a {@link org.apache.streams.core.StreamsProvider} to the data processing stream. The provider will execute
+ * {@link org.apache.streams.core.StreamsProvider:readRange(DateTime, DateTime)} to produce data. Whether the start
+ * and end dates are inclusive or exclusive is up to the implementation.
+ * @param streamId unique if for this provider - must be unique across the entire stream.
+ * @param provider provider to execute
+ * @param start start date
+ * @param end end date
+ * @return this
+ */
+ public StreamBuilder newReadRangeStream(String streamId, StreamsProvider provider, DateTime start, DateTime end);
+
+ /**
+ * Builds the stream, and starts it or submits it based on implementation.
+ */
+ public void start();
+
+ /**
+ * Stops the streams processing. No guarantee on a smooth shutdown. Optional method, may not be implemented in
+ * all cases.
+ */
+ public void stop();
+
+
+
+
+
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/main/java/org/apache/streams/core/StreamsResultSet.java
----------------------------------------------------------------------
diff --git a/streams-core/src/main/java/org/apache/streams/core/StreamsResultSet.java b/streams-core/src/main/java/org/apache/streams/core/StreamsResultSet.java
index d86ce6b..1e1cbe2 100644
--- a/streams-core/src/main/java/org/apache/streams/core/StreamsResultSet.java
+++ b/streams-core/src/main/java/org/apache/streams/core/StreamsResultSet.java
@@ -18,7 +18,6 @@
package org.apache.streams.core;
-import java.math.BigInteger;
import java.util.Iterator;
import java.util.Queue;
@@ -26,12 +25,36 @@ public class StreamsResultSet implements Iterable<StreamsDatum> {
Queue<StreamsDatum> queue;
+ DatumStatusCounter counter;
+
public StreamsResultSet(Queue<StreamsDatum> queue) {
this.queue = queue;
}
+
@Override
public Iterator<StreamsDatum> iterator() {
return queue.iterator();
}
+
+ public int size() {
+ return queue.size();
+ }
+
+ public Queue<StreamsDatum> getQueue() {
+ return queue;
+ }
+
+ public void setQueue(Queue<StreamsDatum> queue) {
+ this.queue = queue;
+ }
+
+ public DatumStatusCounter getCounter() {
+ return counter;
+ }
+
+ public void setCounter(DatumStatusCounter counter) {
+ this.counter = counter;
+ }
}
+
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/main/java/org/apache/streams/core/builders/InvalidStreamException.java
----------------------------------------------------------------------
diff --git a/streams-core/src/main/java/org/apache/streams/core/builders/InvalidStreamException.java b/streams-core/src/main/java/org/apache/streams/core/builders/InvalidStreamException.java
deleted file mode 100644
index 43c5a4c..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/builders/InvalidStreamException.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package org.apache.streams.core.builders;
-
-/**
- * Exception that indicates a malformed data stream in some way.
- */
-public class InvalidStreamException extends RuntimeException {
-
- public InvalidStreamException() {
- super();
- }
-
- public InvalidStreamException(String s) {
- super(s);
- }
-
- public InvalidStreamException(String s, Throwable throwable) {
- super(s, throwable);
- }
-
- public InvalidStreamException(Throwable throwable) {
- super(throwable);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/main/java/org/apache/streams/core/builders/LocalStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streams-core/src/main/java/org/apache/streams/core/builders/LocalStreamBuilder.java b/streams-core/src/main/java/org/apache/streams/core/builders/LocalStreamBuilder.java
deleted file mode 100644
index 7dd5ced..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/builders/LocalStreamBuilder.java
+++ /dev/null
@@ -1,265 +0,0 @@
-package org.apache.streams.core.builders;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.tasks.LocalStreamMonitorThread;
-import org.apache.streams.core.tasks.StreamsProviderTask;
-import org.apache.streams.core.tasks.StreamsTask;
-import org.apache.streams.util.SerializationUtil;
-import org.joda.time.DateTime;
-
-import java.math.BigInteger;
-import java.util.*;
-import java.util.concurrent.*;
-
-/**
- * {@link org.apache.streams.core.builders.StreamBuilder} implementation to run a data processing stream in a single
- * JVM across many threads. Depending on your data stream, the JVM heap may need to be set to a high value. Default
- * implementation uses unbound {@link java.util.concurrent.ConcurrentLinkedQueue} to connect stream components.
- */
-public class LocalStreamBuilder implements StreamBuilder{
-
- private Map<String, StreamComponent> providers;
- private Map<String, StreamComponent> components;
- private Queue<StreamsDatum> queue;
- private Map<String, Object> streamConfig;
- private ExecutorService executor;
- private int totalTasks;
- private LocalStreamMonitorThread monitorThread;
-
- /**
- *
- */
- public LocalStreamBuilder(){
- this(new ConcurrentLinkedQueue<StreamsDatum>(), null);
- }
-
- /**
- *
- * @param streamConfig
- */
- public LocalStreamBuilder(Map<String, Object> streamConfig) {
- this(new ConcurrentLinkedQueue<StreamsDatum>(), streamConfig);
- }
-
- /**
- *
- * @param queueType
- */
- public LocalStreamBuilder(Queue<StreamsDatum> queueType) {
- this(queueType, null);
- }
-
- /**
- *
- * @param queueType
- * @param streamConfig
- */
- public LocalStreamBuilder(Queue<StreamsDatum> queueType, Map<String, Object> streamConfig) {
- this.queue = queueType;
- this.providers = new HashMap<String, StreamComponent>();
- this.components = new HashMap<String, StreamComponent>();
- this.streamConfig = streamConfig;
- this.totalTasks = 0;
- }
-
- @Override
- public StreamBuilder newPerpetualStream(String id, StreamsProvider provider) {
- validateId(id);
- this.providers.put(id, new StreamComponent(id, provider, true));
- ++this.totalTasks;
- return this;
- }
-
- @Override
- public StreamBuilder newReadCurrentStream(String id, StreamsProvider provider) {
- validateId(id);
- this.providers.put(id, new StreamComponent(id, provider, false));
- ++this.totalTasks;
- return this;
- }
-
- @Override
- public StreamBuilder newReadNewStream(String id, StreamsProvider provider, BigInteger sequence) {
- validateId(id);
- this.providers.put(id, new StreamComponent(id, provider, sequence));
- ++this.totalTasks;
- return this;
- }
-
- @Override
- public StreamBuilder newReadRangeStream(String id, StreamsProvider provider, DateTime start, DateTime end) {
- validateId(id);
- this.providers.put(id, new StreamComponent(id, provider, start, end));
- ++this.totalTasks;
- return this;
- }
-
- @Override
- public StreamBuilder addStreamsProcessor(String id, StreamsProcessor processor, int numTasks, String... inBoundIds) {
- validateId(id);
- StreamComponent comp = new StreamComponent(id, processor, cloneQueue(), numTasks);
- this.components.put(id, comp);
- connectToOtherComponents(inBoundIds, comp);
- this.totalTasks += numTasks;
- return this;
- }
-
- @Override
- public StreamBuilder addStreamsPersistWriter(String id, StreamsPersistWriter writer, int numTasks, String... inBoundIds) {
- validateId(id);
- StreamComponent comp = new StreamComponent(id, writer, cloneQueue(), numTasks);
- this.components.put(id, comp);
- connectToOtherComponents(inBoundIds, comp);
- this.totalTasks += numTasks;
- return this;
- }
-
- /**
- * Runs the data stream in the this JVM and blocks till completion.
- */
- @Override
- public void start() {
- boolean isRunning = true;
- this.executor = Executors.newFixedThreadPool(this.totalTasks);
- Map<String, StreamsProviderTask> provTasks = new HashMap<String, StreamsProviderTask>();
- Map<String, List<StreamsTask>> streamsTasks = new HashMap<String, List<StreamsTask>>();
- monitorThread = new LocalStreamMonitorThread(this.executor, 1000);
- try {
- //this.executor.submit(monitorThread);
- for(StreamComponent comp : this.components.values()) {
- int tasks = comp.getNumTasks();
- List<StreamsTask> compTasks = new LinkedList<StreamsTask>();
- for(int i=0; i < tasks; ++i) {
- StreamsTask task = comp.createConnectedTask();
- task.setStreamConfig(this.streamConfig);
- this.executor.submit(task);
- compTasks.add(task);
- }
- streamsTasks.put(comp.getId(), compTasks);
- }
- for(StreamComponent prov : this.providers.values()) {
- StreamsTask task = prov.createConnectedTask();
- task.setStreamConfig(this.streamConfig);
- this.executor.submit(task);
- provTasks.put(prov.getId(), (StreamsProviderTask) task);
- }
- while(isRunning) {
- isRunning = false;
- for(StreamsProviderTask task : provTasks.values()) {
- isRunning = isRunning || task.isRunning();
- }
- if(isRunning) {
- Thread.sleep(3000);
- }
- }
- //monitorThread.shutdown();
- this.executor.shutdown();
- //complete stream shut down gracfully
- for(StreamComponent prov : this.providers.values()) {
- shutDownTask(prov, streamsTasks);
- }
- //need to make this configurable
- if(!this.executor.awaitTermination(10, TimeUnit.SECONDS)) { // all threads should have terminated already.
- this.executor.shutdownNow();
- this.executor.awaitTermination(10, TimeUnit.SECONDS);
- }
- } catch (InterruptedException e){
- //give the stream 30secs to try to shutdown gracefully, then force shutdown otherwise
- for(List<StreamsTask> tasks : streamsTasks.values()) {
- for(StreamsTask task : tasks) {
- task.stopTask();
- }
- }
- this.executor.shutdown();
- try {
- if(!this.executor.awaitTermination(30, TimeUnit.SECONDS)){
- this.executor.shutdownNow();
- }
- }catch (InterruptedException ie) {
- this.executor.shutdownNow();
- throw new RuntimeException(ie);
- }
- }
-
- }
-
- /**
- * Shutsdown the running tasks in sudo depth first search kind of way. Checks that the upstream components have
- * finished running before shutting down. Waits till inbound queue is empty to shutdown.
- * @param comp StreamComponent to shut down.
- * @param streamTasks the list of non-StreamsProvider tasks for this stream.
- * @throws InterruptedException
- */
- private void shutDownTask(StreamComponent comp, Map<String, List<StreamsTask>> streamTasks) throws InterruptedException {
- List<StreamsTask> tasks = streamTasks.get(comp.getId());
- if(tasks != null) { //not a StreamProvider
- boolean parentsShutDown = true;
- for(StreamComponent parent : comp.getUpStreamComponents()) {
- List<StreamsTask> parentTasks = streamTasks.get(parent.getId());
- //if parentTask == null, its a provider and is not running anymore
- if(parentTasks != null) {
- for(StreamsTask task : parentTasks) {
- parentsShutDown = parentsShutDown && !task.isRunning();
- }
- }
- }
- if(parentsShutDown) {
- for(StreamsTask task : tasks) {
- task.stopTask();
- }
- for(StreamsTask task : tasks) {
- while(task.isRunning()) {
- Thread.sleep(500);
- }
- }
- }
- }
- Collection<StreamComponent> children = comp.getDownStreamComponents();
- if(children != null) {
- for(StreamComponent child : comp.getDownStreamComponents()) {
- shutDownTask(child, streamTasks);
- }
- }
- }
-
- /**
- * NOT IMPLEMENTED.
- */
- @Override
- public void stop() {
-
- }
-
- private void connectToOtherComponents(String[] conntectToIds, StreamComponent toBeConnected) {
- for(String id : conntectToIds) {
- StreamComponent upStream = null;
- if(this.providers.containsKey(id)) {
- upStream = this.providers.get(id);
- }
- else if(this.components.containsKey(id)) {
- upStream = this.components.get(id);
- }
- else {
- throw new InvalidStreamException("Cannot connect to id, "+id+", because id does not exist.");
- }
- upStream.addOutBoundQueue(toBeConnected, toBeConnected.getInBoundQueue());
- toBeConnected.addInboundQueue(upStream);
- }
- }
-
- private void validateId(String id) {
- if(this.providers.containsKey(id) || this.components.containsKey(id)) {
- throw new InvalidStreamException("Duplicate id. "+id+" is already assigned to another component");
- }
- }
-
-
- private Queue<StreamsDatum> cloneQueue() {
- return (Queue<StreamsDatum>)SerializationUtil.cloneBySerialization(this.queue);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/main/java/org/apache/streams/core/builders/StreamBuilder.java
----------------------------------------------------------------------
diff --git a/streams-core/src/main/java/org/apache/streams/core/builders/StreamBuilder.java b/streams-core/src/main/java/org/apache/streams/core/builders/StreamBuilder.java
deleted file mode 100644
index 133776d..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/builders/StreamBuilder.java
+++ /dev/null
@@ -1,106 +0,0 @@
-package org.apache.streams.core.builders;
-
-import org.apache.streams.core.*;
-import org.joda.time.DateTime;
-
-import java.math.BigInteger;
-import java.util.Queue;
-
-/**
- * Interface for building data streams.
- *
- * <pre>
- * StreamBuilder builder = ...
- * builder.newReadCurrentStream(. . .)
- * .addStreamsProcessor(. . .)
- * ...
- * .addStreamsPersistWriter(. . .)
- * builder.run();
- * </pre>
- *
- */
-public interface StreamBuilder {
-
-
- /**
- * Add a {@link org.apache.streams.core.StreamsProcessor} to the data processing stream.
- * @param processorId unique id for this processor - must be unique across the entire stream
- * @param processor the processor to execute
- * @param numTasks the number of instances of this processor to run concurrently
- * @param connectToIds the ids of the {@link org.apache.streams.core.StreamsOperation} that this process will
- * receive data from.
- * @return this
- */
- public StreamBuilder addStreamsProcessor(String processorId, StreamsProcessor processor, int numTasks, String... connectToIds);
-
- /**
- * Add a {@link org.apache.streams.core.StreamsPersistWriter} to the data processing stream.
- * @param persistWriterId unique id for this processor - must be unique across the entire stream
- * @param writer the writer to execute
- * @param numTasks the number of instances of this writer to run concurrently
- * @param connectToIds the ids of the {@link org.apache.streams.core.StreamsOperation} that this process will
- * receive data from.
- * @return this
- */
- public StreamBuilder addStreamsPersistWriter(String persistWriterId, StreamsPersistWriter writer, int numTasks, String... connectToIds);
-
- /**
- * Add a {@link org.apache.streams.core.StreamsProvider} to the data processing stream. The provider will execute
- * {@link org.apache.streams.core.StreamsProvider:readCurrent()} to produce data.
- * @param streamId unique if for this provider - must be unique across the entire stream.
- * @param provider provider to execute
- * @return this
- */
- public StreamBuilder newPerpetualStream(String streamId, StreamsProvider provider);
-
- /**
- * Add a {@link org.apache.streams.core.StreamsProvider} to the data processing stream. The provider will execute
- * {@link org.apache.streams.core.StreamsProvider:readCurrent()} to produce data.
- * @param streamId unique if for this provider - must be unique across the entire stream.
- * @param provider provider to execute
- * @return this
- */
- public StreamBuilder newReadCurrentStream(String streamId, StreamsProvider provider);
-
- /**
- * Add a {@link org.apache.streams.core.StreamsProvider} to the data processing stream. The provider will execute
- * {@link org.apache.streams.core.StreamsProvider:readNext(BigInteger)} to produce data.
- * @param streamId unique if for this provider - must be unique across the entire stream.
- * @param provider provider to execute
- * @param sequence sequence to pass to {@link org.apache.streams.core.StreamsProvider:readNext(BigInteger)} method
- * @return this
- */
- public StreamBuilder newReadNewStream(String streamId, StreamsProvider provider, BigInteger sequence);
-
- /**
- * Add a {@link org.apache.streams.core.StreamsProvider} to the data processing stream. The provider will execute
- * {@link org.apache.streams.core.StreamsProvider:readRange(DateTime, DateTime)} to produce data. Whether the start
- * and end dates are inclusive or exclusive is up to the implementation.
- * @param streamId unique if for this provider - must be unique across the entire stream.
- * @param provider provider to execute
- * @param start start date
- * @param end end date
- * @return this
- */
- public StreamBuilder newReadRangeStream(String streamId, StreamsProvider provider, DateTime start, DateTime end);
-
- /**
- * Builds the stream, and starts it or submits it based on implementation.
- */
- public void start();
-
- /**
- * Stops the streams processing. No guarantee on a smooth shutdown. Optional method, may not be implemented in
- * all cases.
- */
- public void stop();
-
-
-
-
-
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/main/java/org/apache/streams/core/builders/StreamComponent.java
----------------------------------------------------------------------
diff --git a/streams-core/src/main/java/org/apache/streams/core/builders/StreamComponent.java b/streams-core/src/main/java/org/apache/streams/core/builders/StreamComponent.java
deleted file mode 100644
index 15cb9e8..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/builders/StreamComponent.java
+++ /dev/null
@@ -1,229 +0,0 @@
-package org.apache.streams.core.builders;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.tasks.StreamsPersistWriterTask;
-import org.apache.streams.core.tasks.StreamsProcessorTask;
-import org.apache.streams.core.tasks.StreamsProviderTask;
-import org.apache.streams.core.tasks.StreamsTask;
-import org.apache.streams.util.SerializationUtil;
-import org.joda.time.DateTime;
-
-import java.math.BigInteger;
-import java.util.*;
-
-/**
- * Stores the implementations of {@link org.apache.streams.core.StreamsOperation}, the StreamsOperations it is connected
- * to and the necessary metadata to construct a data stream.
- */
-public class StreamComponent {
-
- private static final int START = 1;
- private static final int END = 2;
-
- private String id;
- private Set<StreamComponent> inBound;
- private Map<StreamComponent, Queue<StreamsDatum>> outBound;
- private Queue<StreamsDatum> inQueue;
- private StreamsProvider provider;
- private StreamsProcessor processor;
- private StreamsPersistWriter writer;
- private DateTime[] dateRange;
- private BigInteger sequence;
- private int numTasks = 1;
- private boolean perpetual;
-
- /**
- *
- * @param id
- * @param provider
- */
- public StreamComponent(String id, StreamsProvider provider, boolean perpetual) {
- this.id = id;
- this.provider = provider;
- this.perpetual = perpetual;
- initializePrivateVariables();
- }
-
- /**
- *
- * @param id
- * @param provider
- * @param start
- * @param end
- */
- public StreamComponent(String id, StreamsProvider provider, DateTime start, DateTime end) {
- this.id = id;
- this.provider = provider;
- this.dateRange = new DateTime[2];
- this.dateRange[START] = start;
- this.dateRange[END] = end;
- initializePrivateVariables();
- }
-
-
- /**
- *
- * @param id
- * @param provider
- * @param sequence
- */
- public StreamComponent(String id, StreamsProvider provider, BigInteger sequence) {
- this.id = id;
- this.provider = provider;
- this.sequence = sequence;
- }
-
- /**
- *
- * @param id
- * @param processor
- * @param inQueue
- * @param numTasks
- */
- public StreamComponent(String id, StreamsProcessor processor, Queue<StreamsDatum> inQueue, int numTasks) {
- this.id = id;
- this.processor = processor;
- this.inQueue = inQueue;
- this.numTasks = numTasks;
- initializePrivateVariables();
- }
-
- /**
- *
- * @param id
- * @param writer
- * @param inQueue
- * @param numTasks
- */
- public StreamComponent(String id, StreamsPersistWriter writer, Queue<StreamsDatum> inQueue, int numTasks) {
- this.id = id;
- this.writer = writer;
- this.inQueue = inQueue;
- this.numTasks = numTasks;
- initializePrivateVariables();
- }
-
- private void initializePrivateVariables() {
- this.inBound = new HashSet<StreamComponent>();
- this.outBound = new HashMap<StreamComponent, Queue<StreamsDatum>>();
- }
-
- /**
- * Add an outbound queue for this component. The queue should be an inbound queue of a downstream component.
- * @param component the component that this supplying their inbound queue
- * @param queue the queue to to put post processed/provided datums on
- */
- public void addOutBoundQueue(StreamComponent component, Queue<StreamsDatum> queue) {
- this.outBound.put(component, queue);
- }
-
- /**
- * Add a component that supplies data through the inbound queue.
- * @param component that supplies data through the inbound queue
- */
- public void addInboundQueue(StreamComponent component) {
- this.inBound.add(component);
- }
-
- /**
- * The components that are immediately downstream of this component (aka child nodes)
- * @return Collection of child nodes of this component
- */
- public Collection<StreamComponent> getDownStreamComponents() {
- return this.outBound.keySet();
- }
-
- /**
- * The components that are immediately upstream of this component (aka parent nodes)
- * @return Collection of parent nodes of this component
- */
- public Collection<StreamComponent> getUpStreamComponents() {
- return this.inBound;
- }
-
- /**
- * The inbound queue for this component
- * @return inbound queue
- */
- public Queue<StreamsDatum> getInBoundQueue() {
- return this.inQueue;
- }
-
- /**
- * The number of tasks this to run this component
- * @return
- */
- public int getNumTasks() {
- return this.numTasks;
- }
-
- /**
- * Creates a {@link org.apache.streams.core.tasks.StreamsTask} that is running a clone of this component whose
- * inbound and outbound queues are appropriately connected to the parent and child nodes.
- * @return StreamsTask for this component
- */
- public StreamsTask createConnectedTask() {
- StreamsTask task;
- if(this.processor != null) {
- task = new StreamsProcessorTask((StreamsProcessor)SerializationUtil.cloneBySerialization(this.processor));
- task.addInputQueue(this.inQueue);
- for(Queue<StreamsDatum> q : this.outBound.values()) {
- task.addOutputQueue(q);
- }
- }
- else if(this.writer != null) {
- if(this.numTasks > 1) {
- task = new StreamsPersistWriterTask((StreamsPersistWriter) SerializationUtil.cloneBySerialization(this.writer));
- task.addInputQueue(this.inQueue);
- } else {
- task = new StreamsPersistWriterTask(this.writer);
- task.addInputQueue(this.inQueue);
- }
- }
- else if(this.provider != null) {
- StreamsProvider prov;
- if(this.numTasks > 1) {
- prov = (StreamsProvider)SerializationUtil.cloneBySerialization(this.provider);
- } else {
- prov = this.provider;
- }
- if(this.dateRange == null && this.sequence == null)
- task = new StreamsProviderTask(prov, this.perpetual);
- else if(this.sequence != null)
- task = new StreamsProviderTask(prov, this.sequence);
- else
- task = new StreamsProviderTask(prov, this.dateRange[0], this.dateRange[1]);
- for(Queue<StreamsDatum> q : this.outBound.values()) {
- task.addOutputQueue(q);
- }
- }
- else {
- throw new InvalidStreamException("Underlying StreamComponoent was NULL.");
- }
- return task;
- }
-
- /**
- * The unique of this component
- * @return
- */
- public String getId() {
- return this.id;
- }
-
- @Override
- public int hashCode() {
- return this.id.hashCode();
- }
-
- @Override
- public boolean equals(Object o) {
- if(o instanceof StreamComponent)
- return this.id.equals(((StreamComponent) o).id);
- else
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/main/java/org/apache/streams/core/tasks/BaseStreamsTask.java
----------------------------------------------------------------------
diff --git a/streams-core/src/main/java/org/apache/streams/core/tasks/BaseStreamsTask.java b/streams-core/src/main/java/org/apache/streams/core/tasks/BaseStreamsTask.java
deleted file mode 100644
index e382607..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/tasks/BaseStreamsTask.java
+++ /dev/null
@@ -1,148 +0,0 @@
-package org.apache.streams.core.tasks;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.util.SerializationUtil;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Queue;
-
-/**
- *
- */
-public abstract class BaseStreamsTask implements StreamsTask {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(BaseStreamsTask.class);
-
- private List<Queue<StreamsDatum>> inQueues = new ArrayList<Queue<StreamsDatum>>();
- private List<Queue<StreamsDatum>> outQueues = new LinkedList<Queue<StreamsDatum>>();
- private int inIndex = 0;
- private ObjectMapper mapper;
-
- public BaseStreamsTask() {
- this.mapper = new ObjectMapper();
- this.mapper.registerSubtypes(Activity.class);
- }
-
-
- @Override
- public void addInputQueue(Queue<StreamsDatum> inputQueue) {
- this.inQueues.add(inputQueue);
- }
-
- @Override
- public void addOutputQueue(Queue<StreamsDatum> outputQueue) {
- this.outQueues.add(outputQueue);
- }
-
- @Override
- public List<Queue<StreamsDatum>> getInputQueues() {
- return this.inQueues;
- }
-
- @Override
- public List<Queue<StreamsDatum>> getOutputQueues() {
- return this.outQueues;
- }
-
- /**
- * NOTE NECESSARY AT THE MOMENT. MAY BECOME NECESSARY AS WE LOOK AT MAKING JOIN TASKS. CURRENTLY ALL TASK HAVE MAX
- * OF 1 INPUT QUEUE.
- * Round Robins through input queues to get the next StreamsDatum. If all input queues are empty, it will return null.
- * @return the next StreamsDatum or null if all input queues are empty.
- */
- protected StreamsDatum getNextDatum() {
- int startIndex = this.inIndex;
- int index = startIndex;
- StreamsDatum datum = null;
- do {
- datum = this.inQueues.get(index).poll();
- index = getNextInputQueueIndex();
- } while( datum == null && startIndex != index);
- return datum;
- }
-
- /**
- * Adds a StreamDatum to the outgoing queues. If there are multiple queues, it uses serialization to create
- * clones of the datum and adds a new clone to each queue.
- * @param datum
- */
- protected void addToOutgoingQueue(StreamsDatum datum) {
- if(this.outQueues.size() == 1) {
- this.outQueues.get(0).offer(datum);
- }
- else {
- StreamsDatum newDatum = null;
- for(Queue<StreamsDatum> queue : this.outQueues) {
- try {
- newDatum = cloneStreamsDatum(datum);
- if(newDatum != null)
- queue.offer(newDatum);
- } catch (RuntimeException e) {
- LOGGER.debug("Failed to add StreamsDatum to outgoing queue : {}", datum);
- LOGGER.error("Exception while offering StreamsDatum to outgoing queue: {}", e);
- }
- }
- }
- }
-
- /**
- * //TODO LOCAL MODE HACK. Need to fix
- * In order for our data streams to ported to other data flow frame works(Storm, Hadoop, Spark, etc) we need to be able to
- * enforce the serialization required by each framework. This needs some thought and design before a final solution is
- * made.
- *
- * In order to be able to copy/clone StreamDatums the orginal idea was to force all StreamsDatums to be java serializable.
- * This was seen as unacceptable for local mode. So until we come up with a solution to enforce serialization and be
- * compatiable across multiple frame works, this hack is in place.
- *
- * If datum.document is Serializable, we use serialization to clone a new copy. If it is not Serializable we attempt
- * different methods using an com.fasterxml.jackson.databind.ObjectMapper to copy/clone the StreamsDatum. If the object
- * is not clonable by these methods, an error is reported to the logging and a NULL object is returned.
- *
- * @param datum
- * @return
- */
- protected StreamsDatum cloneStreamsDatum(StreamsDatum datum) {
- try {
-
- if(datum.document instanceof ObjectNode) {
- return new StreamsDatum(((ObjectNode) datum.document).deepCopy(), datum.timestamp, datum.sequenceid);
- }
- else if(datum.document instanceof Activity) {
-
- return new StreamsDatum(this.mapper.readValue(this.mapper.writeValueAsString(datum.document), Activity.class),
- datum.timestamp,
- datum.sequenceid);
- }
-// else if(this.mapper.canSerialize(datum.document.getClass())){
-// return new StreamsDatum(this.mapper.readValue(this.mapper.writeValueAsString(datum.document), datum.document.getClass()),
-// datum.timestamp,
-// datum.sequenceid);
-// }
-
- else if(datum.document instanceof Serializable) {
- return (StreamsDatum) SerializationUtil.cloneBySerialization(datum);
- }
- } catch (Exception e) {
- LOGGER.error("Exception while trying to clone/copy StreamsDatum : {}", e);
- }
- LOGGER.error("Failed to clone/copy StreamsDatum with document of class : {}", datum.document.getClass().getName());
- return null;
- }
-
- private int getNextInputQueueIndex() {
- ++this.inIndex;
- if(this.inIndex >= this.inQueues.size()) {
- this.inIndex = 0;
- }
- return this.inIndex;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/main/java/org/apache/streams/core/tasks/LocalStreamMonitorThread.java
----------------------------------------------------------------------
diff --git a/streams-core/src/main/java/org/apache/streams/core/tasks/LocalStreamMonitorThread.java b/streams-core/src/main/java/org/apache/streams/core/tasks/LocalStreamMonitorThread.java
deleted file mode 100644
index 4c73e74..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/tasks/LocalStreamMonitorThread.java
+++ /dev/null
@@ -1,69 +0,0 @@
-package org.apache.streams.core.tasks;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.management.ManagementFactory;
-import java.lang.management.MemoryUsage;
-import java.util.concurrent.Executor;
-
-public class LocalStreamMonitorThread implements Runnable
-{
- private static final Logger LOGGER = LoggerFactory.getLogger(LocalStreamMonitorThread.class);
-
- private Executor executor;
-
- private int seconds;
-
- private boolean run = true;
-
- public LocalStreamMonitorThread(Executor executor, int delayInSeconds) {
- this.executor = executor;
- this.seconds = delayInSeconds;
- }
-
- public void shutdown(){
- this.run = false;
- }
-
- @Override
- public void run()
- {
- while(run){
-
- /**
- *
- * Note:
- * Quick class and method to let us see what is going on with the JVM. We need to make sure
- * that everything is running with as little memory as possible. If we are generating a heap
- * overflow, this will be very apparent by the information shown here.
- */
-
- MemoryUsage memoryUsage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
-
- String maxMemory = memoryUsage.getMax() == Long.MAX_VALUE ? "NO_LIMIT" :
- humanReadableByteCount(memoryUsage.getMax(), true);
-
- String usedMemory = humanReadableByteCount(memoryUsage.getUsed(), true);
-
- LOGGER.info("[monitor] Used Memory: {}, Max: {}",
- usedMemory,
- maxMemory);
-
- try
- {
- Thread.sleep(seconds*1000);
- }
- catch (InterruptedException e)
- { }
- }
- }
-
- public String humanReadableByteCount(long bytes, boolean si) {
- int unit = si ? 1000 : 1024;
- if (bytes < unit) return bytes + " B";
- int exp = (int) (Math.log(bytes) / Math.log(unit));
- String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp-1) + (si ? "" : "i");
- return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsMergeTask.java
----------------------------------------------------------------------
diff --git a/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsMergeTask.java b/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsMergeTask.java
deleted file mode 100644
index f3ad0cc..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsMergeTask.java
+++ /dev/null
@@ -1,58 +0,0 @@
-package org.apache.streams.core.tasks;
-
-import org.apache.streams.core.StreamsDatum;
-
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * NOT USED. When joins/partions are implemented, a similar pattern could be followed. Done only as basic proof
- * of concept.
- */
-public class StreamsMergeTask extends BaseStreamsTask {
-
- private AtomicBoolean keepRunning;
- private long sleepTime;
-
- public StreamsMergeTask() {
- this(DEFAULT_SLEEP_TIME_MS);
- }
-
- public StreamsMergeTask(long sleepTime) {
- this.sleepTime = sleepTime;
- this.keepRunning = new AtomicBoolean(true);
- }
-
-
- @Override
- public void stopTask() {
- this.keepRunning.set(false);
- }
-
- @Override
- public void setStreamConfig(Map<String, Object> config) {
-
- }
-
- @Override
- public boolean isRunning() {
- return false;
- }
-
- @Override
- public void run() {
- while(this.keepRunning.get()) {
- StreamsDatum datum = super.getNextDatum();
- if(datum != null) {
- super.addToOutgoingQueue(datum);
- }
- else {
- try {
- Thread.sleep(this.sleepTime);
- } catch (InterruptedException e) {
- this.keepRunning.set(false);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsPersistWriterTask.java
----------------------------------------------------------------------
diff --git a/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsPersistWriterTask.java b/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsPersistWriterTask.java
deleted file mode 100644
index 1a701a7..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsPersistWriterTask.java
+++ /dev/null
@@ -1,103 +0,0 @@
-package org.apache.streams.core.tasks;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- *
- */
-public class StreamsPersistWriterTask extends BaseStreamsTask {
-
-
-
- private StreamsPersistWriter writer;
- private long sleepTime;
- private AtomicBoolean keepRunning;
- private Map<String, Object> streamConfig;
- private Queue<StreamsDatum> inQueue;
- private AtomicBoolean isRunning;
-
- /**
- * Default constructor. Uses default sleep of 500ms when inbound queue is empty.
- * @param writer writer to execute in task
- */
- public StreamsPersistWriterTask(StreamsPersistWriter writer) {
- this(writer, DEFAULT_SLEEP_TIME_MS);
- }
-
- /**
- *
- * @param writer writer to execute in task
- * @param sleepTime time to sleep when inbound queue is empty.
- */
- public StreamsPersistWriterTask(StreamsPersistWriter writer, long sleepTime) {
- this.writer = writer;
- this.sleepTime = sleepTime;
- this.keepRunning = new AtomicBoolean(true);
- this.isRunning = new AtomicBoolean(true);
- }
-
- @Override
- public void setStreamConfig(Map<String, Object> config) {
- this.streamConfig = config;
- }
-
- @Override
- public void addInputQueue(Queue<StreamsDatum> inputQueue) {
- this.inQueue = inputQueue;
- }
-
- @Override
- public boolean isRunning() {
- return this.isRunning.get();
- }
-
- @Override
- public void run() {
- try {
- this.writer.prepare(this.streamConfig);
- StreamsDatum datum = this.inQueue.poll();
- while(datum != null || this.keepRunning.get()) {
- if(datum != null) {
- this.writer.write(datum);
- }
- else {
- try {
- Thread.sleep(this.sleepTime);
- } catch (InterruptedException e) {
- this.keepRunning.set(false);
- }
- }
- datum = this.inQueue.poll();
- }
-
- } finally {
- this.writer.cleanUp();
- this.isRunning.set(false);
- }
- }
-
- @Override
- public void stopTask() {
- this.keepRunning.set(false);
- }
-
-
- @Override
- public void addOutputQueue(Queue<StreamsDatum> outputQueue) {
- throw new UnsupportedOperationException(this.getClass().getName()+" does not support method - setOutputQueue()");
- }
-
- @Override
- public List<Queue<StreamsDatum>> getInputQueues() {
- List<Queue<StreamsDatum>> queues = new LinkedList<Queue<StreamsDatum>>();
- queues.add(this.inQueue);
- return queues;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsProcessorTask.java
----------------------------------------------------------------------
diff --git a/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsProcessorTask.java b/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsProcessorTask.java
deleted file mode 100644
index 30466a3..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsProcessorTask.java
+++ /dev/null
@@ -1,102 +0,0 @@
-package org.apache.streams.core.tasks;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.core.StreamsProcessor;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- *
- */
-public class StreamsProcessorTask extends BaseStreamsTask {
-
-
- private StreamsProcessor processor;
- private long sleepTime;
- private AtomicBoolean keepRunning;
- private Map<String, Object> streamConfig;
- private Queue<StreamsDatum> inQueue;
- private AtomicBoolean isRunning;
-
- /**
- * Default constructor, uses default sleep time of 500ms when inbound queue is empty
- * @param processor process to run in task
- */
- public StreamsProcessorTask(StreamsProcessor processor) {
- this(processor, DEFAULT_SLEEP_TIME_MS);
- }
-
- /**
- *
- * @param processor processor to run in task
- * @param sleepTime time to sleep when incoming queue is empty
- */
- public StreamsProcessorTask(StreamsProcessor processor, long sleepTime) {
- this.processor = processor;
- this.sleepTime = sleepTime;
- this.keepRunning = new AtomicBoolean(true);
- this.isRunning = new AtomicBoolean(true);
- }
-
- @Override
- public void stopTask() {
- this.keepRunning.set(false);
- }
-
- @Override
- public void setStreamConfig(Map<String, Object> config) {
- this.streamConfig = config;
- }
-
- @Override
- public void addInputQueue(Queue<StreamsDatum> inputQueue) {
- this.inQueue = inputQueue;
- }
-
- @Override
- public boolean isRunning() {
- return this.isRunning.get();
- }
-
- @Override
- public void run() {
- try {
- this.processor.prepare(this.streamConfig);
- StreamsDatum datum = this.inQueue.poll();
- while(datum != null || this.keepRunning.get()) {
- if(datum != null) {
- List<StreamsDatum> output = this.processor.process(datum);
- if(output != null) {
- for(StreamsDatum outDatum : output) {
- super.addToOutgoingQueue(outDatum);
- }
- }
- }
- else {
- try {
- Thread.sleep(this.sleepTime);
- } catch (InterruptedException e) {
- this.keepRunning.set(false);
- }
- }
- datum = this.inQueue.poll();
- }
-
- } finally {
- this.processor.cleanUp();
- this.isRunning.set(false);
- }
- }
-
- @Override
- public List<Queue<StreamsDatum>> getInputQueues() {
- List<Queue<StreamsDatum>> queues = new LinkedList<Queue<StreamsDatum>>();
- queues.add(this.inQueue);
- return queues;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsProviderTask.java
----------------------------------------------------------------------
diff --git a/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsProviderTask.java b/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsProviderTask.java
deleted file mode 100644
index 07235d1..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsProviderTask.java
+++ /dev/null
@@ -1,153 +0,0 @@
-package org.apache.streams.core.tasks;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.joda.time.DateTime;
-
-import java.math.BigInteger;
-import java.util.Map;
-import java.util.Queue;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- *
- */
-public class StreamsProviderTask extends BaseStreamsTask {
-
- private static enum Type {
- PERPETUAL,
- READ_CURRENT,
- READ_NEW,
- READ_RANGE
- }
-
- private static final int START = 0;
- private static final int END = 1;
-
- private StreamsProvider provider;
- private AtomicBoolean keepRunning;
- private Type type;
- private BigInteger sequence;
- private DateTime[] dateRange;
- private Map<String, Object> config;
- private AtomicBoolean isRunning;
-
- /**
- * Constructor for a StreamsProvider to execute {@link org.apache.streams.core.StreamsProvider:readCurrent()}
- * @param provider
- */
- public StreamsProviderTask(StreamsProvider provider, boolean perpetual) {
- this.provider = provider;
- if( perpetual )
- this.type = Type.PERPETUAL;
- else
- this.type = Type.READ_CURRENT;
- this.keepRunning = new AtomicBoolean(true);
- this.isRunning = new AtomicBoolean(true);
- }
-
- /**
- * Constructor for a StreamsProvider to execute {@link org.apache.streams.core.StreamsProvider:readNew(BigInteger)}
- * @param provider
- * @param sequence
- */
- public StreamsProviderTask(StreamsProvider provider, BigInteger sequence) {
- this.provider = provider;
- this.type = Type.READ_NEW;
- this.sequence = sequence;
- this.keepRunning = new AtomicBoolean(true);
- this.isRunning = new AtomicBoolean(true);
- }
-
- /**
- * Constructor for a StreamsProvider to execute {@link org.apache.streams.core.StreamsProvider:readRange(DateTime,DateTime)}
- * @param provider
- * @param start
- * @param end
- */
- public StreamsProviderTask(StreamsProvider provider, DateTime start, DateTime end) {
- this.provider = provider;
- this.type = Type.READ_RANGE;
- this.dateRange = new DateTime[2];
- this.dateRange[START] = start;
- this.dateRange[END] = end;
- this.keepRunning = new AtomicBoolean(true);
- this.isRunning = new AtomicBoolean(true);
- }
-
- @Override
- public void stopTask() {
- this.keepRunning.set(false);
- }
-
- @Override
- public void addInputQueue(Queue<StreamsDatum> inputQueue) {
- throw new UnsupportedOperationException(this.getClass().getName()+" does not support method - setInputQueue()");
- }
-
- @Override
- public void setStreamConfig(Map<String, Object> config) {
- this.config = config;
- }
-
- @Override
- public void run() {
- try {
- this.provider.prepare(this.config); //TODO allow for configuration objects
- StreamsResultSet resultSet = null;
- this.isRunning.set(true);
- switch(this.type) {
- case PERPETUAL: {
- provider.startStream();
- while(this.keepRunning.get() == true) {
- try {
- resultSet = provider.readCurrent();
- flushResults(resultSet);
- Thread.sleep(DEFAULT_SLEEP_TIME_MS);
- } catch (InterruptedException e) {
- this.keepRunning.set(false);
- }
- }
- }
- break;
- case READ_CURRENT: resultSet = this.provider.readCurrent();
- break;
- case READ_NEW: resultSet = this.provider.readNew(this.sequence);
- break;
- case READ_RANGE: resultSet = this.provider.readRange(this.dateRange[START], this.dateRange[END]);
- break;
- default: throw new RuntimeException("Type has not been added to StreamsProviderTask.");
- }
- flushResults(resultSet);
-
- } catch( Exception e ) {
- e.printStackTrace();
- } finally
- {
- this.provider.cleanUp();
- this.isRunning.set(false);
- }
- }
-
- public boolean isRunning() {
- return this.isRunning.get();
- }
-
- public void flushResults(StreamsResultSet resultSet) {
- for(StreamsDatum datum : resultSet) {
- if(!this.keepRunning.get()) {
- break;
- }
- if(datum != null)
- super.addToOutgoingQueue(datum);
- else {
- try {
- Thread.sleep(DEFAULT_SLEEP_TIME_MS);
- } catch (InterruptedException e) {
- this.keepRunning.set(false);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsTask.java
----------------------------------------------------------------------
diff --git a/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsTask.java b/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsTask.java
deleted file mode 100644
index a6cfbb9..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/tasks/StreamsTask.java
+++ /dev/null
@@ -1,58 +0,0 @@
-package org.apache.streams.core.tasks;
-
-import org.apache.streams.core.StreamsDatum;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-
-/**
- * Interface for all task that will be used to execute instances of {@link org.apache.streams.core.StreamsOperation}
- * in local mode.
- */
-public interface StreamsTask extends Runnable{
-
- public static final long DEFAULT_SLEEP_TIME_MS = 5000;
-
- /**
- * Informs the task to stop. Tasks may or may not try to empty its inbound queue before halting.
- */
- public void stopTask();
-
- /**
- * Add an input {@link java.util.Queue} for this task.
- * @param inputQueue
- */
- public void addInputQueue(Queue<StreamsDatum> inputQueue);
-
- /**
- * Add an output {@link java.util.Queue} for this task.
- * @param outputQueue
- */
- public void addOutputQueue(Queue<StreamsDatum> outputQueue);
-
- /**
- * Set the configuration object that will shared and passed to all instances of StreamsTask.
- * @param config optional configuration information
- */
- public void setStreamConfig(Map<String, Object> config);
-
- /**
- * Returns true when the task has not completed. Returns false otherwise
- * @return true when the task has not completed. Returns false otherwise
- */
- public boolean isRunning();
-
- /**
- * Returns the input queues that have been set for this task.
- * @return list of input queues
- */
- public List<Queue<StreamsDatum>> getInputQueues();
-
- /**
- * Returns the output queues that have been set for this task
- * @return list of output queues
- */
- public List<Queue<StreamsDatum>> getOutputQueues();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/test/java/org/apache/streams/core/builders/LocalStreamBuilderTest.java
----------------------------------------------------------------------
diff --git a/streams-core/src/test/java/org/apache/streams/core/builders/LocalStreamBuilderTest.java b/streams-core/src/test/java/org/apache/streams/core/builders/LocalStreamBuilderTest.java
deleted file mode 100644
index 9177459..0000000
--- a/streams-core/src/test/java/org/apache/streams/core/builders/LocalStreamBuilderTest.java
+++ /dev/null
@@ -1,161 +0,0 @@
-package org.apache.streams.core.builders;
-
-import static org.junit.Assert.*;
-import static org.junit.Assert.assertEquals;
-
-import org.apache.streams.core.test.processors.PassthroughDatumCounterProcessor;
-import org.apache.streams.core.test.providers.NumericMessageProvider;
-import org.apache.streams.core.test.writer.DatumCounterWriter;
-import org.apache.streams.core.test.writer.SystemOutWriter;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.*;
-import java.util.HashSet;
-import java.util.Scanner;
-
-/**
- * Basic Tests for the LocalStreamBuilder.
- *
- * Test are performed by redirecting system out and counting the number of lines that the SystemOutWriter prints
- * to System.out. The SystemOutWriter also prints one line when cleanUp() is called, so this is why it tests for
- * the numDatums +1.
- *
- *
- */
-public class LocalStreamBuilderTest {
-
- ByteArrayOutputStream out;
-
- @Before
- public void setSystemOut() {
- out = new ByteArrayOutputStream();
- System.setOut(new PrintStream(out));
- }
-
- @Test
- public void testStreamIdValidations() {
- StreamBuilder builder = new LocalStreamBuilder();
- builder.newReadCurrentStream("id", new NumericMessageProvider(1));
- Exception exp = null;
- try {
- builder.newReadCurrentStream("id", new NumericMessageProvider(1));
- } catch (RuntimeException e) {
- exp = e;
- }
- assertNotNull(exp);
- exp = null;
- builder.addStreamsProcessor("1", new PassthroughDatumCounterProcessor(), 1, "id");
- try {
- builder.addStreamsProcessor("2", new PassthroughDatumCounterProcessor(), 1, "id", "id2");
- } catch (RuntimeException e) {
- exp = e;
- }
- assertNotNull(exp);
- }
-
- @Test
- public void testBasicLinearStream1() {
- int numDatums = 1;
- StreamBuilder builder = new LocalStreamBuilder();
- PassthroughDatumCounterProcessor processor = new PassthroughDatumCounterProcessor();
- SystemOutWriter writer = new SystemOutWriter();
- builder.newReadCurrentStream("sp1", new NumericMessageProvider(numDatums))
- .addStreamsProcessor("proc1", processor, 1, "sp1")
- .addStreamsPersistWriter("writer1", writer, 1, "proc1");
- builder.start();
- int count = 0;
- Scanner scanner = new Scanner(new ByteArrayInputStream(out.toByteArray()));
- while(scanner.hasNextLine()) {
- ++count;
- scanner.nextLine();
- }
- assertEquals(numDatums+1, count);
- }
-
- @Test
- public void testBasicLinearStream2() {
- int numDatums = 100;
- StreamBuilder builder = new LocalStreamBuilder();
- PassthroughDatumCounterProcessor processor = new PassthroughDatumCounterProcessor();
- SystemOutWriter writer = new SystemOutWriter();
- builder.newReadCurrentStream("sp1", new NumericMessageProvider(numDatums))
- .addStreamsProcessor("proc1", processor, 1, "sp1")
- .addStreamsPersistWriter("writer1", writer, 1, "proc1");
- builder.start();
- int count = 0;
- Scanner scanner = new Scanner(new ByteArrayInputStream(out.toByteArray()));
- while(scanner.hasNextLine()) {
- ++count;
- scanner.nextLine();
- }
- assertEquals(numDatums+1, count);
- }
-
- @Test
- public void testParallelLinearStream1() {
- int numDatums = 1000;
- int parallelHint = 20;
- PassthroughDatumCounterProcessor.sawData = new HashSet<Integer>();
- PassthroughDatumCounterProcessor.claimedNumber = new HashSet<Integer>();
- StreamBuilder builder = new LocalStreamBuilder();
- PassthroughDatumCounterProcessor processor = new PassthroughDatumCounterProcessor();
- SystemOutWriter writer = new SystemOutWriter();
- builder.newReadCurrentStream("sp1", new NumericMessageProvider(numDatums))
- .addStreamsProcessor("proc1", processor, parallelHint, "sp1")
- .addStreamsPersistWriter("writer1", writer, 1, "proc1");
- builder.start();
- int count = 0;
- Scanner scanner = new Scanner(new ByteArrayInputStream(out.toByteArray()));
- while(scanner.hasNextLine()) {
- ++count;
- scanner.nextLine();
- }
- assertEquals(numDatums+1, count); //+1 is to make sure cleanup is called on the writer
- assertEquals(parallelHint, PassthroughDatumCounterProcessor.claimedNumber.size()); //test 40 were initialized
- assertTrue(PassthroughDatumCounterProcessor.sawData.size() > 1 && PassthroughDatumCounterProcessor.sawData.size() <= parallelHint); //test more than one processor got data
- }
-
- @Test
- public void testBasicMergeStream() {
- int numDatums1 = 1;
- int numDatums2 = 100;
- PassthroughDatumCounterProcessor processor1 = new PassthroughDatumCounterProcessor();
- PassthroughDatumCounterProcessor processor2 = new PassthroughDatumCounterProcessor();
- SystemOutWriter writer = new SystemOutWriter();
- StreamBuilder builder = new LocalStreamBuilder();
- builder.newReadCurrentStream("sp1", new NumericMessageProvider(numDatums1))
- .newReadCurrentStream("sp2", new NumericMessageProvider(numDatums2))
- .addStreamsProcessor("proc1", processor1, 1, "sp1")
- .addStreamsProcessor("proc2", processor2, 1, "sp2")
- .addStreamsPersistWriter("writer1", writer, 1, "proc1", "proc2");
- builder.start();
- int count = 0;
- Scanner scanner = new Scanner(new ByteArrayInputStream(out.toByteArray()));
- while(scanner.hasNextLine()) {
- ++count;
- scanner.nextLine();
- }
- assertEquals(numDatums1+numDatums2+1, count);
- }
-
- @Test
- public void testBasicBranch() {
- int numDatums = 100;
- StreamBuilder builder = new LocalStreamBuilder();
- builder.newReadCurrentStream("prov1", new NumericMessageProvider(numDatums))
- .addStreamsProcessor("proc1", new PassthroughDatumCounterProcessor(), 1, "prov1")
- .addStreamsProcessor("proc2", new PassthroughDatumCounterProcessor(), 1, "prov1")
- .addStreamsPersistWriter("w1", new SystemOutWriter(), 1, "proc1", "proc2");
- builder.start();
- int count = 0;
- Scanner scanner = new Scanner(new ByteArrayInputStream(out.toByteArray()));
- while(scanner.hasNextLine()) {
- ++count;
- scanner.nextLine();
- }
- assertEquals((numDatums*2)+1, count);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/test/java/org/apache/streams/core/builders/ToyLocalBuilderExample.java
----------------------------------------------------------------------
diff --git a/streams-core/src/test/java/org/apache/streams/core/builders/ToyLocalBuilderExample.java b/streams-core/src/test/java/org/apache/streams/core/builders/ToyLocalBuilderExample.java
deleted file mode 100644
index be92c89..0000000
--- a/streams-core/src/test/java/org/apache/streams/core/builders/ToyLocalBuilderExample.java
+++ /dev/null
@@ -1,28 +0,0 @@
-package org.apache.streams.core.builders;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.test.processors.DoNothingProcessor;
-import org.apache.streams.core.test.providers.NumericMessageProvider;
-import org.apache.streams.core.test.writer.DoNothingWriter;
-import org.apache.streams.core.test.writer.SystemOutWriter;
-
-import java.util.concurrent.LinkedBlockingQueue;
-
-/**
- * Created by rebanks on 2/20/14.
- */
-public class ToyLocalBuilderExample {
-
- /**
- * A simple example of how to run a stream in local mode.
- * @param args
- */
- public static void main(String[] args) {
- StreamBuilder builder = new LocalStreamBuilder(new LinkedBlockingQueue<StreamsDatum>());
- builder.newReadCurrentStream("prov", new NumericMessageProvider(1000000))
- .addStreamsProcessor("proc", new DoNothingProcessor(), 100, "prov")
- .addStreamsPersistWriter("writer", new DoNothingWriter(), 3, "proc");
- builder.start();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/test/java/org/apache/streams/core/tasks/BasicTasksTest.java
----------------------------------------------------------------------
diff --git a/streams-core/src/test/java/org/apache/streams/core/tasks/BasicTasksTest.java b/streams-core/src/test/java/org/apache/streams/core/tasks/BasicTasksTest.java
deleted file mode 100644
index ef81186..0000000
--- a/streams-core/src/test/java/org/apache/streams/core/tasks/BasicTasksTest.java
+++ /dev/null
@@ -1,285 +0,0 @@
-package org.apache.streams.core.tasks;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.test.processors.PassthroughDatumCounterProcessor;
-import org.apache.streams.core.test.providers.NumericMessageProvider;
-import static org.junit.Assert.*;
-
-import org.apache.streams.core.test.writer.DatumCounterWriter;
-import org.junit.Test;
-
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Created by rebanks on 2/18/14.
- */
-public class BasicTasksTest {
-
-
-
- @Test
- public void testProviderTask() {
- int numMessages = 100;
- NumericMessageProvider provider = new NumericMessageProvider(numMessages);
- StreamsProviderTask task = new StreamsProviderTask(provider, false);
- Queue<StreamsDatum> outQueue = new ConcurrentLinkedQueue<StreamsDatum>();
- task.addOutputQueue(outQueue);
- Queue<StreamsDatum> inQueue = createInputQueue(numMessages);
- Exception exp = null;
- try {
- task.addInputQueue(inQueue);
- } catch (UnsupportedOperationException uoe) {
- exp = uoe;
- }
- assertNotNull(exp);
- ExecutorService service = Executors.newFixedThreadPool(1);
- service.submit(task);
- int attempts = 0;
- while(outQueue.size() != numMessages) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- //Ignore
- }
- ++attempts;
- if(attempts == 10) {
- fail("Provider task failed to output "+numMessages+" in a timely fashion.");
- }
- }
- service.shutdown();
- try {
- if(!service.awaitTermination(5, TimeUnit.SECONDS)){
- service.shutdownNow();
- fail("Service did not terminate.");
- }
- assertTrue("Task should have completed running in aloted time.", service.isTerminated());
- } catch (InterruptedException e) {
- fail("Test Interupted.");
- };
- }
-
- @Test
- public void testProcessorTask() {
- int numMessages = 100;
- PassthroughDatumCounterProcessor processor = new PassthroughDatumCounterProcessor();
- StreamsProcessorTask task = new StreamsProcessorTask(processor);
- Queue<StreamsDatum> outQueue = new ConcurrentLinkedQueue<StreamsDatum>();
- Queue<StreamsDatum> inQueue = createInputQueue(numMessages);
- task.addOutputQueue(outQueue);
- task.addInputQueue(inQueue);
- assertEquals(numMessages, task.getInputQueues().get(0).size());
- ExecutorService service = Executors.newFixedThreadPool(1);
- service.submit(task);
- int attempts = 0;
- while(inQueue.size() != 0 && outQueue.size() != numMessages) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- //Ignore
- }
- ++attempts;
- if(attempts == 10) {
- fail("Processor task failed to output "+numMessages+" in a timely fashion.");
- }
- }
- task.stopTask();
- assertEquals(numMessages, processor.getMessageCount());
- service.shutdown();
- try {
- if(!service.awaitTermination(5, TimeUnit.SECONDS)){
- service.shutdownNow();
- fail("Service did not terminate.");
- }
- assertTrue("Task should have completed running in aloted time.", service.isTerminated());
- } catch (InterruptedException e) {
- fail("Test Interupted.");
- }
- }
-
- @Test
- public void testWriterTask() {
- int numMessages = 100;
- DatumCounterWriter writer = new DatumCounterWriter();
- StreamsPersistWriterTask task = new StreamsPersistWriterTask(writer);
- Queue<StreamsDatum> outQueue = new ConcurrentLinkedQueue<StreamsDatum>();
- Queue<StreamsDatum> inQueue = createInputQueue(numMessages);
-
- Exception exp = null;
- try {
- task.addOutputQueue(outQueue);
- } catch (UnsupportedOperationException uoe) {
- exp = uoe;
- }
- assertNotNull(exp);
- task.addInputQueue(inQueue);
- assertEquals(numMessages, task.getInputQueues().get(0).size());
- ExecutorService service = Executors.newFixedThreadPool(1);
- service.submit(task);
- int attempts = 0;
- while(inQueue.size() != 0 ) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- //Ignore
- }
- ++attempts;
- if(attempts == 10) {
- fail("Processor task failed to output "+numMessages+" in a timely fashion.");
- }
- }
- task.stopTask();
- assertEquals(numMessages, writer.getDatumsCounted());
- service.shutdown();
- try {
- if(!service.awaitTermination(5, TimeUnit.SECONDS)){
- service.shutdownNow();
- fail("Service did not terminate.");
- }
- assertTrue("Task should have completed running in aloted time.", service.isTerminated());
- } catch (InterruptedException e) {
- fail("Test Interupted.");
- }
- }
-
- @Test
- public void testMergeTask() {
- int numMessages = 100;
- int incoming = 5;
- StreamsMergeTask task = new StreamsMergeTask();
- Queue<StreamsDatum> outQueue = new ConcurrentLinkedQueue<StreamsDatum>();
- task.addOutputQueue(outQueue);
- for(int i=0; i < incoming; ++i) {
- task.addInputQueue(createInputQueue(numMessages));
- }
- ExecutorService service = Executors.newFixedThreadPool(1);
- service.submit(task);
- int attempts = 0;
- while(outQueue.size() != incoming * numMessages ) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- //Ignore
- }
- ++attempts;
- if(attempts == 10) {
- assertEquals("Processor task failed to output " + (numMessages * incoming) + " in a timely fashion.", (numMessages * incoming), outQueue.size());
- }
- }
- task.stopTask();
- service.shutdown();
- try {
- if(!service.awaitTermination(5, TimeUnit.SECONDS)){
- service.shutdownNow();
- fail("Service did not terminate.");
- }
- assertTrue("Task should have completed running in aloted time.", service.isTerminated());
- } catch (InterruptedException e) {
- fail("Test Interupted.");
- }
- }
-
- @Test
- public void testBranching() {
- int numMessages = 100;
- PassthroughDatumCounterProcessor processor = new PassthroughDatumCounterProcessor();
- StreamsProcessorTask task = new StreamsProcessorTask(processor);
- Queue<StreamsDatum> outQueue1 = new ConcurrentLinkedQueue<StreamsDatum>();
- Queue<StreamsDatum> outQueue2 = new ConcurrentLinkedQueue<StreamsDatum>();
- Queue<StreamsDatum> inQueue = createInputQueue(numMessages);
- task.addOutputQueue(outQueue1);
- task.addOutputQueue(outQueue2);
- task.addInputQueue(inQueue);
- assertEquals(numMessages, task.getInputQueues().get(0).size());
- ExecutorService service = Executors.newFixedThreadPool(1);
- service.submit(task);
- int attempts = 0;
- while(inQueue.size() != 0 ) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- //Ignore
- }
- ++attempts;
- if(attempts == 10) {
- assertEquals("Processor task failed to output "+(numMessages)+" in a timely fashion.", 0, inQueue.size());
- }
- }
- task.stopTask();
-
- service.shutdown();
- try {
- if(!service.awaitTermination(5, TimeUnit.SECONDS)){
- service.shutdownNow();
- fail("Service did not terminate.");
- }
- assertTrue("Task should have completed running in aloted time.", service.isTerminated());
- } catch (InterruptedException e) {
- fail("Test Interupted.");
- }
- assertEquals(numMessages, processor.getMessageCount());
- assertEquals(numMessages, outQueue1.size());
- assertEquals(numMessages, outQueue2.size());
- }
-
- @Test
- public void testBranchingSerialization() {
- int numMessages = 1;
- PassthroughDatumCounterProcessor processor = new PassthroughDatumCounterProcessor();
- StreamsProcessorTask task = new StreamsProcessorTask(processor);
- Queue<StreamsDatum> outQueue1 = new ConcurrentLinkedQueue<StreamsDatum>();
- Queue<StreamsDatum> outQueue2 = new ConcurrentLinkedQueue<StreamsDatum>();
- Queue<StreamsDatum> inQueue = createInputQueue(numMessages);
- task.addOutputQueue(outQueue1);
- task.addOutputQueue(outQueue2);
- task.addInputQueue(inQueue);
- ExecutorService service = Executors.newFixedThreadPool(1);
- service.submit(task);
- int attempts = 0;
- while(inQueue.size() != 0 ) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- //Ignore
- }
- ++attempts;
- if(attempts == 10) {
- assertEquals("Processor task failed to output "+(numMessages)+" in a timely fashion.", 0, inQueue.size());
- }
- }
- task.stopTask();
-
- service.shutdown();
- try {
- if(!service.awaitTermination(5, TimeUnit.SECONDS)){
- service.shutdownNow();
- fail("Service did not terminate.");
- }
- assertTrue("Task should have completed running in aloted time.", service.isTerminated());
- } catch (InterruptedException e) {
- fail("Test Interupted.");
- }
- assertEquals(numMessages, processor.getMessageCount());
- assertEquals(numMessages, outQueue1.size());
- assertEquals(numMessages, outQueue2.size());
- StreamsDatum datum1 = outQueue1.poll();
- StreamsDatum datum2 = outQueue2.poll();
- assertNotNull(datum1);
- assertEquals(datum1, datum2);
- datum1.setDocument("a");
- assertNotEquals(datum1, datum2);
- }
-
- private Queue<StreamsDatum> createInputQueue(int numDatums) {
- Queue<StreamsDatum> queue = new ConcurrentLinkedQueue<StreamsDatum>();
- for(int i=0; i < numDatums; ++i) {
- queue.add(new StreamsDatum(i));
- }
- return queue;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/test/java/org/apache/streams/core/test/processors/DoNothingProcessor.java
----------------------------------------------------------------------
diff --git a/streams-core/src/test/java/org/apache/streams/core/test/processors/DoNothingProcessor.java b/streams-core/src/test/java/org/apache/streams/core/test/processors/DoNothingProcessor.java
deleted file mode 100644
index 25c717b..0000000
--- a/streams-core/src/test/java/org/apache/streams/core/test/processors/DoNothingProcessor.java
+++ /dev/null
@@ -1,32 +0,0 @@
-package org.apache.streams.core.test.processors;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * Created by rebanks on 2/20/14.
- */
-public class DoNothingProcessor implements StreamsProcessor {
-
- List<StreamsDatum> result;
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- this.result = new LinkedList<StreamsDatum>();
- result.add(entry);
- return result;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- }
-
- @Override
- public void cleanUp() {
- System.out.println("Processor clean up!");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/test/java/org/apache/streams/core/test/processors/PassthroughDatumCounterProcessor.java
----------------------------------------------------------------------
diff --git a/streams-core/src/test/java/org/apache/streams/core/test/processors/PassthroughDatumCounterProcessor.java b/streams-core/src/test/java/org/apache/streams/core/test/processors/PassthroughDatumCounterProcessor.java
deleted file mode 100644
index 1010370..0000000
--- a/streams-core/src/test/java/org/apache/streams/core/test/processors/PassthroughDatumCounterProcessor.java
+++ /dev/null
@@ -1,49 +0,0 @@
-package org.apache.streams.core.test.processors;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-
-import java.util.*;
-
-/**
- * Created by rebanks on 2/18/14.
- */
-public class PassthroughDatumCounterProcessor implements StreamsProcessor {
-
- public static Set<Integer> claimedNumber = new HashSet<Integer>();
- public static final Random rand = new Random();
- public static Set<Integer> sawData = new HashSet<Integer>();
-
- private int count = 0;
- private int id;
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- ++this.count;
- List<StreamsDatum> result = new LinkedList<StreamsDatum>();
- result.add(entry);
- synchronized (sawData) {
- sawData.add(this.id);
- }
- return result;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- synchronized (claimedNumber) {
- this.id = rand.nextInt();
- while(!claimedNumber.add(this.id)) {
- this.id = rand.nextInt();
- }
- }
- }
-
- @Override
- public void cleanUp() {
-
- }
-
- public int getMessageCount() {
- return this.count;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/test/java/org/apache/streams/core/test/providers/NumericMessageProvider.java
----------------------------------------------------------------------
diff --git a/streams-core/src/test/java/org/apache/streams/core/test/providers/NumericMessageProvider.java b/streams-core/src/test/java/org/apache/streams/core/test/providers/NumericMessageProvider.java
deleted file mode 100644
index 01b9a25..0000000
--- a/streams-core/src/test/java/org/apache/streams/core/test/providers/NumericMessageProvider.java
+++ /dev/null
@@ -1,104 +0,0 @@
-package org.apache.streams.core.test.providers;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.joda.time.DateTime;
-
-import java.math.BigInteger;
-import java.util.Iterator;
-import java.util.Queue;
-import java.util.Scanner;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-/**
- * Test StreamsProvider that sends out StreamsDatums numbered from 0 to numMessages.
- */
-public class NumericMessageProvider implements StreamsProvider {
-
- private int numMessages;
-
- public NumericMessageProvider(int numMessages) {
- this.numMessages = numMessages;
- }
-
- @Override
- public void startStream() {
- // no op
- }
-
- @Override
- public StreamsResultSet readCurrent() {
- return new ResultSet();
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return new ResultSet();
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return new ResultSet();
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- }
-
- @Override
- public void cleanUp() {
-
- }
-
-
- private class ResultSet extends StreamsResultSet {
-
- private ResultSet() {
- super(new ConcurrentLinkedQueue<StreamsDatum>());
- }
-
-// @Override
-// public long getStartTime() {
-// return 0;
-// }
-//
-// @Override
-// public long getEndTime() {
-// return 0;
-// }
-//
-// @Override
-// public String getSourceId() {
-// return null;
-// }
-//
-// @Override
-// public BigInteger getMaxSequence() {
-// return null;
-// }
-
- @Override
- public Iterator<StreamsDatum> iterator() {
- return new Iterator<StreamsDatum>() {
- private int i = 0;
-
- @Override
- public boolean hasNext() {
- return i < numMessages;
- }
-
- @Override
- public StreamsDatum next() {
- return new StreamsDatum(i++);
- }
-
- @Override
- public void remove() {
-
- }
- };
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/test/java/org/apache/streams/core/test/writer/DatumCounterWriter.java
----------------------------------------------------------------------
diff --git a/streams-core/src/test/java/org/apache/streams/core/test/writer/DatumCounterWriter.java b/streams-core/src/test/java/org/apache/streams/core/test/writer/DatumCounterWriter.java
deleted file mode 100644
index d44d24c..0000000
--- a/streams-core/src/test/java/org/apache/streams/core/test/writer/DatumCounterWriter.java
+++ /dev/null
@@ -1,31 +0,0 @@
-package org.apache.streams.core.test.writer;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-
-/**
- * Created by rebanks on 2/18/14.
- */
-public class DatumCounterWriter implements StreamsPersistWriter{
-
- private int counter = 0;
-
- @Override
- public void write(StreamsDatum entry) {
- ++this.counter;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- }
-
- @Override
- public void cleanUp() {
- System.out.println("clean up called");
- }
-
- public int getDatumsCounted() {
- return this.counter;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/test/java/org/apache/streams/core/test/writer/DoNothingWriter.java
----------------------------------------------------------------------
diff --git a/streams-core/src/test/java/org/apache/streams/core/test/writer/DoNothingWriter.java b/streams-core/src/test/java/org/apache/streams/core/test/writer/DoNothingWriter.java
deleted file mode 100644
index a52347e..0000000
--- a/streams-core/src/test/java/org/apache/streams/core/test/writer/DoNothingWriter.java
+++ /dev/null
@@ -1,24 +0,0 @@
-package org.apache.streams.core.test.writer;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-
-/**
- * Created by rebanks on 2/20/14.
- */
-public class DoNothingWriter implements StreamsPersistWriter {
- @Override
- public void write(StreamsDatum entry) {
-
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- }
-
- @Override
- public void cleanUp() {
- System.out.println("Writer Clean Up!");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-core/src/test/java/org/apache/streams/core/test/writer/SystemOutWriter.java
----------------------------------------------------------------------
diff --git a/streams-core/src/test/java/org/apache/streams/core/test/writer/SystemOutWriter.java b/streams-core/src/test/java/org/apache/streams/core/test/writer/SystemOutWriter.java
deleted file mode 100644
index 1856e34..0000000
--- a/streams-core/src/test/java/org/apache/streams/core/test/writer/SystemOutWriter.java
+++ /dev/null
@@ -1,24 +0,0 @@
-package org.apache.streams.core.test.writer;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-
-/**
- * Created by rebanks on 2/20/14.
- */
-public class SystemOutWriter implements StreamsPersistWriter {
- @Override
- public void write(StreamsDatum entry) {
- System.out.println(entry.document);
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- }
-
- @Override
- public void cleanUp() {
- System.out.println("Clean up called writer!");
- }
-}
[02/13] adding some uncommitted modules reorganizing pom improvements
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/InvalidStreamException.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/InvalidStreamException.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/InvalidStreamException.java
new file mode 100644
index 0000000..5e00392
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/InvalidStreamException.java
@@ -0,0 +1,23 @@
+package org.apache.streams.local.builders;
+
+/**
+ * Exception that indicates a malformed data stream in some way.
+ */
+public class InvalidStreamException extends RuntimeException {
+
+ public InvalidStreamException() {
+ super();
+ }
+
+ public InvalidStreamException(String s) {
+ super(s);
+ }
+
+ public InvalidStreamException(String s, Throwable throwable) {
+ super(s, throwable);
+ }
+
+ public InvalidStreamException(Throwable throwable) {
+ super(throwable);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
new file mode 100644
index 0000000..444c2e1
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
@@ -0,0 +1,276 @@
+package org.apache.streams.local.builders;
+
+import org.apache.streams.core.*;
+import org.apache.streams.local.tasks.LocalStreamProcessMonitorThread;
+import org.apache.streams.local.tasks.StreamsProviderTask;
+import org.apache.streams.local.tasks.StreamsTask;
+import org.apache.streams.util.SerializationUtil;
+import org.joda.time.DateTime;
+
+import java.math.BigInteger;
+import java.util.*;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * {@link org.apache.streams.local.builders.LocalStreamBuilder} implementation to run a data processing stream in a single
+ * JVM across many threads. Depending on your data stream, the JVM heap may need to be set to a high value. Default
+ * implementation uses unbound {@link java.util.concurrent.ConcurrentLinkedQueue} to connect stream components.
+ */
+public class LocalStreamBuilder implements StreamBuilder {
+
+ private Map<String, StreamComponent> providers;
+ private Map<String, StreamComponent> components;
+ private Queue<StreamsDatum> queue;
+ private Map<String, Object> streamConfig;
+ private ExecutorService executor;
+ private ExecutorService monitor;
+ private int totalTasks;
+ private LocalStreamProcessMonitorThread monitorThread;
+
+ /**
+ *
+ */
+ public LocalStreamBuilder(){
+ this(new ConcurrentLinkedQueue<StreamsDatum>(), null);
+ }
+
+ /**
+ *
+ * @param streamConfig
+ */
+ public LocalStreamBuilder(Map<String, Object> streamConfig) {
+ this(new ConcurrentLinkedQueue<StreamsDatum>(), streamConfig);
+ }
+
+ /**
+ *
+ * @param queueType
+ */
+ public LocalStreamBuilder(Queue<StreamsDatum> queueType) {
+ this(queueType, null);
+ }
+
+ /**
+ *
+ * @param queueType
+ * @param streamConfig
+ */
+ public LocalStreamBuilder(Queue<StreamsDatum> queueType, Map<String, Object> streamConfig) {
+ this.queue = queueType;
+ this.providers = new HashMap<String, StreamComponent>();
+ this.components = new HashMap<String, StreamComponent>();
+ this.streamConfig = streamConfig;
+ this.totalTasks = 0;
+ }
+
+ @Override
+ public StreamBuilder newPerpetualStream(String id, StreamsProvider provider) {
+ validateId(id);
+ this.providers.put(id, new StreamComponent(id, provider, true));
+ ++this.totalTasks;
+ return this;
+ }
+
+ @Override
+ public StreamBuilder newReadCurrentStream(String id, StreamsProvider provider) {
+ validateId(id);
+ this.providers.put(id, new StreamComponent(id, provider, false));
+ ++this.totalTasks;
+ return this;
+ }
+
+ @Override
+ public StreamBuilder newReadNewStream(String id, StreamsProvider provider, BigInteger sequence) {
+ validateId(id);
+ this.providers.put(id, new StreamComponent(id, provider, sequence));
+ ++this.totalTasks;
+ return this;
+ }
+
+ @Override
+ public StreamBuilder newReadRangeStream(String id, StreamsProvider provider, DateTime start, DateTime end) {
+ validateId(id);
+ this.providers.put(id, new StreamComponent(id, provider, start, end));
+ ++this.totalTasks;
+ return this;
+ }
+
+ @Override
+ public StreamBuilder addStreamsProcessor(String id, StreamsProcessor processor, int numTasks, String... inBoundIds) {
+ validateId(id);
+ StreamComponent comp = new StreamComponent(id, processor, cloneQueue(), numTasks);
+ this.components.put(id, comp);
+ connectToOtherComponents(inBoundIds, comp);
+ this.totalTasks += numTasks;
+ return this;
+ }
+
+ @Override
+ public StreamBuilder addStreamsPersistWriter(String id, StreamsPersistWriter writer, int numTasks, String... inBoundIds) {
+ validateId(id);
+ StreamComponent comp = new StreamComponent(id, writer, cloneQueue(), numTasks);
+ this.components.put(id, comp);
+ connectToOtherComponents(inBoundIds, comp);
+ this.totalTasks += numTasks;
+ return this;
+ }
+
+ /**
+ * Runs the data stream in the this JVM and blocks till completion.
+ */
+ @Override
+ public void start() {
+ boolean isRunning = true;
+ this.executor = Executors.newFixedThreadPool(this.totalTasks);
+ this.monitor = Executors.newSingleThreadExecutor();
+ Map<String, StreamsProviderTask> provTasks = new HashMap<String, StreamsProviderTask>();
+ Map<String, List<StreamsTask>> streamsTasks = new HashMap<String, List<StreamsTask>>();
+ monitorThread = new LocalStreamProcessMonitorThread(this.monitor, 1000);
+ try {
+ this.monitor.submit(monitorThread);
+ for(StreamComponent comp : this.components.values()) {
+ int tasks = comp.getNumTasks();
+ List<StreamsTask> compTasks = new LinkedList<StreamsTask>();
+ for(int i=0; i < tasks; ++i) {
+ StreamsTask task = comp.createConnectedTask();
+ task.setStreamConfig(this.streamConfig);
+ this.executor.submit(task);
+ compTasks.add(task);
+ }
+ streamsTasks.put(comp.getId(), compTasks);
+ }
+ for(StreamComponent prov : this.providers.values()) {
+ StreamsTask task = prov.createConnectedTask();
+ task.setStreamConfig(this.streamConfig);
+ this.executor.submit(task);
+ provTasks.put(prov.getId(), (StreamsProviderTask) task);
+ }
+ while(isRunning) {
+ isRunning = false;
+ for(StreamsProviderTask task : provTasks.values()) {
+ isRunning = isRunning || task.isRunning();
+ }
+ if(isRunning) {
+ Thread.sleep(3000);
+ }
+ }
+ monitorThread.shutdown();
+ this.executor.shutdown();
+ //complete stream shut down gracfully
+ for(StreamComponent prov : this.providers.values()) {
+ shutDownTask(prov, streamsTasks);
+ }
+ //need to make this configurable
+ if(!this.executor.awaitTermination(10, TimeUnit.SECONDS)) { // all threads should have terminated already.
+ this.executor.shutdownNow();
+ this.executor.awaitTermination(10, TimeUnit.SECONDS);
+ }
+ if(!this.monitor.awaitTermination(5, TimeUnit.SECONDS)) { // all threads should have terminated already.
+ this.monitor.shutdownNow();
+ this.monitor.awaitTermination(5, TimeUnit.SECONDS);
+ }
+ } catch (InterruptedException e){
+ //give the stream 30secs to try to shutdown gracefully, then force shutdown otherwise
+ for(List<StreamsTask> tasks : streamsTasks.values()) {
+ for(StreamsTask task : tasks) {
+ task.stopTask();
+ }
+ }
+ this.executor.shutdown();
+ this.monitor.shutdown();
+ try {
+ if(!this.executor.awaitTermination(3, TimeUnit.SECONDS)){
+ this.executor.shutdownNow();
+ }
+ if(!this.monitor.awaitTermination(3, TimeUnit.SECONDS)){
+ this.monitor.shutdownNow();
+ }
+ }catch (InterruptedException ie) {
+ this.executor.shutdownNow();
+ this.monitor.shutdownNow();
+ throw new RuntimeException(ie);
+ }
+ }
+
+ }
+
+ /**
+ * Shutsdown the running tasks in sudo depth first search kind of way. Checks that the upstream components have
+ * finished running before shutting down. Waits till inbound queue is empty to shutdown.
+ * @param comp StreamComponent to shut down.
+ * @param streamTasks the list of non-StreamsProvider tasks for this stream.
+ * @throws InterruptedException
+ */
+ private void shutDownTask(StreamComponent comp, Map<String, List<StreamsTask>> streamTasks) throws InterruptedException {
+ List<StreamsTask> tasks = streamTasks.get(comp.getId());
+ if(tasks != null) { //not a StreamProvider
+ boolean parentsShutDown = true;
+ for(StreamComponent parent : comp.getUpStreamComponents()) {
+ List<StreamsTask> parentTasks = streamTasks.get(parent.getId());
+ //if parentTask == null, its a provider and is not running anymore
+ if(parentTasks != null) {
+ for(StreamsTask task : parentTasks) {
+ parentsShutDown = parentsShutDown && !task.isRunning();
+ }
+ }
+ }
+ if(parentsShutDown) {
+ for(StreamsTask task : tasks) {
+ task.stopTask();
+ }
+ for(StreamsTask task : tasks) {
+ while(task.isRunning()) {
+ Thread.sleep(500);
+ }
+ }
+ }
+ }
+ Collection<StreamComponent> children = comp.getDownStreamComponents();
+ if(children != null) {
+ for(StreamComponent child : comp.getDownStreamComponents()) {
+ shutDownTask(child, streamTasks);
+ }
+ }
+ }
+
+ /**
+ * NOT IMPLEMENTED.
+ */
+ @Override
+ public void stop() {
+
+ }
+
+ private void connectToOtherComponents(String[] conntectToIds, StreamComponent toBeConnected) {
+ for(String id : conntectToIds) {
+ StreamComponent upStream = null;
+ if(this.providers.containsKey(id)) {
+ upStream = this.providers.get(id);
+ }
+ else if(this.components.containsKey(id)) {
+ upStream = this.components.get(id);
+ }
+ else {
+ throw new InvalidStreamException("Cannot connect to id, "+id+", because id does not exist.");
+ }
+ upStream.addOutBoundQueue(toBeConnected, toBeConnected.getInBoundQueue());
+ toBeConnected.addInboundQueue(upStream);
+ }
+ }
+
+ private void validateId(String id) {
+ if(this.providers.containsKey(id) || this.components.containsKey(id)) {
+ throw new InvalidStreamException("Duplicate id. "+id+" is already assigned to another component");
+ }
+ }
+
+
+ private Queue<StreamsDatum> cloneQueue() {
+ return (Queue<StreamsDatum>) SerializationUtil.cloneBySerialization(this.queue);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/StreamComponent.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/StreamComponent.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/StreamComponent.java
new file mode 100644
index 0000000..ecfb22d
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/StreamComponent.java
@@ -0,0 +1,229 @@
+package org.apache.streams.local.builders;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsPersistWriter;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.core.StreamsProvider;
+import org.apache.streams.local.tasks.StreamsPersistWriterTask;
+import org.apache.streams.local.tasks.StreamsProcessorTask;
+import org.apache.streams.local.tasks.StreamsProviderTask;
+import org.apache.streams.local.tasks.StreamsTask;
+import org.apache.streams.util.SerializationUtil;
+import org.joda.time.DateTime;
+
+import java.math.BigInteger;
+import java.util.*;
+
+/**
+ * Stores the implementations of {@link org.apache.streams.core.StreamsOperation}, the StreamsOperations it is connected
+ * to and the necessary metadata to construct a data stream.
+ */
+public class StreamComponent {
+
+ private static final int START = 1;
+ private static final int END = 2;
+
+ private String id;
+ private Set<StreamComponent> inBound;
+ private Map<StreamComponent, Queue<StreamsDatum>> outBound;
+ private Queue<StreamsDatum> inQueue;
+ private StreamsProvider provider;
+ private StreamsProcessor processor;
+ private StreamsPersistWriter writer;
+ private DateTime[] dateRange;
+ private BigInteger sequence;
+ private int numTasks = 1;
+ private boolean perpetual;
+
+ /**
+ *
+ * @param id
+ * @param provider
+ */
+ public StreamComponent(String id, StreamsProvider provider, boolean perpetual) {
+ this.id = id;
+ this.provider = provider;
+ this.perpetual = perpetual;
+ initializePrivateVariables();
+ }
+
+ /**
+ *
+ * @param id
+ * @param provider
+ * @param start
+ * @param end
+ */
+ public StreamComponent(String id, StreamsProvider provider, DateTime start, DateTime end) {
+ this.id = id;
+ this.provider = provider;
+ this.dateRange = new DateTime[2];
+ this.dateRange[START] = start;
+ this.dateRange[END] = end;
+ initializePrivateVariables();
+ }
+
+
+ /**
+ *
+ * @param id
+ * @param provider
+ * @param sequence
+ */
+ public StreamComponent(String id, StreamsProvider provider, BigInteger sequence) {
+ this.id = id;
+ this.provider = provider;
+ this.sequence = sequence;
+ }
+
+ /**
+ *
+ * @param id
+ * @param processor
+ * @param inQueue
+ * @param numTasks
+ */
+ public StreamComponent(String id, StreamsProcessor processor, Queue<StreamsDatum> inQueue, int numTasks) {
+ this.id = id;
+ this.processor = processor;
+ this.inQueue = inQueue;
+ this.numTasks = numTasks;
+ initializePrivateVariables();
+ }
+
+ /**
+ *
+ * @param id
+ * @param writer
+ * @param inQueue
+ * @param numTasks
+ */
+ public StreamComponent(String id, StreamsPersistWriter writer, Queue<StreamsDatum> inQueue, int numTasks) {
+ this.id = id;
+ this.writer = writer;
+ this.inQueue = inQueue;
+ this.numTasks = numTasks;
+ initializePrivateVariables();
+ }
+
+ private void initializePrivateVariables() {
+ this.inBound = new HashSet<StreamComponent>();
+ this.outBound = new HashMap<StreamComponent, Queue<StreamsDatum>>();
+ }
+
+ /**
+ * Add an outbound queue for this component. The queue should be an inbound queue of a downstream component.
+ * @param component the component that this supplying their inbound queue
+ * @param queue the queue to to put post processed/provided datums on
+ */
+ public void addOutBoundQueue(StreamComponent component, Queue<StreamsDatum> queue) {
+ this.outBound.put(component, queue);
+ }
+
+ /**
+ * Add a component that supplies data through the inbound queue.
+ * @param component that supplies data through the inbound queue
+ */
+ public void addInboundQueue(StreamComponent component) {
+ this.inBound.add(component);
+ }
+
+ /**
+ * The components that are immediately downstream of this component (aka child nodes)
+ * @return Collection of child nodes of this component
+ */
+ public Collection<StreamComponent> getDownStreamComponents() {
+ return this.outBound.keySet();
+ }
+
+ /**
+ * The components that are immediately upstream of this component (aka parent nodes)
+ * @return Collection of parent nodes of this component
+ */
+ public Collection<StreamComponent> getUpStreamComponents() {
+ return this.inBound;
+ }
+
+ /**
+ * The inbound queue for this component
+ * @return inbound queue
+ */
+ public Queue<StreamsDatum> getInBoundQueue() {
+ return this.inQueue;
+ }
+
+ /**
+ * The number of tasks this to run this component
+ * @return
+ */
+ public int getNumTasks() {
+ return this.numTasks;
+ }
+
+ /**
+ * Creates a {@link org.apache.streams.local.tasks.StreamsTask} that is running a clone of this component whose
+ * inbound and outbound queues are appropriately connected to the parent and child nodes.
+ * @return StreamsTask for this component
+ */
+ public StreamsTask createConnectedTask() {
+ StreamsTask task;
+ if(this.processor != null) {
+ task = new StreamsProcessorTask((StreamsProcessor)SerializationUtil.cloneBySerialization(this.processor));
+ task.addInputQueue(this.inQueue);
+ for(Queue<StreamsDatum> q : this.outBound.values()) {
+ task.addOutputQueue(q);
+ }
+ }
+ else if(this.writer != null) {
+ if(this.numTasks > 1) {
+ task = new StreamsPersistWriterTask((StreamsPersistWriter) SerializationUtil.cloneBySerialization(this.writer));
+ task.addInputQueue(this.inQueue);
+ } else {
+ task = new StreamsPersistWriterTask(this.writer);
+ task.addInputQueue(this.inQueue);
+ }
+ }
+ else if(this.provider != null) {
+ StreamsProvider prov;
+ if(this.numTasks > 1) {
+ prov = (StreamsProvider)SerializationUtil.cloneBySerialization(this.provider);
+ } else {
+ prov = this.provider;
+ }
+ if(this.dateRange == null && this.sequence == null)
+ task = new StreamsProviderTask(prov, this.perpetual);
+ else if(this.sequence != null)
+ task = new StreamsProviderTask(prov, this.sequence);
+ else
+ task = new StreamsProviderTask(prov, this.dateRange[0], this.dateRange[1]);
+ for(Queue<StreamsDatum> q : this.outBound.values()) {
+ task.addOutputQueue(q);
+ }
+ }
+ else {
+ throw new InvalidStreamException("Underlying StreamComponoent was NULL.");
+ }
+ return task;
+ }
+
+ /**
+ * The unique of this component
+ * @return
+ */
+ public String getId() {
+ return this.id;
+ }
+
+ @Override
+ public int hashCode() {
+ return this.id.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if(o instanceof StreamComponent)
+ return this.id.equals(((StreamComponent) o).id);
+ else
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
new file mode 100644
index 0000000..5f2620b
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
@@ -0,0 +1,148 @@
+package org.apache.streams.local.tasks;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.util.SerializationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+
+/**
+ *
+ */
+public abstract class BaseStreamsTask implements StreamsTask {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(BaseStreamsTask.class);
+
+ private List<Queue<StreamsDatum>> inQueues = new ArrayList<Queue<StreamsDatum>>();
+ private List<Queue<StreamsDatum>> outQueues = new LinkedList<Queue<StreamsDatum>>();
+ private int inIndex = 0;
+ private ObjectMapper mapper;
+
+ public BaseStreamsTask() {
+ this.mapper = new ObjectMapper();
+ this.mapper.registerSubtypes(Activity.class);
+ }
+
+
+ @Override
+ public void addInputQueue(Queue<StreamsDatum> inputQueue) {
+ this.inQueues.add(inputQueue);
+ }
+
+ @Override
+ public void addOutputQueue(Queue<StreamsDatum> outputQueue) {
+ this.outQueues.add(outputQueue);
+ }
+
+ @Override
+ public List<Queue<StreamsDatum>> getInputQueues() {
+ return this.inQueues;
+ }
+
+ @Override
+ public List<Queue<StreamsDatum>> getOutputQueues() {
+ return this.outQueues;
+ }
+
+ /**
+ * NOTE NECESSARY AT THE MOMENT. MAY BECOME NECESSARY AS WE LOOK AT MAKING JOIN TASKS. CURRENTLY ALL TASK HAVE MAX
+ * OF 1 INPUT QUEUE.
+ * Round Robins through input queues to get the next StreamsDatum. If all input queues are empty, it will return null.
+ * @return the next StreamsDatum or null if all input queues are empty.
+ */
+ protected StreamsDatum getNextDatum() {
+ int startIndex = this.inIndex;
+ int index = startIndex;
+ StreamsDatum datum = null;
+ do {
+ datum = this.inQueues.get(index).poll();
+ index = getNextInputQueueIndex();
+ } while( datum == null && startIndex != index);
+ return datum;
+ }
+
+ /**
+ * Adds a StreamDatum to the outgoing queues. If there are multiple queues, it uses serialization to create
+ * clones of the datum and adds a new clone to each queue.
+ * @param datum
+ */
+ protected void addToOutgoingQueue(StreamsDatum datum) {
+ if(this.outQueues.size() == 1) {
+ this.outQueues.get(0).offer(datum);
+ }
+ else {
+ StreamsDatum newDatum = null;
+ for(Queue<StreamsDatum> queue : this.outQueues) {
+ try {
+ newDatum = cloneStreamsDatum(datum);
+ if(newDatum != null)
+ queue.offer(newDatum);
+ } catch (RuntimeException e) {
+ LOGGER.debug("Failed to add StreamsDatum to outgoing queue : {}", datum);
+ LOGGER.error("Exception while offering StreamsDatum to outgoing queue: {}", e);
+ }
+ }
+ }
+ }
+
+ /**
+ * //TODO LOCAL MODE HACK. Need to fix
+ * In order for our data streams to ported to other data flow frame works(Storm, Hadoop, Spark, etc) we need to be able to
+ * enforce the serialization required by each framework. This needs some thought and design before a final solution is
+ * made.
+ *
+ * In order to be able to copy/clone StreamDatums the orginal idea was to force all StreamsDatums to be java serializable.
+ * This was seen as unacceptable for local mode. So until we come up with a solution to enforce serialization and be
+ * compatiable across multiple frame works, this hack is in place.
+ *
+ * If datum.document is Serializable, we use serialization to clone a new copy. If it is not Serializable we attempt
+ * different methods using an com.fasterxml.jackson.databind.ObjectMapper to copy/clone the StreamsDatum. If the object
+ * is not clonable by these methods, an error is reported to the logging and a NULL object is returned.
+ *
+ * @param datum
+ * @return
+ */
+ protected StreamsDatum cloneStreamsDatum(StreamsDatum datum) {
+ try {
+
+ if(datum.document instanceof ObjectNode) {
+ return new StreamsDatum(((ObjectNode) datum.document).deepCopy(), datum.timestamp, datum.sequenceid);
+ }
+ else if(datum.document instanceof Activity) {
+
+ return new StreamsDatum(this.mapper.readValue(this.mapper.writeValueAsString(datum.document), Activity.class),
+ datum.timestamp,
+ datum.sequenceid);
+ }
+// else if(this.mapper.canSerialize(datum.document.getClass())){
+// return new StreamsDatum(this.mapper.readValue(this.mapper.writeValueAsString(datum.document), datum.document.getClass()),
+// datum.timestamp,
+// datum.sequenceid);
+// }
+
+ else if(datum.document instanceof Serializable) {
+ return (StreamsDatum) SerializationUtil.cloneBySerialization(datum);
+ }
+ } catch (Exception e) {
+ LOGGER.error("Exception while trying to clone/copy StreamsDatum : {}", e);
+ }
+ LOGGER.error("Failed to clone/copy StreamsDatum with document of class : {}", datum.document.getClass().getName());
+ return null;
+ }
+
+ private int getNextInputQueueIndex() {
+ ++this.inIndex;
+ if(this.inIndex >= this.inQueues.size()) {
+ this.inIndex = 0;
+ }
+ return this.inIndex;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/LocalStreamProcessMonitorThread.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/LocalStreamProcessMonitorThread.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/LocalStreamProcessMonitorThread.java
new file mode 100644
index 0000000..1325fd6
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/LocalStreamProcessMonitorThread.java
@@ -0,0 +1,69 @@
+package org.apache.streams.local.tasks;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryUsage;
+import java.util.concurrent.Executor;
+
+public class LocalStreamProcessMonitorThread implements Runnable
+{
+ private static final Logger LOGGER = LoggerFactory.getLogger(LocalStreamProcessMonitorThread.class);
+
+ private Executor executor;
+
+ private int seconds;
+
+ private boolean run = true;
+
+ public LocalStreamProcessMonitorThread(Executor executor, int delayInSeconds) {
+ this.executor = executor;
+ this.seconds = delayInSeconds;
+ }
+
+ public void shutdown(){
+ this.run = false;
+ }
+
+ @Override
+ public void run()
+ {
+ while(run){
+
+ /**
+ *
+ * Note:
+ * Quick class and method to let us see what is going on with the JVM. We need to make sure
+ * that everything is running with as little memory as possible. If we are generating a heap
+ * overflow, this will be very apparent by the information shown here.
+ */
+
+ MemoryUsage memoryUsage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
+
+ String maxMemory = memoryUsage.getMax() == Long.MAX_VALUE ? "NO_LIMIT" :
+ humanReadableByteCount(memoryUsage.getMax(), true);
+
+ String usedMemory = humanReadableByteCount(memoryUsage.getUsed(), true);
+
+ LOGGER.info("[monitor] Used Memory: {}, Max: {}",
+ usedMemory,
+ maxMemory);
+
+ try
+ {
+ Thread.sleep(seconds*1000);
+ }
+ catch (InterruptedException e)
+ { }
+ }
+ }
+
+ public String humanReadableByteCount(long bytes, boolean si) {
+ int unit = si ? 1000 : 1024;
+ if (bytes < unit) return bytes + " B";
+ int exp = (int) (Math.log(bytes) / Math.log(unit));
+ String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp-1) + (si ? "" : "i");
+ return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
new file mode 100644
index 0000000..89ca440
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
@@ -0,0 +1,58 @@
+package org.apache.streams.local.tasks;
+
+import org.apache.streams.core.StreamsDatum;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * NOT USED. When joins/partions are implemented, a similar pattern could be followed. Done only as basic proof
+ * of concept.
+ */
+public class StreamsMergeTask extends BaseStreamsTask {
+
+ private AtomicBoolean keepRunning;
+ private long sleepTime;
+
+ public StreamsMergeTask() {
+ this(DEFAULT_SLEEP_TIME_MS);
+ }
+
+ public StreamsMergeTask(long sleepTime) {
+ this.sleepTime = sleepTime;
+ this.keepRunning = new AtomicBoolean(true);
+ }
+
+
+ @Override
+ public void stopTask() {
+ this.keepRunning.set(false);
+ }
+
+ @Override
+ public void setStreamConfig(Map<String, Object> config) {
+
+ }
+
+ @Override
+ public boolean isRunning() {
+ return false;
+ }
+
+ @Override
+ public void run() {
+ while(this.keepRunning.get()) {
+ StreamsDatum datum = super.getNextDatum();
+ if(datum != null) {
+ super.addToOutgoingQueue(datum);
+ }
+ else {
+ try {
+ Thread.sleep(this.sleepTime);
+ } catch (InterruptedException e) {
+ this.keepRunning.set(false);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
new file mode 100644
index 0000000..882bcb7
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
@@ -0,0 +1,103 @@
+package org.apache.streams.local.tasks;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsPersistWriter;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ *
+ */
+public class StreamsPersistWriterTask extends BaseStreamsTask {
+
+
+
+ private StreamsPersistWriter writer;
+ private long sleepTime;
+ private AtomicBoolean keepRunning;
+ private Map<String, Object> streamConfig;
+ private Queue<StreamsDatum> inQueue;
+ private AtomicBoolean isRunning;
+
+ /**
+ * Default constructor. Uses default sleep of 500ms when inbound queue is empty.
+ * @param writer writer to execute in task
+ */
+ public StreamsPersistWriterTask(StreamsPersistWriter writer) {
+ this(writer, DEFAULT_SLEEP_TIME_MS);
+ }
+
+ /**
+ *
+ * @param writer writer to execute in task
+ * @param sleepTime time to sleep when inbound queue is empty.
+ */
+ public StreamsPersistWriterTask(StreamsPersistWriter writer, long sleepTime) {
+ this.writer = writer;
+ this.sleepTime = sleepTime;
+ this.keepRunning = new AtomicBoolean(true);
+ this.isRunning = new AtomicBoolean(true);
+ }
+
+ @Override
+ public void setStreamConfig(Map<String, Object> config) {
+ this.streamConfig = config;
+ }
+
+ @Override
+ public void addInputQueue(Queue<StreamsDatum> inputQueue) {
+ this.inQueue = inputQueue;
+ }
+
+ @Override
+ public boolean isRunning() {
+ return this.isRunning.get();
+ }
+
+ @Override
+ public void run() {
+ try {
+ this.writer.prepare(this.streamConfig);
+ StreamsDatum datum = this.inQueue.poll();
+ while(datum != null || this.keepRunning.get()) {
+ if(datum != null) {
+ this.writer.write(datum);
+ }
+ else {
+ try {
+ Thread.sleep(this.sleepTime);
+ } catch (InterruptedException e) {
+ this.keepRunning.set(false);
+ }
+ }
+ datum = this.inQueue.poll();
+ }
+
+ } finally {
+ this.writer.cleanUp();
+ this.isRunning.set(false);
+ }
+ }
+
+ @Override
+ public void stopTask() {
+ this.keepRunning.set(false);
+ }
+
+
+ @Override
+ public void addOutputQueue(Queue<StreamsDatum> outputQueue) {
+ throw new UnsupportedOperationException(this.getClass().getName()+" does not support method - setOutputQueue()");
+ }
+
+ @Override
+ public List<Queue<StreamsDatum>> getInputQueues() {
+ List<Queue<StreamsDatum>> queues = new LinkedList<Queue<StreamsDatum>>();
+ queues.add(this.inQueue);
+ return queues;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
new file mode 100644
index 0000000..d1ac905
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
@@ -0,0 +1,101 @@
+package org.apache.streams.local.tasks;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ *
+ */
+public class StreamsProcessorTask extends BaseStreamsTask {
+
+
+ private StreamsProcessor processor;
+ private long sleepTime;
+ private AtomicBoolean keepRunning;
+ private Map<String, Object> streamConfig;
+ private Queue<StreamsDatum> inQueue;
+ private AtomicBoolean isRunning;
+
+ /**
+ * Default constructor, uses default sleep time of 500ms when inbound queue is empty
+ * @param processor process to run in task
+ */
+ public StreamsProcessorTask(StreamsProcessor processor) {
+ this(processor, DEFAULT_SLEEP_TIME_MS);
+ }
+
+ /**
+ *
+ * @param processor processor to run in task
+ * @param sleepTime time to sleep when incoming queue is empty
+ */
+ public StreamsProcessorTask(StreamsProcessor processor, long sleepTime) {
+ this.processor = processor;
+ this.sleepTime = sleepTime;
+ this.keepRunning = new AtomicBoolean(true);
+ this.isRunning = new AtomicBoolean(true);
+ }
+
+ @Override
+ public void stopTask() {
+ this.keepRunning.set(false);
+ }
+
+ @Override
+ public void setStreamConfig(Map<String, Object> config) {
+ this.streamConfig = config;
+ }
+
+ @Override
+ public void addInputQueue(Queue<StreamsDatum> inputQueue) {
+ this.inQueue = inputQueue;
+ }
+
+ @Override
+ public boolean isRunning() {
+ return this.isRunning.get();
+ }
+
+ @Override
+ public void run() {
+ try {
+ this.processor.prepare(this.streamConfig);
+ StreamsDatum datum = this.inQueue.poll();
+ while(datum != null || this.keepRunning.get()) {
+ if(datum != null) {
+ List<StreamsDatum> output = this.processor.process(datum);
+ if(output != null) {
+ for(StreamsDatum outDatum : output) {
+ super.addToOutgoingQueue(outDatum);
+ }
+ }
+ }
+ else {
+ try {
+ Thread.sleep(this.sleepTime);
+ } catch (InterruptedException e) {
+ this.keepRunning.set(false);
+ }
+ }
+ datum = this.inQueue.poll();
+ }
+
+ } finally {
+ this.processor.cleanUp();
+ this.isRunning.set(false);
+ }
+ }
+
+ @Override
+ public List<Queue<StreamsDatum>> getInputQueues() {
+ List<Queue<StreamsDatum>> queues = new LinkedList<Queue<StreamsDatum>>();
+ queues.add(this.inQueue);
+ return queues;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
new file mode 100644
index 0000000..b4c929d
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
@@ -0,0 +1,175 @@
+package org.apache.streams.local.tasks;
+
+import org.apache.streams.core.DatumStatusCounter;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProvider;
+import org.apache.streams.core.StreamsResultSet;
+import org.joda.time.DateTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.math.BigInteger;
+import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ *
+ */
+public class StreamsProviderTask extends BaseStreamsTask {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(StreamsProviderTask.class);
+
+ private static enum Type {
+ PERPETUAL,
+ READ_CURRENT,
+ READ_NEW,
+ READ_RANGE
+ }
+
+ private static final int START = 0;
+ private static final int END = 1;
+
+ private static final int DEFAULT_TIMEOUT_MS = 30000;
+
+ private StreamsProvider provider;
+ private AtomicBoolean keepRunning;
+ private Type type;
+ private BigInteger sequence;
+ private DateTime[] dateRange;
+ private Map<String, Object> config;
+ private AtomicBoolean isRunning;
+
+ private int zeros = 0;
+ private DatumStatusCounter statusCounter;
+
+ /**
+ * Constructor for a StreamsProvider to execute {@link org.apache.streams.core.StreamsProvider:readCurrent()}
+ * @param provider
+ */
+ public StreamsProviderTask(StreamsProvider provider, boolean perpetual) {
+ this.provider = provider;
+ if( perpetual )
+ this.type = Type.PERPETUAL;
+ else
+ this.type = Type.READ_CURRENT;
+ this.keepRunning = new AtomicBoolean(true);
+ this.isRunning = new AtomicBoolean(true);
+ }
+
+ /**
+ * Constructor for a StreamsProvider to execute {@link org.apache.streams.core.StreamsProvider:readNew(BigInteger)}
+ * @param provider
+ * @param sequence
+ */
+ public StreamsProviderTask(StreamsProvider provider, BigInteger sequence) {
+ this.provider = provider;
+ this.type = Type.READ_NEW;
+ this.sequence = sequence;
+ this.keepRunning = new AtomicBoolean(true);
+ this.isRunning = new AtomicBoolean(true);
+ }
+
+ /**
+ * Constructor for a StreamsProvider to execute {@link org.apache.streams.core.StreamsProvider:readRange(DateTime,DateTime)}
+ * @param provider
+ * @param start
+ * @param end
+ */
+ public StreamsProviderTask(StreamsProvider provider, DateTime start, DateTime end) {
+ this.provider = provider;
+ this.type = Type.READ_RANGE;
+ this.dateRange = new DateTime[2];
+ this.dateRange[START] = start;
+ this.dateRange[END] = end;
+ this.keepRunning = new AtomicBoolean(true);
+ this.isRunning = new AtomicBoolean(true);
+ }
+
+ @Override
+ public void stopTask() {
+ this.keepRunning.set(false);
+ }
+
+ @Override
+ public void addInputQueue(Queue<StreamsDatum> inputQueue) {
+ throw new UnsupportedOperationException(this.getClass().getName()+" does not support method - setInputQueue()");
+ }
+
+ @Override
+ public void setStreamConfig(Map<String, Object> config) {
+ this.config = config;
+ }
+
+ @Override
+ public void run() {
+ try {
+ this.provider.prepare(this.config); //TODO allow for configuration objects
+ StreamsResultSet resultSet = null;
+ this.isRunning.set(true);
+ switch(this.type) {
+ case PERPETUAL: {
+ provider.startStream();
+ while(this.keepRunning.get() == true) {
+ try {
+ resultSet = provider.readCurrent();
+ if( resultSet.size() == 0 )
+ zeros++;
+ else {
+ zeros = 0;
+ if( resultSet.getCounter() != null ) {
+ LOGGER.debug(resultSet.getCounter().toString());
+ statusCounter.add(resultSet.getCounter());
+ }
+ }
+ flushResults(resultSet);
+ if( zeros > (DEFAULT_TIMEOUT_MS / DEFAULT_SLEEP_TIME_MS))
+ this.keepRunning.set(false);
+ Thread.sleep(DEFAULT_SLEEP_TIME_MS);
+ } catch (InterruptedException e) {
+ this.keepRunning.set(false);
+ }
+ }
+ }
+ break;
+ case READ_CURRENT: resultSet = this.provider.readCurrent();
+ break;
+ case READ_NEW: resultSet = this.provider.readNew(this.sequence);
+ break;
+ case READ_RANGE: resultSet = this.provider.readRange(this.dateRange[START], this.dateRange[END]);
+ break;
+ default: throw new RuntimeException("Type has not been added to StreamsProviderTask.");
+ }
+ flushResults(resultSet);
+
+ } catch( Exception e ) {
+ e.printStackTrace();
+ } finally
+ {
+ this.provider.cleanUp();
+ this.isRunning.set(false);
+ }
+ }
+
+ public boolean isRunning() {
+ return this.isRunning.get();
+ }
+
+ public void flushResults(StreamsResultSet resultSet) {
+ for(StreamsDatum datum : resultSet) {
+ if(!this.keepRunning.get()) {
+ break;
+ }
+ if(datum != null)
+ super.addToOutgoingQueue(datum);
+ else {
+ try {
+ Thread.sleep(DEFAULT_SLEEP_TIME_MS);
+ } catch (InterruptedException e) {
+ this.keepRunning.set(false);
+ }
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
new file mode 100644
index 0000000..e7f5a2c
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
@@ -0,0 +1,58 @@
+package org.apache.streams.local.tasks;
+
+import org.apache.streams.core.StreamsDatum;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+
+/**
+ * Interface for all task that will be used to execute instances of {@link org.apache.streams.core.StreamsOperation}
+ * in local mode.
+ */
+public interface StreamsTask extends Runnable{
+
+ public static final long DEFAULT_SLEEP_TIME_MS = 5000;
+
+ /**
+ * Informs the task to stop. Tasks may or may not try to empty its inbound queue before halting.
+ */
+ public void stopTask();
+
+ /**
+ * Add an input {@link java.util.Queue} for this task.
+ * @param inputQueue
+ */
+ public void addInputQueue(Queue<StreamsDatum> inputQueue);
+
+ /**
+ * Add an output {@link java.util.Queue} for this task.
+ * @param outputQueue
+ */
+ public void addOutputQueue(Queue<StreamsDatum> outputQueue);
+
+ /**
+ * Set the configuration object that will shared and passed to all instances of StreamsTask.
+ * @param config optional configuration information
+ */
+ public void setStreamConfig(Map<String, Object> config);
+
+ /**
+ * Returns true when the task has not completed. Returns false otherwise
+ * @return true when the task has not completed. Returns false otherwise
+ */
+ public boolean isRunning();
+
+ /**
+ * Returns the input queues that have been set for this task.
+ * @return list of input queues
+ */
+ public List<Queue<StreamsDatum>> getInputQueues();
+
+ /**
+ * Returns the output queues that have been set for this task
+ * @return list of output queues
+ */
+ public List<Queue<StreamsDatum>> getOutputQueues();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/LocalStreamBuilderTest.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/LocalStreamBuilderTest.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/LocalStreamBuilderTest.java
new file mode 100644
index 0000000..ac4fb5e
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/LocalStreamBuilderTest.java
@@ -0,0 +1,162 @@
+package org.apache.streams.local.builders;
+
+import org.apache.streams.core.StreamBuilder;
+import org.apache.streams.core.test.processors.PassthroughDatumCounterProcessor;
+import org.apache.streams.core.test.providers.NumericMessageProvider;
+import org.apache.streams.core.test.writer.SystemOutWriter;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.HashSet;
+import java.util.Scanner;
+
+import static org.junit.Assert.*;
+
+/**
+ * Basic Tests for the LocalStreamBuilder.
+ *
+ * Test are performed by redirecting system out and counting the number of lines that the SystemOutWriter prints
+ * to System.out. The SystemOutWriter also prints one line when cleanUp() is called, so this is why it tests for
+ * the numDatums +1.
+ *
+ *
+ */
+public class LocalStreamBuilderTest {
+
+ ByteArrayOutputStream out;
+
+ @Before
+ public void setSystemOut() {
+ out = new ByteArrayOutputStream();
+ System.setOut(new PrintStream(out));
+ }
+
+ @Test
+ public void testStreamIdValidations() {
+ StreamBuilder builder = new LocalStreamBuilder();
+ builder.newReadCurrentStream("id", new NumericMessageProvider(1));
+ Exception exp = null;
+ try {
+ builder.newReadCurrentStream("id", new NumericMessageProvider(1));
+ } catch (RuntimeException e) {
+ exp = e;
+ }
+ assertNotNull(exp);
+ exp = null;
+ builder.addStreamsProcessor("1", new PassthroughDatumCounterProcessor(), 1, "id");
+ try {
+ builder.addStreamsProcessor("2", new PassthroughDatumCounterProcessor(), 1, "id", "id2");
+ } catch (RuntimeException e) {
+ exp = e;
+ }
+ assertNotNull(exp);
+ }
+
+ @Test
+ public void testBasicLinearStream1() {
+ int numDatums = 1;
+ StreamBuilder builder = new LocalStreamBuilder();
+ PassthroughDatumCounterProcessor processor = new PassthroughDatumCounterProcessor();
+ SystemOutWriter writer = new SystemOutWriter();
+ builder.newReadCurrentStream("sp1", new NumericMessageProvider(numDatums))
+ .addStreamsProcessor("proc1", processor, 1, "sp1")
+ .addStreamsPersistWriter("writer1", writer, 1, "proc1");
+ builder.start();
+ int count = 0;
+ Scanner scanner = new Scanner(new ByteArrayInputStream(out.toByteArray()));
+ while(scanner.hasNextLine()) {
+ ++count;
+ scanner.nextLine();
+ }
+ assertEquals(numDatums+2, count);
+ }
+
+ @Test
+ public void testBasicLinearStream2() {
+ int numDatums = 100;
+ StreamBuilder builder = new LocalStreamBuilder();
+ PassthroughDatumCounterProcessor processor = new PassthroughDatumCounterProcessor();
+ SystemOutWriter writer = new SystemOutWriter();
+ builder.newReadCurrentStream("sp1", new NumericMessageProvider(numDatums))
+ .addStreamsProcessor("proc1", processor, 1, "sp1")
+ .addStreamsPersistWriter("writer1", writer, 1, "proc1");
+ builder.start();
+ int count = 0;
+ Scanner scanner = new Scanner(new ByteArrayInputStream(out.toByteArray()));
+ while(scanner.hasNextLine()) {
+ ++count;
+ scanner.nextLine();
+ }
+ assertEquals(numDatums+2, count);
+ }
+
+ @Test
+ public void testParallelLinearStream1() {
+ int numDatums = 1000;
+ int parallelHint = 20;
+ PassthroughDatumCounterProcessor.sawData = new HashSet<Integer>();
+ PassthroughDatumCounterProcessor.claimedNumber = new HashSet<Integer>();
+ StreamBuilder builder = new LocalStreamBuilder();
+ PassthroughDatumCounterProcessor processor = new PassthroughDatumCounterProcessor();
+ SystemOutWriter writer = new SystemOutWriter();
+ builder.newReadCurrentStream("sp1", new NumericMessageProvider(numDatums))
+ .addStreamsProcessor("proc1", processor, parallelHint, "sp1")
+ .addStreamsPersistWriter("writer1", writer, 1, "proc1");
+ builder.start();
+ int count = 0;
+ Scanner scanner = new Scanner(new ByteArrayInputStream(out.toByteArray()));
+ while(scanner.hasNextLine()) {
+ ++count;
+ scanner.nextLine();
+ }
+ assertEquals(numDatums+2, count); //+1 is to make sure cleanup is called on the writer
+ assertEquals(parallelHint, PassthroughDatumCounterProcessor.claimedNumber.size()); //test 40 were initialized
+ assertTrue(PassthroughDatumCounterProcessor.sawData.size() > 1 && PassthroughDatumCounterProcessor.sawData.size() <= parallelHint); //test more than one processor got data
+ }
+
+ @Test
+ public void testBasicMergeStream() {
+ int numDatums1 = 1;
+ int numDatums2 = 100;
+ PassthroughDatumCounterProcessor processor1 = new PassthroughDatumCounterProcessor();
+ PassthroughDatumCounterProcessor processor2 = new PassthroughDatumCounterProcessor();
+ SystemOutWriter writer = new SystemOutWriter();
+ StreamBuilder builder = new LocalStreamBuilder();
+ builder.newReadCurrentStream("sp1", new NumericMessageProvider(numDatums1))
+ .newReadCurrentStream("sp2", new NumericMessageProvider(numDatums2))
+ .addStreamsProcessor("proc1", processor1, 1, "sp1")
+ .addStreamsProcessor("proc2", processor2, 1, "sp2")
+ .addStreamsPersistWriter("writer1", writer, 1, "proc1", "proc2");
+ builder.start();
+ int count = 0;
+ Scanner scanner = new Scanner(new ByteArrayInputStream(out.toByteArray()));
+ while(scanner.hasNextLine()) {
+ ++count;
+ scanner.nextLine();
+ }
+ assertEquals(numDatums1+numDatums2+2, count);
+ }
+
+ @Test
+ public void testBasicBranch() {
+ int numDatums = 100;
+ StreamBuilder builder = new LocalStreamBuilder();
+ builder.newReadCurrentStream("prov1", new NumericMessageProvider(numDatums))
+ .addStreamsProcessor("proc1", new PassthroughDatumCounterProcessor(), 1, "prov1")
+ .addStreamsProcessor("proc2", new PassthroughDatumCounterProcessor(), 1, "prov1")
+ .addStreamsPersistWriter("w1", new SystemOutWriter(), 1, "proc1", "proc2");
+ builder.start();
+ int count = 0;
+ Scanner scanner = new Scanner(new ByteArrayInputStream(out.toByteArray()));
+ while(scanner.hasNextLine()) {
+ ++count;
+ scanner.nextLine();
+ }
+ assertEquals((numDatums*2)+2, count);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/ToyLocalBuilderExample.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/ToyLocalBuilderExample.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/ToyLocalBuilderExample.java
new file mode 100644
index 0000000..f812a16
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/ToyLocalBuilderExample.java
@@ -0,0 +1,28 @@
+package org.apache.streams.local.builders;
+
+import org.apache.streams.core.StreamBuilder;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.test.processors.DoNothingProcessor;
+import org.apache.streams.core.test.providers.NumericMessageProvider;
+import org.apache.streams.core.test.writer.DoNothingWriter;
+
+import java.util.concurrent.LinkedBlockingQueue;
+
+/**
+ * Created by rebanks on 2/20/14.
+ */
+public class ToyLocalBuilderExample {
+
+ /**
+ * A simple example of how to run a stream in local mode.
+ * @param args
+ */
+ public static void main(String[] args) {
+ StreamBuilder builder = new LocalStreamBuilder(new LinkedBlockingQueue<StreamsDatum>());
+ builder.newReadCurrentStream("prov", new NumericMessageProvider(1000000))
+ .addStreamsProcessor("proc", new DoNothingProcessor(), 100, "prov")
+ .addStreamsPersistWriter("writer", new DoNothingWriter(), 3, "proc");
+ builder.start();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
new file mode 100644
index 0000000..d5007bf
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
@@ -0,0 +1,285 @@
+package org.apache.streams.local.tasks;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.test.processors.PassthroughDatumCounterProcessor;
+import org.apache.streams.core.test.providers.NumericMessageProvider;
+import org.apache.streams.core.test.writer.DatumCounterWriter;
+import org.junit.Test;
+
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.*;
+
+/**
+ * Created by rebanks on 2/18/14.
+ */
+public class BasicTasksTest {
+
+
+
+ @Test
+ public void testProviderTask() {
+ int numMessages = 100;
+ NumericMessageProvider provider = new NumericMessageProvider(numMessages);
+ StreamsProviderTask task = new StreamsProviderTask(provider, false);
+ Queue<StreamsDatum> outQueue = new ConcurrentLinkedQueue<StreamsDatum>();
+ task.addOutputQueue(outQueue);
+ Queue<StreamsDatum> inQueue = createInputQueue(numMessages);
+ Exception exp = null;
+ try {
+ task.addInputQueue(inQueue);
+ } catch (UnsupportedOperationException uoe) {
+ exp = uoe;
+ }
+ assertNotNull(exp);
+ ExecutorService service = Executors.newFixedThreadPool(1);
+ service.submit(task);
+ int attempts = 0;
+ while(outQueue.size() != numMessages) {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ //Ignore
+ }
+ ++attempts;
+ if(attempts == 10) {
+ fail("Provider task failed to output "+numMessages+" in a timely fashion.");
+ }
+ }
+ service.shutdown();
+ try {
+ if(!service.awaitTermination(5, TimeUnit.SECONDS)){
+ service.shutdownNow();
+ fail("Service did not terminate.");
+ }
+ assertTrue("Task should have completed running in aloted time.", service.isTerminated());
+ } catch (InterruptedException e) {
+ fail("Test Interupted.");
+ };
+ }
+
+ @Test
+ public void testProcessorTask() {
+ int numMessages = 100;
+ PassthroughDatumCounterProcessor processor = new PassthroughDatumCounterProcessor();
+ StreamsProcessorTask task = new StreamsProcessorTask(processor);
+ Queue<StreamsDatum> outQueue = new ConcurrentLinkedQueue<StreamsDatum>();
+ Queue<StreamsDatum> inQueue = createInputQueue(numMessages);
+ task.addOutputQueue(outQueue);
+ task.addInputQueue(inQueue);
+ assertEquals(numMessages, task.getInputQueues().get(0).size());
+ ExecutorService service = Executors.newFixedThreadPool(1);
+ service.submit(task);
+ int attempts = 0;
+ while(inQueue.size() != 0 && outQueue.size() != numMessages) {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ //Ignore
+ }
+ ++attempts;
+ if(attempts == 10) {
+ fail("Processor task failed to output "+numMessages+" in a timely fashion.");
+ }
+ }
+ task.stopTask();
+ assertEquals(numMessages, processor.getMessageCount());
+ service.shutdown();
+ try {
+ if(!service.awaitTermination(5, TimeUnit.SECONDS)){
+ service.shutdownNow();
+ fail("Service did not terminate.");
+ }
+ assertTrue("Task should have completed running in aloted time.", service.isTerminated());
+ } catch (InterruptedException e) {
+ fail("Test Interupted.");
+ }
+ }
+
+ @Test
+ public void testWriterTask() {
+ int numMessages = 100;
+ DatumCounterWriter writer = new DatumCounterWriter();
+ StreamsPersistWriterTask task = new StreamsPersistWriterTask(writer);
+ Queue<StreamsDatum> outQueue = new ConcurrentLinkedQueue<StreamsDatum>();
+ Queue<StreamsDatum> inQueue = createInputQueue(numMessages);
+
+ Exception exp = null;
+ try {
+ task.addOutputQueue(outQueue);
+ } catch (UnsupportedOperationException uoe) {
+ exp = uoe;
+ }
+ assertNotNull(exp);
+ task.addInputQueue(inQueue);
+ assertEquals(numMessages, task.getInputQueues().get(0).size());
+ ExecutorService service = Executors.newFixedThreadPool(1);
+ service.submit(task);
+ int attempts = 0;
+ while(inQueue.size() != 0 ) {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ //Ignore
+ }
+ ++attempts;
+ if(attempts == 10) {
+ fail("Processor task failed to output "+numMessages+" in a timely fashion.");
+ }
+ }
+ task.stopTask();
+ assertEquals(numMessages, writer.getDatumsCounted());
+ service.shutdown();
+ try {
+ if(!service.awaitTermination(5, TimeUnit.SECONDS)){
+ service.shutdownNow();
+ fail("Service did not terminate.");
+ }
+ assertTrue("Task should have completed running in aloted time.", service.isTerminated());
+ } catch (InterruptedException e) {
+ fail("Test Interupted.");
+ }
+ }
+
+ @Test
+ public void testMergeTask() {
+ int numMessages = 100;
+ int incoming = 5;
+ StreamsMergeTask task = new StreamsMergeTask();
+ Queue<StreamsDatum> outQueue = new ConcurrentLinkedQueue<StreamsDatum>();
+ task.addOutputQueue(outQueue);
+ for(int i=0; i < incoming; ++i) {
+ task.addInputQueue(createInputQueue(numMessages));
+ }
+ ExecutorService service = Executors.newFixedThreadPool(1);
+ service.submit(task);
+ int attempts = 0;
+ while(outQueue.size() != incoming * numMessages ) {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ //Ignore
+ }
+ ++attempts;
+ if(attempts == 10) {
+ assertEquals("Processor task failed to output " + (numMessages * incoming) + " in a timely fashion.", (numMessages * incoming), outQueue.size());
+ }
+ }
+ task.stopTask();
+ service.shutdown();
+ try {
+ if(!service.awaitTermination(5, TimeUnit.SECONDS)){
+ service.shutdownNow();
+ fail("Service did not terminate.");
+ }
+ assertTrue("Task should have completed running in aloted time.", service.isTerminated());
+ } catch (InterruptedException e) {
+ fail("Test Interupted.");
+ }
+ }
+
+ @Test
+ public void testBranching() {
+ int numMessages = 100;
+ PassthroughDatumCounterProcessor processor = new PassthroughDatumCounterProcessor();
+ StreamsProcessorTask task = new StreamsProcessorTask(processor);
+ Queue<StreamsDatum> outQueue1 = new ConcurrentLinkedQueue<StreamsDatum>();
+ Queue<StreamsDatum> outQueue2 = new ConcurrentLinkedQueue<StreamsDatum>();
+ Queue<StreamsDatum> inQueue = createInputQueue(numMessages);
+ task.addOutputQueue(outQueue1);
+ task.addOutputQueue(outQueue2);
+ task.addInputQueue(inQueue);
+ assertEquals(numMessages, task.getInputQueues().get(0).size());
+ ExecutorService service = Executors.newFixedThreadPool(1);
+ service.submit(task);
+ int attempts = 0;
+ while(inQueue.size() != 0 ) {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ //Ignore
+ }
+ ++attempts;
+ if(attempts == 10) {
+ assertEquals("Processor task failed to output "+(numMessages)+" in a timely fashion.", 0, inQueue.size());
+ }
+ }
+ task.stopTask();
+
+ service.shutdown();
+ try {
+ if(!service.awaitTermination(5, TimeUnit.SECONDS)){
+ service.shutdownNow();
+ fail("Service did not terminate.");
+ }
+ assertTrue("Task should have completed running in aloted time.", service.isTerminated());
+ } catch (InterruptedException e) {
+ fail("Test Interupted.");
+ }
+ assertEquals(numMessages, processor.getMessageCount());
+ assertEquals(numMessages, outQueue1.size());
+ assertEquals(numMessages, outQueue2.size());
+ }
+
+ @Test
+ public void testBranchingSerialization() {
+ int numMessages = 1;
+ PassthroughDatumCounterProcessor processor = new PassthroughDatumCounterProcessor();
+ StreamsProcessorTask task = new StreamsProcessorTask(processor);
+ Queue<StreamsDatum> outQueue1 = new ConcurrentLinkedQueue<StreamsDatum>();
+ Queue<StreamsDatum> outQueue2 = new ConcurrentLinkedQueue<StreamsDatum>();
+ Queue<StreamsDatum> inQueue = createInputQueue(numMessages);
+ task.addOutputQueue(outQueue1);
+ task.addOutputQueue(outQueue2);
+ task.addInputQueue(inQueue);
+ ExecutorService service = Executors.newFixedThreadPool(1);
+ service.submit(task);
+ int attempts = 0;
+ while(inQueue.size() != 0 ) {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ //Ignore
+ }
+ ++attempts;
+ if(attempts == 10) {
+ assertEquals("Processor task failed to output "+(numMessages)+" in a timely fashion.", 0, inQueue.size());
+ }
+ }
+ task.stopTask();
+
+ service.shutdown();
+ try {
+ if(!service.awaitTermination(5, TimeUnit.SECONDS)){
+ service.shutdownNow();
+ fail("Service did not terminate.");
+ }
+ assertTrue("Task should have completed running in aloted time.", service.isTerminated());
+ } catch (InterruptedException e) {
+ fail("Test Interupted.");
+ }
+ assertEquals(numMessages, processor.getMessageCount());
+ assertEquals(numMessages, outQueue1.size());
+ assertEquals(numMessages, outQueue2.size());
+ StreamsDatum datum1 = outQueue1.poll();
+ StreamsDatum datum2 = outQueue2.poll();
+ assertNotNull(datum1);
+ assertEquals(datum1, datum2);
+ datum1.setDocument("a");
+ assertNotEquals(datum1, datum2);
+ }
+
+ private Queue<StreamsDatum> createInputQueue(int numDatums) {
+ Queue<StreamsDatum> queue = new ConcurrentLinkedQueue<StreamsDatum>();
+ for(int i=0; i < numDatums; ++i) {
+ queue.add(new StreamsDatum(i));
+ }
+ return queue;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/processors/DoNothingProcessor.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/processors/DoNothingProcessor.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/processors/DoNothingProcessor.java
new file mode 100644
index 0000000..25c717b
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/processors/DoNothingProcessor.java
@@ -0,0 +1,32 @@
+package org.apache.streams.core.test.processors;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Created by rebanks on 2/20/14.
+ */
+public class DoNothingProcessor implements StreamsProcessor {
+
+ List<StreamsDatum> result;
+
+ @Override
+ public List<StreamsDatum> process(StreamsDatum entry) {
+ this.result = new LinkedList<StreamsDatum>();
+ result.add(entry);
+ return result;
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+
+ }
+
+ @Override
+ public void cleanUp() {
+ System.out.println("Processor clean up!");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/processors/PassthroughDatumCounterProcessor.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/processors/PassthroughDatumCounterProcessor.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/processors/PassthroughDatumCounterProcessor.java
new file mode 100644
index 0000000..1010370
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/processors/PassthroughDatumCounterProcessor.java
@@ -0,0 +1,49 @@
+package org.apache.streams.core.test.processors;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+
+import java.util.*;
+
+/**
+ * Created by rebanks on 2/18/14.
+ */
+public class PassthroughDatumCounterProcessor implements StreamsProcessor {
+
+ public static Set<Integer> claimedNumber = new HashSet<Integer>();
+ public static final Random rand = new Random();
+ public static Set<Integer> sawData = new HashSet<Integer>();
+
+ private int count = 0;
+ private int id;
+
+ @Override
+ public List<StreamsDatum> process(StreamsDatum entry) {
+ ++this.count;
+ List<StreamsDatum> result = new LinkedList<StreamsDatum>();
+ result.add(entry);
+ synchronized (sawData) {
+ sawData.add(this.id);
+ }
+ return result;
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+ synchronized (claimedNumber) {
+ this.id = rand.nextInt();
+ while(!claimedNumber.add(this.id)) {
+ this.id = rand.nextInt();
+ }
+ }
+ }
+
+ @Override
+ public void cleanUp() {
+
+ }
+
+ public int getMessageCount() {
+ return this.count;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/providers/NumericMessageProvider.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/providers/NumericMessageProvider.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/providers/NumericMessageProvider.java
new file mode 100644
index 0000000..740c9b6
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/providers/NumericMessageProvider.java
@@ -0,0 +1,102 @@
+package org.apache.streams.core.test.providers;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProvider;
+import org.apache.streams.core.StreamsResultSet;
+import org.joda.time.DateTime;
+
+import java.math.BigInteger;
+import java.util.Iterator;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+/**
+ * Test StreamsProvider that sends out StreamsDatums numbered from 0 to numMessages.
+ */
+public class NumericMessageProvider implements StreamsProvider {
+
+ private int numMessages;
+
+ public NumericMessageProvider(int numMessages) {
+ this.numMessages = numMessages;
+ }
+
+ @Override
+ public void startStream() {
+ // no op
+ }
+
+ @Override
+ public StreamsResultSet readCurrent() {
+ return new ResultSet();
+ }
+
+ @Override
+ public StreamsResultSet readNew(BigInteger sequence) {
+ return new ResultSet();
+ }
+
+ @Override
+ public StreamsResultSet readRange(DateTime start, DateTime end) {
+ return new ResultSet();
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+
+ }
+
+ @Override
+ public void cleanUp() {
+
+ }
+
+
+ private class ResultSet extends StreamsResultSet {
+
+ private ResultSet() {
+ super(new ConcurrentLinkedQueue<StreamsDatum>());
+ }
+
+// @Override
+// public long getStartTime() {
+// return 0;
+// }
+//
+// @Override
+// public long getEndTime() {
+// return 0;
+// }
+//
+// @Override
+// public String getSourceId() {
+// return null;
+// }
+//
+// @Override
+// public BigInteger getMaxSequence() {
+// return null;
+// }
+
+ @Override
+ public Iterator<StreamsDatum> iterator() {
+ return new Iterator<StreamsDatum>() {
+ private int i = 0;
+
+ @Override
+ public boolean hasNext() {
+ return i < numMessages;
+ }
+
+ @Override
+ public StreamsDatum next() {
+ return new StreamsDatum(i++);
+ }
+
+ @Override
+ public void remove() {
+
+ }
+ };
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/DatumCounterWriter.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/DatumCounterWriter.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/DatumCounterWriter.java
new file mode 100644
index 0000000..d44d24c
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/DatumCounterWriter.java
@@ -0,0 +1,31 @@
+package org.apache.streams.core.test.writer;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsPersistWriter;
+
+/**
+ * Created by rebanks on 2/18/14.
+ */
+public class DatumCounterWriter implements StreamsPersistWriter{
+
+ private int counter = 0;
+
+ @Override
+ public void write(StreamsDatum entry) {
+ ++this.counter;
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+
+ }
+
+ @Override
+ public void cleanUp() {
+ System.out.println("clean up called");
+ }
+
+ public int getDatumsCounted() {
+ return this.counter;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/DoNothingWriter.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/DoNothingWriter.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/DoNothingWriter.java
new file mode 100644
index 0000000..a52347e
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/DoNothingWriter.java
@@ -0,0 +1,24 @@
+package org.apache.streams.core.test.writer;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsPersistWriter;
+
+/**
+ * Created by rebanks on 2/20/14.
+ */
+public class DoNothingWriter implements StreamsPersistWriter {
+ @Override
+ public void write(StreamsDatum entry) {
+
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+
+ }
+
+ @Override
+ public void cleanUp() {
+ System.out.println("Writer Clean Up!");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/SystemOutWriter.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/SystemOutWriter.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/SystemOutWriter.java
new file mode 100644
index 0000000..1856e34
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/SystemOutWriter.java
@@ -0,0 +1,24 @@
+package org.apache.streams.core.test.writer;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsPersistWriter;
+
+/**
+ * Created by rebanks on 2/20/14.
+ */
+public class SystemOutWriter implements StreamsPersistWriter {
+ @Override
+ public void write(StreamsDatum entry) {
+ System.out.println(entry.document);
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+
+ }
+
+ @Override
+ public void cleanUp() {
+ System.out.println("Clean up called writer!");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/ExpectedDatumsPersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/ExpectedDatumsPersistWriter.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/ExpectedDatumsPersistWriter.java
new file mode 100644
index 0000000..675edd7
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/ExpectedDatumsPersistWriter.java
@@ -0,0 +1,54 @@
+package org.apache.streams.test.component;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsPersistWriter;
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Scanner;
+
+/**
+ * Created by rebanks on 2/27/14.
+ */
+public class ExpectedDatumsPersistWriter implements StreamsPersistWriter{
+
+ private StreamsDatumConverter converter;
+ private String fileName;
+ private List<StreamsDatum> expectedDatums;
+ private int counted = 0;
+ private int expectedSize = 0;
+
+ public ExpectedDatumsPersistWriter(StreamsDatumConverter converter, String filePathInResources) {
+ this.converter = converter;
+ this.fileName = filePathInResources;
+ }
+
+
+
+ @Override
+ public void write(StreamsDatum entry) {
+ int index = this.expectedDatums.indexOf(entry);
+ assertNotEquals("Datum not expected. "+entry.toString(), -1, index);
+ this.expectedDatums.remove(index);
+ ++this.counted;
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+ Scanner scanner = new Scanner(ExpectedDatumsPersistWriter.class.getResourceAsStream(this.fileName));
+ this.expectedDatums = new LinkedList<StreamsDatum>();
+ while(scanner.hasNextLine()) {
+ this.expectedDatums.add(this.converter.convert(scanner.nextLine()));
+ }
+ this.expectedSize = this.expectedDatums.size();
+ }
+
+ @Override
+ public void cleanUp() {
+ assertEquals("Did not received the expected number of StreamsDatums", this.expectedSize, this.counted);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/FileReaderProvider.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/FileReaderProvider.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/FileReaderProvider.java
new file mode 100644
index 0000000..a7c98c9
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/FileReaderProvider.java
@@ -0,0 +1,98 @@
+package org.apache.streams.test.component;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProvider;
+import org.apache.streams.core.StreamsResultSet;
+import org.joda.time.DateTime;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.math.BigInteger;
+import java.sql.ResultSet;
+import java.util.Iterator;
+import java.util.Queue;
+import java.util.Scanner;
+
+/**
+ * FOR TESTING PURPOSES ONLY.
+ *
+ * The StreamProvider reads from a File or InputStream. Each line of the file will be emitted as the document of a
+ * streams datum.
+ *
+ */
+public class FileReaderProvider implements StreamsProvider {
+
+ private String fileName;
+ private InputStream inStream;
+ private Scanner scanner;
+ private StreamsDatumConverter converter;
+
+ public FileReaderProvider(String filePathInResources, StreamsDatumConverter converter) {
+ this.fileName = filePathInResources;
+ this.converter = converter;
+ }
+
+ @Override
+ public void startStream() {
+
+ }
+
+ @Override
+ public StreamsResultSet readCurrent() {
+ return new ResultSet();
+ }
+
+ @Override
+ public StreamsResultSet readNew(BigInteger sequence) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public StreamsResultSet readRange(DateTime start, DateTime end) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+ this.scanner = new Scanner(FileReaderProvider.class.getResourceAsStream(this.fileName));
+ }
+
+ @Override
+ public void cleanUp() {
+ this.scanner.close();
+ }
+
+ private class ResultSet extends StreamsResultSet {
+
+ public ResultSet() {
+ super(null);
+ }
+
+
+ @Override
+ public Iterator<StreamsDatum> iterator() {
+ return new FileProviderIterator();
+ }
+
+ private class FileProviderIterator implements Iterator<StreamsDatum> {
+
+
+
+ @Override
+ public boolean hasNext() {
+ return scanner.hasNextLine();
+ }
+
+ @Override
+ public StreamsDatum next() {
+ return converter.convert(scanner.nextLine());
+ }
+
+ @Override
+ public void remove() {
+
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/StreamsDatumConverter.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/StreamsDatumConverter.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/StreamsDatumConverter.java
new file mode 100644
index 0000000..d6ec2f4
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/StreamsDatumConverter.java
@@ -0,0 +1,13 @@
+package org.apache.streams.test.component;
+
+import org.apache.streams.core.StreamsDatum;
+
+import java.io.Serializable;
+
+/**
+ * Created by rebanks on 2/27/14.
+ */
+public interface StreamsDatumConverter extends Serializable {
+
+ public StreamsDatum convert(String s);
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/StringToDocumentConverter.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/StringToDocumentConverter.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/StringToDocumentConverter.java
new file mode 100644
index 0000000..d97de91
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/StringToDocumentConverter.java
@@ -0,0 +1,15 @@
+package org.apache.streams.test.component;
+
+import org.apache.streams.core.StreamsDatum;
+
+/**
+ * Created by rebanks on 2/28/14.
+ */
+public class StringToDocumentConverter implements StreamsDatumConverter {
+
+ @Override
+ public StreamsDatum convert(String s) {
+ return new StreamsDatum(s);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1993313b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestComponentsLocalStream.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestComponentsLocalStream.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestComponentsLocalStream.java
new file mode 100644
index 0000000..c7d8511
--- /dev/null
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestComponentsLocalStream.java
@@ -0,0 +1,23 @@
+package org.apache.streams.test.component.tests;
+
+import org.apache.streams.local.builders.LocalStreamBuilder;
+import org.apache.streams.test.component.ExpectedDatumsPersistWriter;
+import org.apache.streams.test.component.FileReaderProvider;
+import org.apache.streams.test.component.StringToDocumentConverter;
+import org.junit.Test;
+
+/**
+ * Created by rebanks on 2/28/14.
+ */
+public class TestComponentsLocalStream {
+
+ @Test
+ public void testLocalStreamWithComponent() {
+ LocalStreamBuilder builder = new LocalStreamBuilder();
+ builder.newReadCurrentStream("provider", new FileReaderProvider("/TestFile.txt",
+ new StringToDocumentConverter()));
+ builder.addStreamsPersistWriter("writer", new ExpectedDatumsPersistWriter(new StringToDocumentConverter(),
+ "/TestFile.txt"), 1, "provider")
+ .start();
+ }
+}
[10/13] dropping streams-pojo-extensions,
BC activity can be extended without it move streams cassandra under
contrib move streams-eip-routes under runtimes,
renaming streams-runtime-webapp
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/src/test/java/org/apache/streams/messaging/service/impl/CassandraActivityServiceTest.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/src/test/java/org/apache/streams/messaging/service/impl/CassandraActivityServiceTest.java b/streams-runtimes/streams-runtime-webapp/src/test/java/org/apache/streams/messaging/service/impl/CassandraActivityServiceTest.java
new file mode 100644
index 0000000..0812c47
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/src/test/java/org/apache/streams/messaging/service/impl/CassandraActivityServiceTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.messaging.service.impl;
+
+import org.apache.camel.Exchange;
+import org.apache.camel.Message;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import static org.easymock.EasyMock.*;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+
+public class CassandraActivityServiceTest {
+
+ private CassandraActivityService cassandraActivityService;
+
+ @Before
+ public void setup(){
+// cassandraActivityService = new CassandraActivityService();
+ }
+
+ @Ignore
+ @Test
+ public void getActivititiesForFilterTest(){
+ List<String> activities = cassandraActivityService.getActivitiesForFilters(Arrays.asList("r501"), new Date(0));
+ }
+
+ @Ignore
+ @Test
+ public void receiveExchangeTest(){
+ Exchange e = createMock(Exchange.class);
+ List<Exchange> grouped = new ArrayList<Exchange>();
+ Exchange e2 = createMock(Exchange.class);
+ grouped.add(e2);
+ Message m = createMock(Message.class);
+
+ String activityJson = "{\n" +
+ "\"id\":\"id2\",\n" +
+ "\"verb\":\"verb2\",\n" +
+ "\"displayName\":\"displayname2\",\n" +
+ "\"target\":{\n" +
+ "\t\"id\":\"targetid2\",\n" +
+ "\t\"displayName\":\"targetname2\"\n" +
+ "\t},\n" +
+ "\t\"object\":{\n" +
+ "\t\"id\":\"objectid2\",\n" +
+ "\t\"displayName\":\"objectname2\"\n" +
+ "\t},\n" +
+ "\t\"actor\":{\n" +
+ "\t\"id\":\"actorid2\",\n" +
+ "\t\"displayName\":\"actorname2\"\n" +
+ "\t}\n" +
+ "\t\n" +
+ "\t}";
+
+ expect(e.getProperty(Exchange.GROUPED_EXCHANGE, List.class)).andReturn(grouped);
+ expect(e2.getIn()).andReturn(m);
+ expect(m.getBody(String.class)).andReturn(activityJson);
+
+ replay(e, e2, m);
+
+ cassandraActivityService.receiveExchange(e);
+ //List<String> myTest = cassandraActivityService.getActivitiesForQuery("select * from coltest");
+ }
+}
[11/13] dropping streams-pojo-extensions,
BC activity can be extended without it move streams cassandra under
contrib move streams-eip-routes under runtimes,
renaming streams-runtime-webapp
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-pojo-extensions/src/test/java/org/apache/streams/pojo/test/ActivityExtendedSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-pojo-extensions/src/test/java/org/apache/streams/pojo/test/ActivityExtendedSerDeTest.java b/streams-pojo-extensions/src/test/java/org/apache/streams/pojo/test/ActivityExtendedSerDeTest.java
deleted file mode 100644
index 5abd42d..0000000
--- a/streams-pojo-extensions/src/test/java/org/apache/streams/pojo/test/ActivityExtendedSerDeTest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.pojo.json.test;
-
-import com.google.common.base.Joiner;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.input.BoundedInputStream;
-import org.apache.streams.pojo.json.ActivityExtended;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.Extensions;
-import org.codehaus.jackson.map.DeserializationConfig;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.Ignore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.InputStream;
-import java.util.Map;
-
-/**
- * Created with IntelliJ IDEA.
- * User: sblackmon
- * Date: 8/20/13
- * Time: 5:57 PM
- * To change this template use File | Settings | File Templates.
- */
-public class ActivityExtendedSerDeTest {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(ActivityExtendedSerDeTest.class);
-
- private ObjectMapper mapper = new ObjectMapper();
-
- @Test
- public void TestActivity()
- {
- mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- mapper.configure(DeserializationConfig.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationConfig.Feature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- InputStream is = ActivityExtendedSerDeTest.class.getResourceAsStream("/gnip_twitter_extended.json");
- Joiner joiner = Joiner.on(" ").skipNulls();
- is = new BoundedInputStream(is, 10000);
- String json;
- try {
- json = joiner.join(IOUtils.readLines(is));
- LOGGER.debug(json);
-
- Activity ser = mapper.readValue(json, Activity.class);
-
- String des = mapper.writeValueAsString(ser);
- LOGGER.debug(des);
-
- } catch( Exception e ) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-
- @Ignore
- @Test
- public void TestActivityExtended()
- {
- mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- mapper.configure(DeserializationConfig.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationConfig.Feature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- InputStream is = ActivityExtendedSerDeTest.class.getResourceAsStream("/gnip_twitter_extended.json");
- Joiner joiner = Joiner.on(" ").skipNulls();
- is = new BoundedInputStream(is, 10000);
- String json;
- try {
- json = joiner.join(IOUtils.readLines(is));
- LOGGER.debug(json);
-
- ActivityExtended ser = mapper.readValue(json, ActivityExtended.class);
-
- Extensions extensions = ser.getExtensions();
-
- String des = mapper.writeValueAsString(extensions);
-
- Assert.assertTrue(extensions.getAdditionalProperties().size() > 0);
- LOGGER.debug(des);
-
- } catch( Exception e ) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-pojo-extensions/src/test/resources/gnip_twitter_extended.json
----------------------------------------------------------------------
diff --git a/streams-pojo-extensions/src/test/resources/gnip_twitter_extended.json b/streams-pojo-extensions/src/test/resources/gnip_twitter_extended.json
deleted file mode 100644
index ee764cb..0000000
--- a/streams-pojo-extensions/src/test/resources/gnip_twitter_extended.json
+++ /dev/null
@@ -1,146 +0,0 @@
-{
- "gnip": {
- "matching_rules": [
- {
- "tag": "cascade_CA_CA_en"
- }
- ],
- "klout_score": 10,
- "urls": [
- {
- "expanded_url": "https://itunes.apple.com/us/app/smurfs-village/id399648212?mt=8",
- "url": "http://t.co/Ytn45Pbttk"
- }
- ],
- "klout_profile": {
- "topics": [],
- "klout_user_id": "257268143479895040",
- "link": "http://klout.com/user/id/257268143479895040"
- },
- "language": {
- "value": "fr"
- }
- },
- "body": "Le Grand Schtroumpf confirme que la cascade magique n'est \"Plus tr?�s loin.\" http://t.co/Ytn45Pbttk #SmurfsVillage @BeelineGames",
- "favoritesCount": 0,
- "link": "http://twitter.com/spoffff/statuses/372802927385403392",
- "retweetCount": 0,
- "twitter_lang": "fr",
- "postedTime": "2013-08-28T19:28:38.000Z",
- "provider": {
- "id": "{link}",
- "upstreamDuplicates": [],
- "link": "http://www.twitter.com",
- "attachments": [],
- "displayName": "Twitter",
- "objectType": "service",
- "downstreamDuplicates": []
- },
- "links": [],
- "actor": {
- "twitterTimeZone": "Brussels",
- "friendsCount": 6,
- "favoritesCount": 0,
- "link": "http://www.twitter.com/spoffff",
- "image": {
- "url": "https://si0.twimg.com/sticky/default_profile_images/default_profile_1_normal.png"
- },
- "postedTime": "2012-05-18T15:14:35.000Z",
- "links": [
- {
- "rel": "me",
- "href": null
- }
- ],
- "listedCount": 0,
- "downstreamDuplicates": [],
- "id": "id:twitter.com:583891967",
- "upstreamDuplicates": [],
- "languages": [
- "fr"
- ],
- "verified": false,
- "utcOffset": "7200",
- "followersCount": 0,
- "attachments": [],
- "displayName": "Sabine Chappuis",
- "preferredUsername": "spoffff",
- "statusesCount": 87,
- "objectType": "person"
- },
- "object": {
- "id": "object:search.twitter.com,2005:372802927385403392",
- "summary": "Le Grand Schtroumpf confirme que la cascade magique n'est \"Plus tr?�s loin.\" http://t.co/Ytn45Pbttk #SmurfsVillage @BeelineGames",
- "upstreamDuplicates": [],
- "link": "http://twitter.com/spoffff/statuses/372802927385403392",
- "postedTime": "2013-08-28T19:28:38.000Z",
- "attachments": [],
- "objectType": "note",
- "downstreamDuplicates": []
- },
- "twitter_entities": {
- "symbols": [],
- "urls": [
- {
- "expanded_url": "http://bit.ly/hUmoRz",
- "indices": [
- 77,
- 99
- ],
- "display_url": "bit.ly/hUmoRz",
- "url": "http://t.co/Ytn45Pbttk"
- }
- ],
- "hashtags": [
- {
- "text": "SmurfsVillage",
- "indices": [
- 100,
- 114
- ]
- }
- ],
- "user_mentions": [
- {
- "id": 188075479,
- "name": "Beeline Interactive",
- "indices": [
- 115,
- 128
- ],
- "screen_name": "BeelineGames",
- "id_str": "188075479"
- }
- ]
- },
- "extensions": {
- "w2o": {
- "tags": [
- "brand-cascade",
- "language-en",
- "country-ca"
- ],
- "provider": "twitter",
- "analyzer": "romance_analyzer",
- "lang": {
- "primaryLanguage": "en"
- }
- }
- },
- "twitter_filter_level": "medium",
- "guid": "A8fccSz7rpKfDJY078VLyw==_201308",
- "content": "Le Grand Schtroumpf confirme que la cascade magique n'est \"Plus tr?�s loin.\" http://t.co/Ytn45Pbttk #SmurfsVillage @BeelineGames",
- "id": "tag:search.twitter.com,2005:372802927385403392",
- "verb": "post",
- "generator": {
- "id": "{link}",
- "upstreamDuplicates": [],
- "link": "https://itunes.apple.com/us/app/smurfs-village/id399648212?mt=8&uo=4",
- "attachments": [],
- "displayName": "Smurfs' Village on iOS",
- "downstreamDuplicates": []
- },
- "published": "2013-08-28T19:28:38Z",
- "objectType": "activity"
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/pom.xml
----------------------------------------------------------------------
diff --git a/streams-runtimes/pom.xml b/streams-runtimes/pom.xml
index cf0d57f..1abe74e 100644
--- a/streams-runtimes/pom.xml
+++ b/streams-runtimes/pom.xml
@@ -35,5 +35,6 @@
<modules>
<module>streams-runtime-local</module>
<module>streams-runtime-storm</module>
+ <module>streams-runtime-webapp</module>
</modules>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/ReadMe.txt
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/ReadMe.txt b/streams-runtimes/streams-runtime-webapp/ReadMe.txt
new file mode 100644
index 0000000..19a1d19
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/ReadMe.txt
@@ -0,0 +1,32 @@
+Camel Router WAR Project with Web Console and REST Support
+==========================================================
+
+This project bundles the Camel Web Console, REST API, and some
+sample routes as a WAR. You can build the WAR by running
+
+mvn install
+
+You can then run the project by dropping the WAR into your
+favorite web container or just run
+
+mvn jetty:run
+
+to start up and deploy to Jetty.
+
+
+Web Console
+===========
+
+You can view the Web Console by pointing your browser to http://localhost:8080/
+
+You should be able to do things like
+
+ * browse the available endpoints
+ * browse the messages on an endpoint if it is a BrowsableEndpoint
+ * send a message to an endpoint
+ * create new endpoints
+
+For more help see the Apache Camel documentation
+
+ http://camel.apache.org/
+
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/pom.xml
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/pom.xml b/streams-runtimes/streams-runtime-webapp/pom.xml
new file mode 100644
index 0000000..8753bb7
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/pom.xml
@@ -0,0 +1,209 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://maven.apache.org/POM/4.0.0"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-runtimes</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>streams-runtime-webapp</artifactId>
+
+ <name>${bundle.symbolicName} [${bundle.namespace}]</name>
+
+ <properties>
+ <bundle.symbolicName>streams-eip-routes</bundle.symbolicName>
+ <bundle.namespace>org.apache.streams</bundle.namespace>
+ <jackson.old.version>1.9.11</jackson.old.version>
+ </properties>
+
+ <packaging>bundle</packaging>
+
+ <repositories>
+ <repository>
+ <id>clojars.org</id>
+ <url>http://clojars.org/repo</url>
+ </repository>
+ </repositories>
+
+ <build>
+
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ <!--
+ | example additional resource entries, useful when building Eclipse RCP applications
+ -->
+ <resource>
+ <directory>.</directory>
+ <includes>
+ <include>plugin.xml</include>
+ <include>plugin.properties</include>
+ <include>icons/**</include>
+ <include>META-INF/*</include>
+ </includes>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.ops4j</groupId>
+ <artifactId>maven-pax-plugin</artifactId>
+ <!--
+ | enable improved OSGi compilation support for the bundle life-cycle.
+ | to switch back to the standard bundle life-cycle, move this setting
+ | down to the maven-bundle-plugin section
+ -->
+ <extensions>true</extensions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <version>1.4.3</version>
+ <!--
+ | the following instructions build a simple set of public/private classes into an OSGi bundle
+ -->
+ <configuration>
+ <instructions>
+ <Bundle-SymbolicName>${bundle.symbolicName}</Bundle-SymbolicName>
+ <Bundle-Version>${project.version}</Bundle-Version>
+ <Export-Package>${bundle.namespace};version="${project.version}",org.apache.streams.messaging.configuration,org.apache.streams.messaging.routers,org.apache.streams.messaging.rules,org.apache.streams.messaging.processors,org.apache.streams.messaging.aggregation,org.apache.streams.messaging.service, org.apache.streams.messaging.storm,org.apache.activemq,org.codehaus.jackson.*;version="${jackson.version}"</Export-Package>
+ <Private-Package>${bundle.namespace}.messaging.routers.impl.*,${bundle.namespace}.messaging.rules.impl.*, ${bundle.namespace}.messaging.service.impl.*</Private-Package>
+ <Import-Package>org.apache.camel.*;version="2.8.5",org.apache.streams.messaging.configuration,org.apache.activemq.camel.component,org.apache.activemq,org.apache.activemq.pool,org.apache.camel.component.jms,org.springframework.*;version="3.0.6.RELEASE",org.apache.commons.logging,org.apache.streams.*,org.apache.streams.osgi.components,org.apache.streams.osgi.components.activityconsumer,org.apache.streams.osgi.components.activityconsumer.impl,org.apache.streams.osgi.components.activitysubscriber,org.apache.streams.osgi.components.activitysubscriber.impl,org.apache.streams.messaging.processors,org.apache.streams.messaging.aggregation,javax.jms, javax.net.ssl, javax.transaction.xa, org.apache.activemq.advisory, org.apache.activemq.blob, org.apache.activemq.broker, org.apache.activemq.broker.region, org.apache.activemq.command, org.apache.activemq.filter, org.apache.activemq.jndi, org.apache.activemq.management, org.apache.activemq.selector, org.apache.activemq.sta
te, org.apache.activemq.thread, org.apache.activemq.transaction, org.apache.activemq.transport, org.apache.activemq.transport.failover, org.apache.activemq.transport.tcp, org.apache.activemq.usage, org.apache.activemq.util, org.slf4j,org.codehaus.jackson;version="${jackson.version}",javax.xml.datatype, javax.xml.namespace, javax.xml.parsers, org.joda.time, org.joda.time.format, org.w3c.dom, org.w3c.dom.bootstrap, org.w3c.dom.ls, org.xml.sax, org.apache.rave.model, org.apache.rave.portal.model.impl, backtype.storm, backtype.storm.coordination, backtype.storm.generated, backtype.storm.spout, backtype.storm.task, backtype.storm.topology, backtype.storm.topology.base, backtype.storm.tuple, javax.annotation, backtype.storm.utils</Import-Package>
+ </instructions>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-core</artifactId>
+ <version>2.9.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.activemq</groupId>
+ <artifactId>activemq-camel</artifactId>
+ <version>5.5.1</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.activemq</groupId>
+ <artifactId>activemq-pool</artifactId>
+ <version>5.5.1</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-core</artifactId>
+ <version>3.0.6.RELEASE</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ <version>3.0.6.RELEASE</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mrbean</artifactId>
+ <version>${jackson.old.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ <version>${jackson.old.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.osgi</groupId>
+ <artifactId>osgi_R4_core</artifactId>
+ <version>1.0</version>
+ <scope>provided</scope>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.osgi</groupId>
+ <artifactId>osgi_R4_compendium</artifactId>
+ <version>1.0</version>
+ <scope>provided</scope>
+ <optional>true</optional>
+ </dependency>
+
+
+ <dependency>
+ <groupId>org.apache.streams.osgi.components</groupId>
+ <artifactId>activity-registration</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.streams.osgi.components</groupId>
+ <artifactId>activity-consumer</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.streams.osgi.components</groupId>
+ <artifactId>activity-subscriber</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-cassandra</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.rave</groupId>
+ <artifactId>rave-core</artifactId>
+ <version>${rave.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.rave</groupId>
+ <artifactId>rave-core-api</artifactId>
+ <version>${rave.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>storm</groupId>
+ <artifactId>storm</artifactId>
+ <version>0.8.2</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
+ <version>3.1</version>
+ </dependency>
+ </dependencies>
+
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/aggregation/ActivityAggregator.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/aggregation/ActivityAggregator.java b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/aggregation/ActivityAggregator.java
new file mode 100644
index 0000000..dc7ba0c
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/aggregation/ActivityAggregator.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.messaging.aggregation;
+
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.streams.messaging.service.impl.CassandraActivityService;
+import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscriber;
+import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscriberWarehouse;
+import org.springframework.scheduling.annotation.Scheduled;
+
+import java.util.*;
+
+public class ActivityAggregator {
+
+ private ActivityStreamsSubscriberWarehouse activityStreamsSubscriberWarehouse;
+ private CassandraActivityService activityService;
+ private static final transient Log LOG = LogFactory.getLog(ActivityAggregator.class);
+
+ public void setActivityStreamsSubscriberWarehouse(ActivityStreamsSubscriberWarehouse activityStreamsSubscriberWarehouse) {
+ this.activityStreamsSubscriberWarehouse = activityStreamsSubscriberWarehouse;
+ }
+
+ public void setActivityService(CassandraActivityService activityService) {
+ this.activityService = activityService;
+ }
+
+ @Scheduled(fixedRate=30000)
+ public void distributeToSubscribers() {
+ for (ActivityStreamsSubscriber subscriber : activityStreamsSubscriberWarehouse.getAllSubscribers()) {
+ updateSubscriber(subscriber);
+ }
+ }
+
+ public void updateSubscriber(ActivityStreamsSubscriber subscriber){
+ Set<String> activities = new TreeSet<String>();
+ activities.addAll(activityService.getActivitiesForFilters(subscriber.getActivityStreamsSubscriberConfiguration().getFilters(), subscriber.getLastUpdated()));
+ //TODO: an activity posted in between the cql query and setting the lastUpdated field will be lost
+ subscriber.setLastUpdated(new Date());
+ subscriber.receive(new ArrayList<String>(activities));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/configuration/EipConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/configuration/EipConfigurator.java b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/configuration/EipConfigurator.java
new file mode 100644
index 0000000..460c43a
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/configuration/EipConfigurator.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.messaging.configuration;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class EipConfigurator {
+
+
+
+ @Value("${consumer.inRouteHost}")
+ private String consumerInRouteHost;
+
+ @Value("${consumer.inRoutePort}")
+ private String consumerInRoutePort;
+
+ @Value("${subscriber.inRouteHost}")
+ private String subscriberInRouteHost;
+
+ @Value("${subscriber.inRoutePort}")
+ private String subscriberInRoutePort;
+
+
+ @Value("${consumer.activityQUri}")
+ private String consumerActivityQUri;
+
+ @Value("${consumer.publisherEndpointProtocol}")
+ private String publisherEndpointProtocol;
+
+ @Value("${consumer.publisherEndpointUrlResource}")
+ private String publisherEndpointUrlResource;
+
+ @Value("${consumer.receiveMethod}")
+ private String consumerReceiveMethod;
+
+ @Value("${consumer.splitMethod}")
+ private String consumerSplitMethod;
+
+ @Value("${subscriber.subscriberEndpointProtocol}")
+ private String subscriberEndpointProtocol;
+
+ @Value("${subscriber.subscriberEndpointUrlResource}")
+ private String subscriberEndpointUrlResource;
+
+ @Value("${subscriber.receiveMethod}")
+ private String subscriberReceiveMethod;
+
+ @Value("${subscriber.postMethod}")
+ private String subscriberPostMethod;
+
+ @Value("${subscriber.getMethod}")
+ private String subscriberGetMethod;
+
+
+ @Value("${servlet.baseUrlPath}")
+ private String baseUrlPath;
+
+
+ public static String ENDPOINT_PROTOCOL_JETTY="jetty:http://";
+ public static String ENDPOINT_PROTOCOL_SERVLET="servlet:///";
+
+ public String getConsumerInRouteHost() {
+ return consumerInRouteHost;
+ }
+
+ public String getConsumerInRoutePort() {
+ return consumerInRoutePort;
+ }
+
+ public String getConsumerActivityQUri() {
+ return consumerActivityQUri;
+ }
+
+ public void setConsumerActivityQUri(String consumerActivityQUri) {
+ this.consumerActivityQUri = consumerActivityQUri;
+ }
+
+ public void setConsumerInRoutePort(String consumerInRoutePort) {
+ this.consumerInRoutePort = consumerInRoutePort;
+ }
+
+ public void setConsumerInRouteHost(String consumerInRouteHost) {
+ this.consumerInRouteHost = consumerInRouteHost;
+ }
+
+ public String getSubscriberInRouteHost() {
+ return subscriberInRouteHost;
+ }
+
+ public void setSubscriberInRouteHost(String subscriberInRouteHost) {
+ this.subscriberInRouteHost = subscriberInRouteHost;
+ }
+
+ public String getSubscriberInRoutePort() {
+ return subscriberInRoutePort;
+ }
+
+ public void setSubscriberInRoutePort(String subscriberInRoutePort) {
+ this.subscriberInRoutePort = subscriberInRoutePort;
+ }
+
+ public String getPublisherEndpointProtocol() {
+ return publisherEndpointProtocol;
+ }
+
+ public void setPublisherEndpointProtocol(String publisherEndpointProtocol) {
+ this.publisherEndpointProtocol = publisherEndpointProtocol;
+ }
+
+ public String getPublisherEndpointUrlResource() {
+ return publisherEndpointUrlResource;
+ }
+
+ public void setPublisherEndpointUrlResource(String publisherEndpointUrlResource) {
+ this.publisherEndpointUrlResource = publisherEndpointUrlResource;
+ }
+
+ public String getConsumerReceiveMethod() {
+ return consumerReceiveMethod;
+ }
+
+ public void setConsumerReceiveMethod(String consumerReceiveMethod) {
+ this.consumerReceiveMethod = consumerReceiveMethod;
+ }
+
+ public String getConsumerSplitMethod() {
+ return consumerSplitMethod;
+ }
+
+ public void setConsumerSplitMethod(String consumerSplitMethod) {
+ this.consumerSplitMethod = consumerSplitMethod;
+ }
+
+ public String getSubscriberEndpointProtocol() {
+ return subscriberEndpointProtocol;
+ }
+
+ public void setSubscriberEndpointProtocol(String subscriberEndpointProtocol) {
+ this.subscriberEndpointProtocol = subscriberEndpointProtocol;
+ }
+
+ public String getSubscriberEndpointUrlResource() {
+ return subscriberEndpointUrlResource;
+ }
+
+ public void setSubscriberEndpointUrlResource(String subscriberEndpointUrlResource) {
+ this.subscriberEndpointUrlResource = subscriberEndpointUrlResource;
+ }
+
+ public String getSubscriberReceiveMethod() {
+ return subscriberReceiveMethod;
+ }
+
+ public void setSubscriberReceiveMethod(String subscriberReceiveMethod) {
+ this.subscriberReceiveMethod = subscriberReceiveMethod;
+ }
+
+ public String getSubscriberPostMethod() {
+ return subscriberPostMethod;
+ }
+
+ public void setSubscriberPostMethod(String subscriberPostMethod) {
+ this.subscriberPostMethod = subscriberPostMethod;
+ }
+
+ public String getSubscriberGetMethod() {
+ return subscriberGetMethod;
+ }
+
+ public void setSubscriberGetMethod(String subscriberGetMethod) {
+ this.subscriberGetMethod = subscriberGetMethod;
+ }
+
+ public String getBaseUrlPath() {
+ return baseUrlPath;
+ }
+
+ public void setBaseUrlPath(String baseUrlPath) {
+ this.baseUrlPath = baseUrlPath;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/processors/ActivityPublisherRegistrationProcessor.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/processors/ActivityPublisherRegistrationProcessor.java b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/processors/ActivityPublisherRegistrationProcessor.java
new file mode 100644
index 0000000..741a63c
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/processors/ActivityPublisherRegistrationProcessor.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.messaging.processors;
+
+import org.apache.camel.Exchange;
+import org.apache.camel.Processor;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.streams.osgi.components.activityconsumer.ActivityConsumer;
+import org.codehaus.jackson.map.DeserializationConfig;
+import org.codehaus.jackson.map.ObjectMapper;
+
+
+public class ActivityPublisherRegistrationProcessor implements Processor{
+ private static final transient Log LOG = LogFactory.getLog(ActivityStreamsSubscriberRegistrationProcessor.class);
+ public void process(Exchange exchange){
+ //add the necessary headers to the message so that the activity registration component
+ //can do a lookup to either make a new processor and endpoint, or pass the message to the right one
+ String httpMethod = exchange.getIn().getHeader("CamelHttpMethod").toString();
+
+ if (!httpMethod.equals("POST")){
+ //reject anything that isn't a post...Camel 2.10 solves needing this check, however, SM 4.4 doesn't have the latest
+ exchange.getOut().setFault(true);
+ exchange.getOut().setHeader(Exchange.HTTP_RESPONSE_CODE,405);
+ } else {
+
+ //for now...just expect a post with a uri in the body...should have some checking here with http response codes
+ // authentication, all that good stuff...happens in the registration module
+
+ String body = exchange.getIn().getBody(String.class);
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES,false);
+
+ try {
+
+ // read from file, convert it to user class
+ ActivityConsumer configuration = mapper.readValue(body, ActivityConsumer.class);
+ if (configuration.getSrc()==null){
+ LOG.info("configuration src is null");
+ throw new Exception();
+ }
+
+ exchange.getOut().setBody(configuration);
+
+ } catch (Exception e) {
+ LOG.info("error: " + e);
+ exchange.getOut().setFault(true);
+ exchange.getOut().setHeader(Exchange.HTTP_RESPONSE_CODE,400);
+ exchange.getOut().setBody("POST should contain a valid JSON configuration for registering as an Activity Publisher (check that src element is valid).");
+ }
+ }
+
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/processors/ActivityStreamsSubscriberRegistrationProcessor.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/processors/ActivityStreamsSubscriberRegistrationProcessor.java b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/processors/ActivityStreamsSubscriberRegistrationProcessor.java
new file mode 100644
index 0000000..201eebd
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/processors/ActivityStreamsSubscriberRegistrationProcessor.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.messaging.processors;
+
+import org.apache.camel.Exchange;
+import org.apache.camel.Processor;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.streams.messaging.service.SubscriptionService;
+import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscription;
+import org.codehaus.jackson.map.DeserializationConfig;
+import org.codehaus.jackson.map.ObjectMapper;
+
+
+public class ActivityStreamsSubscriberRegistrationProcessor implements Processor{
+ private static final transient Log LOG = LogFactory.getLog(ActivityStreamsSubscriberRegistrationProcessor.class);
+ private SubscriptionService subscriptionService;
+
+ public ActivityStreamsSubscriberRegistrationProcessor(SubscriptionService subscriptionService){
+ this.subscriptionService = subscriptionService;
+ }
+
+ public void process(Exchange exchange){
+ LOG.info("processing the subscriber...");
+ //add the necessary headers to the message so that the activity registration component
+ //can do a lookup to either make a new processor and endpoint, or pass the message to the right one
+ String httpMethod = exchange.getIn().getHeader("CamelHttpMethod").toString();
+
+ if (!httpMethod.equals("POST")){
+ //reject anything that isn't a post...Camel 2.10 solves needing this check, however, SM 4.4 doesn't have the latest
+ exchange.getOut().setFault(true);
+ exchange.getOut().setHeader(Exchange.HTTP_RESPONSE_CODE,405);
+ } else {
+
+ //for now...just expect a post with a uri in the body...should have some checking here with http response codes
+ // authentication, all that good stuff...happens in the registration module
+
+
+ String body = exchange.getIn().getBody(String.class);
+
+ LOG.info("receiving the subscriber: "+body);
+ //OAuth token? What does subscriber post to init a subscription URL?
+ //maybe its a list of URLs to subscribe to subscriptions=1,2,3,4&auth_token=XXXX
+
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES,false);
+
+ try {
+
+ // read from file, convert it to user class
+ ActivityStreamsSubscription configuration = mapper.readValue(body, ActivityStreamsSubscription.class);
+ if(configuration.getFilters() == null){
+ configuration.setFilters(subscriptionService.getFilters(configuration.getAuthToken()));
+ }else{
+ subscriptionService.saveFilters(configuration);
+ }
+ exchange.getOut().setBody(configuration);
+
+ } catch (Exception e) {
+ LOG.info("exception" + e);
+ exchange.getOut().setFault(true);
+ exchange.getOut().setHeader(Exchange.HTTP_RESPONSE_CODE,400);
+ exchange.getOut().setBody("POST should contain a valid Subscription configuration object.");
+ }
+
+
+
+ //just pass this on to the route creator, body will be the dedicated URL for this subscriber
+
+ }
+
+
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/routers/ActivityConsumerRouteBuilder.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/routers/ActivityConsumerRouteBuilder.java b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/routers/ActivityConsumerRouteBuilder.java
new file mode 100644
index 0000000..dea8781
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/routers/ActivityConsumerRouteBuilder.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.messaging.routers;
+
+
+
+import org.apache.camel.Exchange;
+import org.apache.streams.osgi.components.activityconsumer.ActivityConsumer;
+
+
+public interface ActivityConsumerRouteBuilder {
+
+
+ void createNewRouteForConsumer(Exchange exchange, ActivityConsumer activityConsumer);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/routers/ActivityStreamsSubscriberRouteBuilder.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/routers/ActivityStreamsSubscriberRouteBuilder.java b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/routers/ActivityStreamsSubscriberRouteBuilder.java
new file mode 100644
index 0000000..6947722
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/routers/ActivityStreamsSubscriberRouteBuilder.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.messaging.routers;
+
+
+
+import org.apache.camel.Exchange;
+import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscriber;
+
+
+public interface ActivityStreamsSubscriberRouteBuilder {
+
+
+ void createNewRouteForSubscriber(Exchange exchange, ActivityStreamsSubscriber activityStreamsSubscriber);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/routers/impl/ActivityConsumerRouter.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/routers/impl/ActivityConsumerRouter.java b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/routers/impl/ActivityConsumerRouter.java
new file mode 100644
index 0000000..20b8246
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/routers/impl/ActivityConsumerRouter.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.messaging.routers.impl;
+
+
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.streams.messaging.routers.ActivityConsumerRouteBuilder;
+
+
+import org.apache.streams.osgi.components.activityconsumer.ActivityConsumerWarehouse;
+import org.apache.streams.osgi.components.activityconsumer.ActivityConsumer;
+import org.apache.streams.messaging.configuration.EipConfigurator;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.apache.camel.Exchange;
+import org.apache.camel.CamelContext;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.util.UUID;
+
+
+public class ActivityConsumerRouter extends RouteBuilder implements ActivityConsumerRouteBuilder {
+ private static final transient Log LOG = LogFactory.getLog(ActivityConsumerRouter.class);
+
+ @Autowired
+ private EipConfigurator configuration;
+
+ protected CamelContext camelContext;
+
+ private ActivityConsumerWarehouse activityConsumerWarehouse;
+
+ public void setCamelContext(CamelContext camelContext) {
+ this.camelContext = camelContext;
+ }
+
+ public void setActivityConsumerWarehouse(ActivityConsumerWarehouse activityConsumerWarehouse) {
+ this.activityConsumerWarehouse = activityConsumerWarehouse;
+ }
+
+
+ public void createNewRouteForConsumer(Exchange exchange, ActivityConsumer activityConsumer){
+
+ //todo: add some better scheme then getCount for URL...
+ //todo: make the route again if consumer exists...and context doesn't have route
+ if (activityConsumer.isAuthenticated()){
+ ActivityConsumer existingConsumer = activityConsumerWarehouse.findConsumerBySrc(activityConsumer.getSrc().toASCIIString());
+
+ if (existingConsumer==null){
+
+ try{
+
+ if (configuration.getPublisherEndpointProtocol().equals(EipConfigurator.ENDPOINT_PROTOCOL_JETTY)){
+ activityConsumer.setInRoute(configuration.getConsumerInRouteHost()+ ":" + configuration.getConsumerInRoutePort() +"/" + configuration.getPublisherEndpointUrlResource() + "/" + UUID.randomUUID());
+ //set the body to the url the producer should post to
+ exchange.getOut().setBody("http://" + activityConsumer.getInRoute());
+ }else if (configuration.getPublisherEndpointProtocol().equals(EipConfigurator.ENDPOINT_PROTOCOL_SERVLET)){
+ activityConsumer.setInRoute( configuration.getPublisherEndpointUrlResource() + "/" + UUID.randomUUID());
+ //set the body to the url the producer should post to
+ exchange.getOut().setBody(configuration.getBaseUrlPath() + activityConsumer.getInRoute());
+ } else{
+ throw new Exception("No supported endpoint protocol is configured.");
+ }
+
+
+ //setup a message queue for this consumer.getInRoute()
+ camelContext.addRoutes(new DynamicConsumerRouteBuilder(configuration,camelContext, configuration.getPublisherEndpointProtocol() + activityConsumer.getInRoute(), activityConsumer));
+
+
+ LOG.info("all messages sent from " + activityConsumer.getSrc() + " must be posted to " + activityConsumer.getInRoute());
+ //only add the route to the warehouse after its been created in messaging system...
+ activityConsumerWarehouse.register(activityConsumer);
+ }catch (Exception e){
+ exchange.getOut().setHeader(Exchange.HTTP_RESPONSE_CODE,500);
+ exchange.getOut().setBody("error creating route: " + e);
+ LOG.error("error creating route: " + e);
+ }
+
+ } else{
+
+ exchange.getOut().setBody(configuration.getBaseUrlPath() + existingConsumer.getInRoute());
+ }
+
+ }else{
+ exchange.getOut().setFault(true);
+ exchange.getOut().setHeader(Exchange.HTTP_RESPONSE_CODE,401);
+ exchange.getOut().setBody("Authentication failed.");
+ }
+
+ }
+
+
+ public void configure() throws java.lang.Exception{
+ //nothing...set the context?
+
+ }
+
+ /**
+ * This route builder is a skeleton to add new routes at runtime
+ */
+ private static final class DynamicConsumerRouteBuilder extends RouteBuilder {
+ private final String from;
+ private ActivityConsumer activityConsumer;
+
+
+ private EipConfigurator configuration;
+
+ private DynamicConsumerRouteBuilder(EipConfigurator configuration, CamelContext context, String from, ActivityConsumer activityConsumer) {
+ super(context);
+ this.from = from;
+ this.activityConsumer = activityConsumer;
+ this.configuration = configuration;
+ }
+
+ @Override
+ public void configure() throws Exception {
+
+
+ from(from)
+ .bean(activityConsumer, configuration.getConsumerReceiveMethod()).setBody(body())
+ .split()
+ .method(activityConsumer, configuration.getConsumerSplitMethod())
+ .to(configuration.getConsumerActivityQUri());
+
+
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/routers/impl/ActivityStreamsSubscriberRouter.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/routers/impl/ActivityStreamsSubscriberRouter.java b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/routers/impl/ActivityStreamsSubscriberRouter.java
new file mode 100644
index 0000000..68ef0a5
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/routers/impl/ActivityStreamsSubscriberRouter.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.messaging.routers.impl;
+
+
+import org.apache.camel.CamelContext;
+import org.apache.camel.Exchange;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.streams.messaging.aggregation.ActivityAggregator;
+import org.apache.streams.messaging.configuration.EipConfigurator;
+import org.apache.streams.messaging.routers.ActivityStreamsSubscriberRouteBuilder;
+import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscriber;
+import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscriberWarehouse;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.UUID;
+
+
+public class ActivityStreamsSubscriberRouter extends RouteBuilder implements ActivityStreamsSubscriberRouteBuilder {
+ private static final transient Log LOG = LogFactory.getLog(ActivityStreamsSubscriberRouter.class);
+
+ @Autowired
+ private EipConfigurator configuration;
+
+ protected CamelContext camelContext;
+
+ @Autowired
+ private ActivityAggregator activityAggregator;
+
+ private ActivityStreamsSubscriberWarehouse activityStreamsSubscriberWarehouse;
+
+ public void setCamelContext(CamelContext camelContext) {
+ this.camelContext = camelContext;
+ }
+
+ public void setActivityStreamsSubscriberWarehouse(ActivityStreamsSubscriberWarehouse activityStreamsSubscriberWarehouse) {
+ this.activityStreamsSubscriberWarehouse = activityStreamsSubscriberWarehouse;
+ }
+
+
+ public void createNewRouteForSubscriber(Exchange exchange, ActivityStreamsSubscriber activityStreamsSubscriber){
+
+ //todo: add some better scheme then getCount for URL...
+ //todo: make the route again if subscriber exists...and context doesn't have route
+ if (activityStreamsSubscriber.isAuthenticated()){
+
+ try{
+
+ if (configuration.getSubscriberEndpointProtocol().equals(EipConfigurator.ENDPOINT_PROTOCOL_JETTY)){
+ activityStreamsSubscriber.setInRoute(configuration.getSubscriberInRouteHost()+ ":" + configuration.getSubscriberInRoutePort() +"/" + configuration.getSubscriberEndpointUrlResource() + "/" + UUID.randomUUID());
+ //set the body to the url the producer should post to
+ exchange.getOut().setBody("http://" + activityStreamsSubscriber.getInRoute());
+ }else if (configuration.getSubscriberEndpointProtocol().equals(EipConfigurator.ENDPOINT_PROTOCOL_SERVLET)){
+ activityStreamsSubscriber.setInRoute( configuration.getSubscriberEndpointUrlResource() + "/" + UUID.randomUUID());
+ //set the body to the url the producer should post to
+ exchange.getOut().setBody(configuration.getBaseUrlPath() + activityStreamsSubscriber.getInRoute());
+ } else{
+ throw new Exception("No supported endpoint protocol is configured.");
+ }
+
+ //setup a message queue for this consumer.getInRoute()
+ camelContext.addRoutes(new DynamicSubscriberRouteBuilder(configuration,camelContext, configuration.getSubscriberEndpointProtocol() + activityStreamsSubscriber.getInRoute(), activityStreamsSubscriber));
+
+ activityAggregator.updateSubscriber(activityStreamsSubscriber);
+ activityStreamsSubscriberWarehouse.register(activityStreamsSubscriber);
+ }catch (Exception e){
+ exchange.getOut().setHeader(Exchange.HTTP_RESPONSE_CODE,500);
+ exchange.getOut().setBody("error creating route: " + e);
+ LOG.error("error creating route: " + e);
+ }
+
+ }else{
+ exchange.getOut().setFault(true);
+ exchange.getOut().setHeader(Exchange.HTTP_RESPONSE_CODE,401);
+ exchange.getOut().setBody("Authentication failed.");
+ }
+
+ }
+
+
+
+
+ public void configure() throws Exception{
+ //nothing...set the context?
+
+ }
+
+ /**
+ * This route builder is a skeleton to add new routes at runtime
+ */
+ private static final class DynamicSubscriberRouteBuilder extends RouteBuilder {
+ private final String from;
+ private ActivityStreamsSubscriber activityStreamsSubscriber;
+
+
+ private EipConfigurator configuration;
+
+ private DynamicSubscriberRouteBuilder(EipConfigurator configuration, CamelContext context, String from, ActivityStreamsSubscriber activityStreamsSubscriber) {
+ super(context);
+ this.from = from;
+ this.activityStreamsSubscriber = activityStreamsSubscriber;
+ this.configuration = configuration;
+ }
+
+ @Override
+ public void configure() throws Exception {
+
+
+ from(from)
+ .choice()
+ .when(header("CamelHttpMethod").isEqualTo("POST"))
+ //when its a post...it goes to adding a new src
+ .bean(activityStreamsSubscriber, configuration.getSubscriberPostMethod()).setBody(body())
+ .when(header("CamelHttpMethod").isEqualTo("GET"))
+ // when its a GET it goes to getStream()
+ .bean(activityStreamsSubscriber, configuration.getSubscriberGetMethod()) ;
+
+
+
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/service/ActivityService.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/service/ActivityService.java b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/service/ActivityService.java
new file mode 100644
index 0000000..0c85974
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/service/ActivityService.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.messaging.service;
+
+import org.apache.camel.Exchange;
+
+import java.util.Date;
+import java.util.List;
+
+public interface ActivityService {
+
+ void receiveExchange(Exchange exchange);
+
+ List<String> getActivitiesForFilters(List<String> filters, Date lastUpdated);
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/service/SubscriptionService.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/service/SubscriptionService.java b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/service/SubscriptionService.java
new file mode 100644
index 0000000..98f585d
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/service/SubscriptionService.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.messaging.service;
+
+import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscription;
+
+import java.util.List;
+
+public interface SubscriptionService {
+
+ List<String> getFilters(String authToken);
+ void saveFilters(ActivityStreamsSubscription subscription);
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/service/impl/CassandraActivityService.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/service/impl/CassandraActivityService.java b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/service/impl/CassandraActivityService.java
new file mode 100644
index 0000000..89f71ab
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/service/impl/CassandraActivityService.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.messaging.service.impl;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.camel.Exchange;
+import org.apache.rave.model.ActivityStreamsEntry;
+import org.apache.streams.cassandra.model.CassandraActivityStreamsEntry;
+import org.apache.streams.cassandra.repository.impl.CassandraActivityStreamsRepository;
+import org.apache.streams.messaging.service.ActivityService;
+import org.codehaus.jackson.map.DeserializationConfig;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+
+public class CassandraActivityService implements ActivityService {
+
+ private static final transient Log LOG = LogFactory.getLog(CassandraActivityService.class);
+
+ private CassandraActivityStreamsRepository cassandraActivityStreamsRepository;
+ private ObjectMapper mapper;
+
+ @Autowired
+ public CassandraActivityService(CassandraActivityStreamsRepository cassandraActivityStreamsRepository, ObjectMapper mapper) {
+ this.cassandraActivityStreamsRepository = cassandraActivityStreamsRepository;
+ this.mapper = mapper;
+ mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ @Override
+ public void receiveExchange(Exchange exchange) {
+
+ //receive the exchange as a list
+ List<Exchange> grouped = exchange.getProperty(Exchange.GROUPED_EXCHANGE, List.class);
+
+ for (Exchange e : grouped) {
+ //get activity off of exchange
+ LOG.info("Exchange: " + e);
+
+ //extract the ActivityStreamsEntry object and save it in the database
+ LOG.info("About to preform the translation to JSON Object");
+ String activityJson = e.getIn().getBody(String.class);
+
+ try {
+ ActivityStreamsEntry streamsEntry = mapper.readValue(activityJson, CassandraActivityStreamsEntry.class);
+ streamsEntry.setPublished(new Date());
+ cassandraActivityStreamsRepository.save(streamsEntry);
+ } catch (IOException err) {
+ LOG.error("there was an error while converting the json string to an object and saving to the database", err);
+ }
+
+ }
+ }
+
+ @Override
+ public List<String> getActivitiesForFilters(List<String> filters, Date lastUpdated) {
+ List<CassandraActivityStreamsEntry> activityObjects = cassandraActivityStreamsRepository.getActivitiesForFilters(filters, lastUpdated);
+ Collections.sort(activityObjects, Collections.reverseOrder());
+ //TODO: make the number of streams returned configurable
+ return getJsonList(activityObjects.subList(0,Math.min(activityObjects.size(),10)));
+ }
+
+ private List<String> getJsonList(List<CassandraActivityStreamsEntry> activities) {
+ List<String> jsonList = new ArrayList<String>();
+ for (ActivityStreamsEntry entry : activities) {
+ try {
+ jsonList.add(mapper.writeValueAsString(entry));
+ } catch (IOException e) {
+ LOG.error("There was an error while trying to convert the java object to a string: " + entry, e);
+ }
+ }
+ return jsonList;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/service/impl/CassandraSubscriptionService.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/service/impl/CassandraSubscriptionService.java b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/service/impl/CassandraSubscriptionService.java
new file mode 100644
index 0000000..8972d1e
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/src/main/java/org/apache/streams/messaging/service/impl/CassandraSubscriptionService.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.messaging.service.impl;
+
+import org.apache.streams.cassandra.repository.impl.CassandraSubscriptionRepository;
+import org.apache.streams.messaging.service.SubscriptionService;
+import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscription;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class CassandraSubscriptionService implements SubscriptionService {
+
+ private CassandraSubscriptionRepository repository;
+
+ public CassandraSubscriptionService(CassandraSubscriptionRepository repository){
+ this.repository = repository;
+ }
+
+ public List<String> getFilters(String authToken){
+ return Arrays.asList(repository.getFilters(authToken).split(" "));
+ }
+
+ public void saveFilters(ActivityStreamsSubscription subscription){
+ repository.save(subscription);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/spring/propertiesLoader.xml
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/spring/propertiesLoader.xml b/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/spring/propertiesLoader.xml
new file mode 100644
index 0000000..60a3f1f
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/spring/propertiesLoader.xml
@@ -0,0 +1,35 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<!--<beans-->
+ <!--xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"-->
+ <!--xmlns="http://www.springframework.org/schema/beans"-->
+ <!--xmlns:context="http://www.springframework.org/schema/context"-->
+ <!--xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd-->
+ <!--http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">-->
+
+
+
+ <!--<context:component-scan base-package="org.apache.streams.messaging" annotation-config="true"/>-->
+
+ <!--<context:property-placeholder location="/META-INF/streams.properties"/>-->
+
+ <!--<bean id="configuration" class="org.apache.streams.messaging.configuration.EipConfigurator" />-->
+
+
+<!--</beans>-->
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/spring/streams-eip-applicationContext.xml
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/spring/streams-eip-applicationContext.xml b/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/spring/streams-eip-applicationContext.xml
new file mode 100644
index 0000000..a9b97a7
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/spring/streams-eip-applicationContext.xml
@@ -0,0 +1,113 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<beans
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://www.springframework.org/schema/beans"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:task="http://www.springframework.org/schema/task"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+ http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task.xsd">
+
+
+ <bean id="activityConsumerRouter" class="org.apache.streams.messaging.routers.impl.ActivityConsumerRouter">
+ <property name="activityConsumerWarehouse" ref="activityConsumerWarehouse"/>
+ <property name="camelContext" ref="context"/>
+ </bean>
+
+ <bean id="activityRegistrationProcessor"
+ class="org.apache.streams.messaging.processors.ActivityPublisherRegistrationProcessor"/>
+
+
+ <bean id="activityStreamsSubscriberRouter"
+ class="org.apache.streams.messaging.routers.impl.ActivityStreamsSubscriberRouter">
+ <property name="activityStreamsSubscriberWarehouse" ref="activityStreamsSubscriberWarehouse"/>
+ <property name="camelContext" ref="context"/>
+ </bean>
+
+ <bean id="subscriberRegistrationProcessor"
+ class="org.apache.streams.messaging.processors.ActivityStreamsSubscriberRegistrationProcessor">
+ <constructor-arg ref="subscriptionService"/>
+ </bean>
+
+ <bean id="cassandraKeyspace" class="org.apache.streams.cassandra.repository.impl.CassandraKeyspace">
+ <constructor-arg ref="cassandraConfig"/>
+ </bean>
+
+ <bean id="cassandraActivityStreamsRepository"
+ class="org.apache.streams.cassandra.repository.impl.CassandraActivityStreamsRepository">
+ <constructor-arg ref="cassandraKeyspace"/>
+ <constructor-arg ref="cassandraConfig"/>
+ </bean>
+
+ <bean id="cassandraSubscriptionRepository"
+ class="org.apache.streams.cassandra.repository.impl.CassandraSubscriptionRepository">
+ <constructor-arg ref="cassandraKeyspace"/>
+ <constructor-arg ref="cassandraConfig"/>
+ </bean>
+
+ <bean id="objectMapper" class="org.codehaus.jackson.map.ObjectMapper"/>
+
+ <bean id="subscriptionService" class="org.apache.streams.messaging.service.impl.CassandraSubscriptionService">
+ <constructor-arg ref="cassandraSubscriptionRepository"/>
+ </bean>
+
+ <bean id="activityService" class="org.apache.streams.messaging.service.impl.CassandraActivityService">
+ <constructor-arg ref="cassandraActivityStreamsRepository"/>
+ <constructor-arg ref="objectMapper"/>
+ </bean>
+
+ <!--<bean id="stromActivityAggregator" class="org.apache.streams.messaging.storm.StormActivityAggregator">-->
+ <!--<constructor-arg ref="bolt"/>-->
+ <!--<constructor-arg ref="spout"/>-->
+ <!--</bean>-->
+
+ <!--<bean id="bolt" class="org.apache.streams.messaging.storm.StormSubscriberBolt"/>-->
+ <!--<bean id="spout" class="org.apache.streams.messaging.storm.StormSubscriberSpout"/>-->
+
+ <bean id="activityAggregator" class="org.apache.streams.messaging.aggregation.ActivityAggregator">
+ <property name="activityService" ref="activityService"/>
+ <property name="activityStreamsSubscriberWarehouse" ref="activityStreamsSubscriberWarehouse"/>
+ </bean>
+
+ <task:annotation-driven/>
+
+ <bean id="jmsConnectionFactory"
+ class="org.apache.activemq.ActiveMQConnectionFactory">
+ <property name="brokerURL" value="${activemq.jmsConnectionFactoryUrl}"/>
+ </bean>
+
+ <bean id="pooledConnectionFactory"
+ class="org.apache.activemq.pool.PooledConnectionFactory" init-method="start" destroy-method="stop">
+ <property name="maxConnections" value="8"/>
+ <property name="connectionFactory" ref="jmsConnectionFactory"/>
+ </bean>
+
+ <bean id="jmsConfig"
+ class="org.apache.camel.component.jms.JmsConfiguration">
+ <property name="connectionFactory" ref="pooledConnectionFactory"/>
+ <property name="concurrentConsumers" value="10"/>
+ </bean>
+
+ <bean id="activemq"
+ class="org.apache.activemq.camel.component.ActiveMQComponent">
+ <property name="configuration" ref="jmsConfig"/>
+ </bean>
+
+
+</beans>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/spring/streams-eip-osgi-component-import.xml
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/spring/streams-eip-osgi-component-import.xml b/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/spring/streams-eip-osgi-component-import.xml
new file mode 100644
index 0000000..9066206
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/spring/streams-eip-osgi-component-import.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:osgi="http://www.springframework.org/schema/osgi"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans.xsd
+ http://www.springframework.org/schema/osgi
+ http://www.springframework.org/schema/osgi/spring-osgi.xsd">
+
+
+ <osgi:reference id="activityPublisherRegistration" interface="org.apache.streams.osgi.components.ActivityPublisherRegistration" />
+ <osgi:reference id="activityConsumerWarehouse" interface="org.apache.streams.osgi.components.activityconsumer.ActivityConsumerWarehouse" />
+
+ <osgi:reference id="activityStreamsSubscriberRegistration" interface="org.apache.streams.osgi.components.ActivityStreamsSubscriberRegistration" />
+ <osgi:reference id="activityStreamsSubscriberWarehouse" interface="org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscriberWarehouse" />
+
+
+
+
+</beans>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/spring/streamsCamelContext.xml
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/spring/streamsCamelContext.xml b/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/spring/streamsCamelContext.xml
new file mode 100644
index 0000000..4360f39
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/spring/streamsCamelContext.xml
@@ -0,0 +1,96 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:osgi="http://www.springframework.org/schema/osgi"
+ xsi:schemaLocation="
+ http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+ http://camel.apache.org/schema/spring
+ http://camel.apache.org/schema/spring/camel-spring-2.0.0.xsd
+ http://www.springframework.org/schema/osgi
+ http://www.springframework.org/schema/osgi/spring-osgi.xsd">
+
+
+
+
+ <camelContext id="context" xmlns="http://camel.apache.org/schema/spring">
+
+ <endpoint id="consumerRegistrationEndpoint" uri="${consumer.registrationEndpoint}"/>
+ <endpoint id="subscriberRegistrationEndpoint" uri="${subscriber.registrationEndpoint}"/>
+ <!--publisher registration route setup -->
+ <route>
+ <from uri="ref:consumerRegistrationEndpoint"/>
+ <bean ref="activityRegistrationProcessor" />
+ <to uri="direct:publisher-register"/>
+ </route>
+
+ <route>
+ <from uri="direct:publisher-register"/>
+ <bean ref="activityPublisherRegistration" method="register"/>
+ <to uri="direct:add-publisher-route"/>
+ </route>
+
+ <route>
+ <from uri="direct:add-publisher-route"/>
+ <bean ref="activityConsumerRouter" method="createNewRouteForConsumer"/>
+ <to uri="log:ExampleLog"/>
+ </route>
+
+ <!--split activities on Q, waiting for aggregation -->
+ <route>
+ <from uri="direct:activityQ"/>
+ <inOnly uri="activemq:queue:activities"/>
+ </route>
+
+ <route>
+ <from uri="activemq:queue:activities"/>
+ <aggregate completionInterval="500" groupExchanges="true">
+ <correlationExpression>
+ <constant>true</constant>
+ </correlationExpression>
+ <bean ref="activityService" method="receiveExchange"/>
+ </aggregate>
+ </route>
+
+
+ <!-- register as a subscriber - returned the endpoint to poll and add to subscription sources - GET/POST -->
+ <route>
+ <from uri="ref:subscriberRegistrationEndpoint"/>
+ <bean ref="subscriberRegistrationProcessor" />
+ <to uri="direct:subscriber-register"/>
+ </route>
+
+ <route>
+ <from uri="direct:subscriber-register"/>
+ <bean ref="activityStreamsSubscriberRegistration" method="register"/>
+ <to uri="direct:add-subscriber-route"/>
+ </route>
+
+ <route>
+ <from uri="direct:add-subscriber-route"/>
+ <bean ref="activityStreamsSubscriberRouter" method="createNewRouteForSubscriber"/>
+ <to uri="log:ExampleLog"/>
+ </route>
+
+
+ </camelContext>
+
+</beans>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adb43b29/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/streams.properties
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/streams.properties b/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/streams.properties
new file mode 100644
index 0000000..b7bbfce
--- /dev/null
+++ b/streams-runtimes/streams-runtime-webapp/src/main/resources/META-INF/streams.properties
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+servlet.baseUrlPath=http://localhost:8080/streams-web/
+
+consumer.inRouteHost=localhost
+consumer.inRoutePort=8000
+consumer.activityQUri = direct:activityQ
+
+consumer.publisherEndpointProtocol=jetty:http://
+consumer.publisherEndpointUrlResource=streams/publish
+consumer.receiveMethod=receive
+consumer.splitMethod=split
+
+consumer.registrationEndpoint=jetty:http://localhost:8000/streams/publisher/register
+
+subscriber.inRouteHost=localhost
+subscriber.inRoutePort=8000
+subscriber.subscriberEndpointUrlResource=streams/subscriber
+subscriber.receiveMethod=receive
+subscriber.postMethod=updateActivityStreamsSubscriberConfiguration
+subscriber.getMethod=getStream
+subscriber.registrationEndpoint=jetty:http://localhost:8000/streams/subscriber/register
+subscriber.subscriberEndpointProtocol=jetty:http://
+
+activemq.jmsConnectionFactoryUrl=tcp://localhost:61616
\ No newline at end of file