You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2016/10/11 21:17:12 UTC
[7/7] incubator-streams-examples git commit: twitter-follow-graph
docker and twitter IT
twitter-follow-graph docker and twitter IT
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/commit/97fca1ac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/tree/97fca1ac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/diff/97fca1ac
Branch: refs/heads/master
Commit: 97fca1ac5a3ead3aa4e00b014a99c441b312e69e
Parents: 476d356
Author: Steve Blackmon @steveblackmon <sb...@apache.org>
Authored: Mon Oct 10 17:26:22 2016 -0500
Committer: Steve Blackmon @steveblackmon <sb...@apache.org>
Committed: Mon Oct 10 17:26:22 2016 -0500
----------------------------------------------------------------------
local/twitter-follow-graph/pom.xml | 78 ++++++++++++++++++-
.../example/graph/TwitterFollowGraph.java | 33 ++++++--
.../TwitterFollowGraphConfiguration.json | 2 +-
.../src/site/markdown/index.md | 25 +++++++
.../twitter/example/TwitterFollowGraphIT.java | 79 ++++++++++++++++++++
.../test/resources/TwitterFollowGraphIT.conf | 28 +++++++
6 files changed, 233 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/97fca1ac/local/twitter-follow-graph/pom.xml
----------------------------------------------------------------------
diff --git a/local/twitter-follow-graph/pom.xml b/local/twitter-follow-graph/pom.xml
index 9bf980d..4ce6a64 100644
--- a/local/twitter-follow-graph/pom.xml
+++ b/local/twitter-follow-graph/pom.xml
@@ -236,11 +236,81 @@
</execution>
</executions>
</plugin>
- <plugin>
- <groupId>io.fabric8</groupId>
- <artifactId>docker-maven-plugin</artifactId>
- </plugin>
</plugins>
</build>
+ <profiles>
+ <profile>
+ <id>dockerITs</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ <property>
+ <name>skipITs</name>
+ <value>false</value>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>io.fabric8</groupId>
+ <artifactId>docker-maven-plugin</artifactId>
+ <version>${docker.plugin.version}</version>
+ <configuration combine.self="override">
+ <watchInterval>500</watchInterval>
+ <logDate>default</logDate>
+ <verbose>true</verbose>
+ <autoPull>on</autoPull>
+ <images>
+ <image>
+ <name>neo4j</name>
+ <alias>graph</alias>
+ <run>
+ <env>
+ <NEO4J_AUTH>none</NEO4J_AUTH>
+ </env>
+ <namingStrategy>none</namingStrategy>
+ <ports>
+ <port>${graph.http.host}:${graph.http.port}:7474</port>
+ <port>${graph.tcp.host}:${graph.tcp.port}:7687</port>
+ </ports>
+ <portPropertyFile>graph.properties</portPropertyFile>
+ <wait>
+ <log>graph startup</log>
+ <http>
+ <url>http://${graph.http.host}:${graph.http.port}</url>
+ <method>GET</method>
+ <status>200</status>
+ </http>
+ <time>20000</time>
+ <kill>1000</kill>
+ <shutdown>500</shutdown>
+ <!--<tcp>-->
+ <!--<host>${es.transport.host}</host>-->
+ <!--<ports>-->
+ <!--<port>${es.transport.port}</port>-->
+ <!--</ports>-->
+ <!--</tcp>-->
+ </wait>
+ <log>
+ <enabled>true</enabled>
+ <date>default</date>
+ <color>cyan</color>
+ </log>
+ </run>
+ <watch>
+ <mode>none</mode>
+ </watch>
+ </image>
+
+ </images>
+ </configuration>
+
+ </plugin>
+
+ </plugins>
+ </build>
+
+ </profile>
+ </profiles>
+
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/97fca1ac/local/twitter-follow-graph/src/main/java/org/apache/streams/example/graph/TwitterFollowGraph.java
----------------------------------------------------------------------
diff --git a/local/twitter-follow-graph/src/main/java/org/apache/streams/example/graph/TwitterFollowGraph.java b/local/twitter-follow-graph/src/main/java/org/apache/streams/example/graph/TwitterFollowGraph.java
index e88f299..11c52bb 100644
--- a/local/twitter-follow-graph/src/main/java/org/apache/streams/example/graph/TwitterFollowGraph.java
+++ b/local/twitter-follow-graph/src/main/java/org/apache/streams/example/graph/TwitterFollowGraph.java
@@ -47,19 +47,23 @@ import java.util.Map;
* Collects friend and follow connections for a set of twitter users and builds a graph
* database in neo4j.
*/
-public class TwitterFollowGraph {
+public class TwitterFollowGraph implements Runnable {
private final static Logger LOGGER = LoggerFactory.getLogger(TwitterFollowGraph.class);
- public static void main(String[] args) {
+ TwitterFollowGraphConfiguration config;
- LOGGER.info(StreamsConfigurator.config.toString());
+ public TwitterFollowGraph() {
+ this(new ComponentConfigurator<>(TwitterFollowGraphConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig()));
+ }
- StreamsConfiguration streams = StreamsConfigurator.detectConfiguration();
+ public TwitterFollowGraph(TwitterFollowGraphConfiguration config) {
+ this.config = config;
+ }
- TwitterFollowingGraphConfiguration configuration = new ComponentConfigurator<>(TwitterFollowingGraphConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig());
+ public void run() {
- TwitterFollowingConfiguration twitterFollowingConfiguration = configuration.getTwitter();
+ TwitterFollowingConfiguration twitterFollowingConfiguration = config.getTwitter();
TwitterFollowingProvider followingProvider = new TwitterFollowingProvider(twitterFollowingConfiguration);
TypeConverterProcessor converter = new TypeConverterProcessor(String.class);
@@ -69,7 +73,7 @@ public class TwitterFollowGraph {
.withConverters(Lists.newArrayList((ActivityConverter) new TwitterFollowActivityConverter()));
ActivityConverterProcessor activity = new ActivityConverterProcessor(activityConverterProcessorConfiguration);
- GraphHttpConfiguration graphWriterConfiguration = configuration.getGraph();
+ GraphHttpConfiguration graphWriterConfiguration = config.getGraph();
GraphHttpPersistWriter graphPersistWriter = new GraphHttpPersistWriter(graphWriterConfiguration);
StreamBuilder builder = new LocalStreamBuilder();
@@ -81,4 +85,19 @@ public class TwitterFollowGraph {
builder.start();
}
+ public static void main(String[] args) {
+
+ LOGGER.info(StreamsConfigurator.config.toString());
+
+ TwitterFollowGraph stream = new TwitterFollowGraph();
+
+ stream.run();
+
+ LOGGER.info(StreamsConfigurator.config.toString());
+
+ StreamsConfiguration streams = StreamsConfigurator.detectConfiguration();
+
+
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/97fca1ac/local/twitter-follow-graph/src/main/jsonschema/TwitterFollowGraphConfiguration.json
----------------------------------------------------------------------
diff --git a/local/twitter-follow-graph/src/main/jsonschema/TwitterFollowGraphConfiguration.json b/local/twitter-follow-graph/src/main/jsonschema/TwitterFollowGraphConfiguration.json
index 129be4c..f9c4ac1 100644
--- a/local/twitter-follow-graph/src/main/jsonschema/TwitterFollowGraphConfiguration.json
+++ b/local/twitter-follow-graph/src/main/jsonschema/TwitterFollowGraphConfiguration.json
@@ -4,7 +4,7 @@
"http://www.apache.org/licenses/LICENSE-2.0"
],
"type": "object",
- "javaType" : "org.apache.streams.example.graph.TwitterFollowingGraphConfiguration",
+ "javaType" : "org.apache.streams.example.graph.TwitterFollowGraphConfiguration",
"javaInterfaces": ["java.io.Serializable"],
"properties": {
"twitter": { "javaType": "org.apache.streams.twitter.TwitterFollowingConfiguration", "type": "object", "required": true },
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/97fca1ac/local/twitter-follow-graph/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/local/twitter-follow-graph/src/site/markdown/index.md b/local/twitter-follow-graph/src/site/markdown/index.md
index b8494cd..3991688 100644
--- a/local/twitter-follow-graph/src/site/markdown/index.md
+++ b/local/twitter-follow-graph/src/site/markdown/index.md
@@ -30,6 +30,31 @@ Build:
mvn clean package verify
+Test:
+-----
+Create a local file `application.conf` with valid twitter credentials
+
+ twitter {
+ oauth {
+ consumerKey = ""
+ consumerSecret = ""
+ accessToken = ""
+ accessTokenSecret = ""
+ }
+ }
+
+Start up neo4j with docker:
+
+ mvn -PdockerITs docker:start
+
+Build with integration testing enabled, using your credentials
+
+ mvn clean test verify -DskipITs=false -DargLine="-Dconfig.file=`pwd`/application.conf"
+
+Shutdown neo4j when finished:
+
+ mvn -PdockerITs docker:stop
+
Run (Local):
------------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/97fca1ac/local/twitter-follow-graph/src/test/java/org/apache/streams/twitter/example/TwitterFollowGraphIT.java
----------------------------------------------------------------------
diff --git a/local/twitter-follow-graph/src/test/java/org/apache/streams/twitter/example/TwitterFollowGraphIT.java b/local/twitter-follow-graph/src/test/java/org/apache/streams/twitter/example/TwitterFollowGraphIT.java
new file mode 100644
index 0000000..c5254fe
--- /dev/null
+++ b/local/twitter-follow-graph/src/test/java/org/apache/streams/twitter/example/TwitterFollowGraphIT.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.twitter.example;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigParseOptions;
+import org.apache.streams.config.ComponentConfigurator;
+import org.apache.streams.config.StreamsConfiguration;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.example.graph.TwitterFollowGraph;
+import org.apache.streams.example.graph.TwitterFollowGraphConfiguration;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.Properties;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+
+/**
+ * Example stream that populates elasticsearch with activities from twitter userstream in real-time
+ */
+public class TwitterFollowGraphIT {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(TwitterFollowGraphIT.class);
+
+ protected TwitterFollowGraphConfiguration testConfiguration;
+
+ private int count = 0;
+
+ @Before
+ public void prepareTest() throws Exception {
+
+ Config reference = ConfigFactory.load();
+ File conf_file = new File("target/test-classes/TwitterFollowGraphIT.conf");
+ assert(conf_file.exists());
+ Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false));
+ Properties graph_properties = new Properties();
+ InputStream graph_stream = new FileInputStream("graph.properties");
+ graph_properties.load(graph_stream);
+ Config graphProps = ConfigFactory.parseProperties(graph_properties);
+ Config typesafe = testResourceConfig.withFallback(graphProps).withFallback(reference).resolve();
+ StreamsConfiguration streams = StreamsConfigurator.detectConfiguration(typesafe);
+ testConfiguration = new ComponentConfigurator<>(TwitterFollowGraphConfiguration.class).detectConfiguration(typesafe);
+
+ }
+
+ @Test
+ public void testTwitterFollowGraph() throws Exception {
+
+ TwitterFollowGraph stream = new TwitterFollowGraph(testConfiguration);
+
+ stream.run();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/97fca1ac/local/twitter-follow-graph/src/test/resources/TwitterFollowGraphIT.conf
----------------------------------------------------------------------
diff --git a/local/twitter-follow-graph/src/test/resources/TwitterFollowGraphIT.conf b/local/twitter-follow-graph/src/test/resources/TwitterFollowGraphIT.conf
new file mode 100644
index 0000000..ecd4fd4
--- /dev/null
+++ b/local/twitter-follow-graph/src/test/resources/TwitterFollowGraphIT.conf
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+twitter {
+ endpoint = "friends"
+ info = [
+ 18055613
+ ]
+ twitter.max_items = 1000
+}
+graph {
+ hostname = ${graph.http.host}
+ port = ${graph.http.port}
+ type = "neo4j"
+ graph = "data"
+}
\ No newline at end of file