You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2015/03/27 17:50:34 UTC
[3/6] incubator-streams-examples git commit: integration test will
now pass, after merge of https://github.com/apache/incubator-streams/pull/197
integration test will now pass, after merge of https://github.com/apache/incubator-streams/pull/197
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/commit/8e08b4c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/tree/8e08b4c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/diff/8e08b4c2
Branch: refs/heads/master
Commit: 8e08b4c2268af80540a4ed3761b6f3b47d5cd2ad
Parents: 6cb1263
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Mon Mar 23 13:10:49 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Mon Mar 23 13:10:49 2015 -0500
----------------------------------------------------------------------
local/elasticsearch-hdfs/pom.xml | 3 ---
.../example/HdfsElasticsearch.java | 3 ++-
.../src/main/resources/ElasticsearchHdfs.json | 23 -----------------
.../src/main/resources/HdfsElasticsearch.json | 19 --------------
.../elasticsearch/test/ElasticsearchHdfsIT.java | 11 ++++++++
.../src/test/resources/log4j.properties | 8 ++++++
.../src/test/resources/logback.xml | 27 ++++++++++++++++++++
.../src/test/resources/testBackup.json | 6 +----
.../src/test/resources/testRestore.json | 6 +----
local/pom.xml | 1 -
10 files changed, 50 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/8e08b4c2/local/elasticsearch-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/pom.xml b/local/elasticsearch-hdfs/pom.xml
index 4ac7a33..0a81b75 100644
--- a/local/elasticsearch-hdfs/pom.xml
+++ b/local/elasticsearch-hdfs/pom.xml
@@ -339,9 +339,6 @@
</plugins>
</build>
- <activation>
- <activeByDefault/>
- </activation>
</profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/8e08b4c2/local/elasticsearch-hdfs/src/main/java/org/apache/streams/elasticsearch/example/HdfsElasticsearch.java
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/main/java/org/apache/streams/elasticsearch/example/HdfsElasticsearch.java b/local/elasticsearch-hdfs/src/main/java/org/apache/streams/elasticsearch/example/HdfsElasticsearch.java
index 4574ce8..0a65479 100644
--- a/local/elasticsearch-hdfs/src/main/java/org/apache/streams/elasticsearch/example/HdfsElasticsearch.java
+++ b/local/elasticsearch-hdfs/src/main/java/org/apache/streams/elasticsearch/example/HdfsElasticsearch.java
@@ -33,6 +33,7 @@ import org.apache.streams.local.builders.LocalStreamBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.math.BigInteger;
import java.util.Map;
import java.util.concurrent.*;
@@ -75,7 +76,7 @@ public class HdfsElasticsearch implements Runnable {
Map<String, Object> streamConfig = Maps.newHashMap();
streamConfig.put(LocalStreamBuilder.STREAM_IDENTIFIER_KEY, STREAMS_ID);
- streamConfig.put(LocalStreamBuilder.TIMEOUT_KEY, 7 * 24 * 60 * 1000);
+ streamConfig.put(LocalStreamBuilder.TIMEOUT_KEY, 1000 * 1000);
StreamBuilder builder = new LocalStreamBuilder(1000, streamConfig);
builder.newPerpetualStream(WebHdfsPersistReader.STREAMS_ID, webHdfsPersistReader);
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/8e08b4c2/local/elasticsearch-hdfs/src/main/resources/ElasticsearchHdfs.json
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/main/resources/ElasticsearchHdfs.json b/local/elasticsearch-hdfs/src/main/resources/ElasticsearchHdfs.json
deleted file mode 100644
index f114dbc..0000000
--- a/local/elasticsearch-hdfs/src/main/resources/ElasticsearchHdfs.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "source": {
- "hosts": [
- "localhost"
- ],
- "port": 9300,
- "clusterName": "elasticsearch",
- "indexes": [
- "activity"
- ],
- "types": [
- "activity"
- ],
- "forceUseConfig": true
- },
- "destination": {
- "scheme": "file",
- "host": "localhost",
- "user": "cloudera",
- "path": "/tmp",
- "writerPath": "activity"
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/8e08b4c2/local/elasticsearch-hdfs/src/main/resources/HdfsElasticsearch.json
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/main/resources/HdfsElasticsearch.json b/local/elasticsearch-hdfs/src/main/resources/HdfsElasticsearch.json
deleted file mode 100644
index b975e34..0000000
--- a/local/elasticsearch-hdfs/src/main/resources/HdfsElasticsearch.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "source": {
- "scheme": "file",
- "host": "localhost",
- "user": "cloudera",
- "path": "/tmp",
- "writerPath": "activity"
- },
- "destination": {
- "hosts": [
- "localhost"
- ],
- "port": 9300,
- "clusterName": "elasticsearch",
- "index": "activity2",
- "type": "activity",
- "forceUseConfig": true
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/8e08b4c2/local/elasticsearch-hdfs/src/test/java/org/apache/streams/example/elasticsearch/test/ElasticsearchHdfsIT.java
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/test/java/org/apache/streams/example/elasticsearch/test/ElasticsearchHdfsIT.java b/local/elasticsearch-hdfs/src/test/java/org/apache/streams/example/elasticsearch/test/ElasticsearchHdfsIT.java
index f4c34cb..ad6970c 100644
--- a/local/elasticsearch-hdfs/src/test/java/org/apache/streams/example/elasticsearch/test/ElasticsearchHdfsIT.java
+++ b/local/elasticsearch-hdfs/src/test/java/org/apache/streams/example/elasticsearch/test/ElasticsearchHdfsIT.java
@@ -20,6 +20,7 @@ package org.apache.streams.example.elasticsearch.test;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.Uninterruptibles;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.IOUtils;
import org.apache.streams.core.StreamsDatum;
@@ -48,6 +49,7 @@ import org.slf4j.LoggerFactory;
import java.io.InputStream;
import java.io.File;
import java.util.*;
+import java.util.concurrent.TimeUnit;
/**
* Test copying documents between hdfs and elasticsearch
@@ -127,10 +129,19 @@ public class ElasticsearchHdfsIT extends ElasticsearchIntegrationTest {
Thread restoreThread = new Thread(restore);
restoreThread.start();
+
+ Uninterruptibles.sleepUninterruptibly(30, TimeUnit.SECONDS);
+
restoreThread.join();
+ Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS);
+
flushAndRefresh();
+ Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS);
+
+ assert(indexExists("destination"));
+
long destCount = client().count(client().prepareCount("destination").request()).get().getCount();
assert srcCount == destCount;
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/8e08b4c2/local/elasticsearch-hdfs/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/test/resources/log4j.properties b/local/elasticsearch-hdfs/src/test/resources/log4j.properties
new file mode 100644
index 0000000..71255bb
--- /dev/null
+++ b/local/elasticsearch-hdfs/src/test/resources/log4j.properties
@@ -0,0 +1,8 @@
+# Root logger option
+log4j.rootLogger=DEBUG, stdout
+
+# Direct log messages to stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/8e08b4c2/local/elasticsearch-hdfs/src/test/resources/logback.xml
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/test/resources/logback.xml b/local/elasticsearch-hdfs/src/test/resources/logback.xml
new file mode 100644
index 0000000..fc47894
--- /dev/null
+++ b/local/elasticsearch-hdfs/src/test/resources/logback.xml
@@ -0,0 +1,27 @@
+<configuration debug="true" scan="true" scanPeriod="5 seconds">
+
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <append>true</append>
+ <!-- encoders are assigned the type
+ ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
+ <encoder>
+ <pattern>%-4relative [%thread] %-5level %logger{35} - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="FILE" class="ch.qos.logback.core.FileAppender">
+ <file>target/logback.txt</file>
+ <append>true</append>
+ <!-- encoders are assigned the type
+ ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
+ <encoder>
+ <pattern>%-4relative [%thread] %-5level %logger{35} - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <root level="DEBUG">
+ <appender-ref ref="CONSOLE" />
+ <appender-ref ref="FILE" />
+ </root>
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/8e08b4c2/local/elasticsearch-hdfs/src/test/resources/testBackup.json
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/test/resources/testBackup.json b/local/elasticsearch-hdfs/src/test/resources/testBackup.json
index a92d67d..2002cb8 100644
--- a/local/elasticsearch-hdfs/src/test/resources/testBackup.json
+++ b/local/elasticsearch-hdfs/src/test/resources/testBackup.json
@@ -18,11 +18,7 @@
"user": "cloudera",
"path": "target",
"writerPath": "test",
- "writerPrefix": "activities",
- "fields": [
- "ID",
- "DOC"
- ]
+ "writerFilePrefix": "activities"
}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/8e08b4c2/local/elasticsearch-hdfs/src/test/resources/testRestore.json
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/test/resources/testRestore.json b/local/elasticsearch-hdfs/src/test/resources/testRestore.json
index 3f2ba2c..63b42c5 100644
--- a/local/elasticsearch-hdfs/src/test/resources/testRestore.json
+++ b/local/elasticsearch-hdfs/src/test/resources/testRestore.json
@@ -4,11 +4,7 @@
"host": "localhost",
"user": "cloudera",
"path": "target",
- "readerPath": "test",
- "fields": [
- "ID",
- "DOC"
- ]
+ "readerPath": "test"
},
"destination": {
"hosts": [
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/8e08b4c2/local/pom.xml
----------------------------------------------------------------------
diff --git a/local/pom.xml b/local/pom.xml
index 87c2f0f..910c708 100644
--- a/local/pom.xml
+++ b/local/pom.xml
@@ -38,7 +38,6 @@
<modules>
<module>elasticsearch-hdfs</module>
-
<module>elasticsearch-reindex</module>
</modules>