You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2015/08/28 00:12:45 UTC
[01/13] storm git commit: Add hamcrest-all test dependency to
storm-elasticsearch
Repository: storm
Updated Branches:
refs/heads/master 59ed4d23c -> 4faee0e29
Add hamcrest-all test dependency to storm-elasticsearch
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/642ab9d0
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/642ab9d0
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/642ab9d0
Branch: refs/heads/master
Commit: 642ab9d009f0b195ce0a60ef8363f3692c2e99b5
Parents: 9c94188
Author: Alex Panov <al...@teradata.com>
Authored: Thu Aug 20 12:24:46 2015 +0200
Committer: Alex Panov <al...@teradata.com>
Committed: Thu Aug 20 12:24:46 2015 +0200
----------------------------------------------------------------------
external/storm-elasticsearch/pom.xml | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/642ab9d0/external/storm-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/pom.xml b/external/storm-elasticsearch/pom.xml
index 88ea312..6cc06d1 100644
--- a/external/storm-elasticsearch/pom.xml
+++ b/external/storm-elasticsearch/pom.xml
@@ -59,6 +59,12 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <version>1.3</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.9.0</version>
@@ -92,4 +98,4 @@
</plugins>
</build>
-</project>
\ No newline at end of file
+</project>
[09/13] storm git commit: Move the ES output adapter to upper level
Posted by ka...@apache.org.
Move the ES output adapter to upper level
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/4be31c5d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/4be31c5d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/4be31c5d
Branch: refs/heads/master
Commit: 4be31c5d1f5d7a782330772c5211f441bbc24922
Parents: a2db49f
Author: Alex Panov <al...@teradata.com>
Authored: Thu Aug 20 18:01:31 2015 +0200
Committer: Alex Panov <al...@teradata.com>
Committed: Thu Aug 20 18:01:31 2015 +0200
----------------------------------------------------------------------
.../elasticsearch/EsLookupResultOutput.java | 32 ++++++++++++++++++++
.../storm/elasticsearch/bolt/EsLookupBolt.java | 1 +
.../bolt/EsLookupResultOutput.java | 32 --------------------
.../bolt/EsLookupBoltIntegrationTest.java | 1 +
.../elasticsearch/bolt/EsLookupBoltTest.java | 1 +
5 files changed, 35 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/4be31c5d/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/EsLookupResultOutput.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/EsLookupResultOutput.java b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/EsLookupResultOutput.java
new file mode 100644
index 0000000..68d7dba
--- /dev/null
+++ b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/EsLookupResultOutput.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.elasticsearch;
+
+import java.util.Collection;
+
+import org.elasticsearch.action.get.GetResponse;
+
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Values;
+
+public interface EsLookupResultOutput {
+
+ Collection<Values> toValues(GetResponse response);
+
+ Fields fields();
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/4be31c5d/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupBolt.java b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupBolt.java
index b6063d4..d9982f2 100644
--- a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupBolt.java
+++ b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupBolt.java
@@ -20,6 +20,7 @@ package org.apache.storm.elasticsearch.bolt;
import java.util.Collection;
import org.apache.storm.elasticsearch.ElasticsearchGetRequest;
+import org.apache.storm.elasticsearch.EsLookupResultOutput;
import org.apache.storm.elasticsearch.common.EsConfig;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
http://git-wip-us.apache.org/repos/asf/storm/blob/4be31c5d/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupResultOutput.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupResultOutput.java b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupResultOutput.java
deleted file mode 100644
index c21fba6..0000000
--- a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupResultOutput.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.storm.elasticsearch.bolt;
-
-import java.util.Collection;
-
-import org.elasticsearch.action.get.GetResponse;
-
-import backtype.storm.tuple.Fields;
-import backtype.storm.tuple.Values;
-
-public interface EsLookupResultOutput {
-
- Collection<Values> toValues(GetResponse response);
-
- Fields fields();
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/4be31c5d/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltIntegrationTest.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltIntegrationTest.java b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltIntegrationTest.java
index a869281..c391688 100644
--- a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltIntegrationTest.java
+++ b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltIntegrationTest.java
@@ -22,6 +22,7 @@ import java.util.Collections;
import java.util.UUID;
import org.apache.storm.elasticsearch.ElasticsearchGetRequest;
+import org.apache.storm.elasticsearch.EsLookupResultOutput;
import org.apache.storm.elasticsearch.common.EsConfig;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.get.GetRequest;
http://git-wip-us.apache.org/repos/asf/storm/blob/4be31c5d/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltTest.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltTest.java b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltTest.java
index 9f19036..70ea140 100644
--- a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltTest.java
+++ b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltTest.java
@@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.UUID;
import org.apache.storm.elasticsearch.ElasticsearchGetRequest;
+import org.apache.storm.elasticsearch.EsLookupResultOutput;
import org.apache.storm.elasticsearch.common.EsConfig;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.get.GetRequest;
[13/13] storm git commit: Merge branch 'STORM-978'
Posted by ka...@apache.org.
Merge branch 'STORM-978'
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/4faee0e2
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/4faee0e2
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/4faee0e2
Branch: refs/heads/master
Commit: 4faee0e29d3698b9623209c248c4ab1ddd992518
Parents: 59ed4d2 95985c5
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Fri Aug 28 07:12:30 2015 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Fri Aug 28 07:12:30 2015 +0900
----------------------------------------------------------------------
external/storm-elasticsearch/README.md | 20 ++-
external/storm-elasticsearch/pom.xml | 10 +-
.../elasticsearch/ElasticsearchGetRequest.java | 36 +++++
.../elasticsearch/EsLookupResultOutput.java | 43 ++++++
.../elasticsearch/bolt/AbstractEsBolt.java | 50 ++++---
.../elasticsearch/bolt/ElasticSearchClient.java | 57 ++++++++
.../storm/elasticsearch/bolt/EsLookupBolt.java | 71 ++++++++++
.../elasticsearch/bolt/TransportAddresses.java | 72 ++++++++++
.../bolt/AbstractEsBoltIntegrationTest.java | 91 +++++++++++++
.../elasticsearch/bolt/AbstractEsBoltTest.java | 76 ++++-------
.../elasticsearch/bolt/EsIndexBoltTest.java | 33 +++--
.../bolt/EsLookupBoltIntegrationTest.java | 132 +++++++++++++++++++
.../elasticsearch/bolt/EsLookupBoltTest.java | 120 +++++++++++++++++
.../elasticsearch/bolt/EsPercolateBoltTest.java | 30 ++---
.../bolt/TransportAddressesTest.java | 81 ++++++++++++
15 files changed, 803 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
[11/13] storm git commit: Make ES lookup bolt dependency interfaces
Serializable, add javadoc and more README
Posted by ka...@apache.org.
Make ES lookup bolt dependency interfaces Serializable, add javadoc and more README
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8618eb33
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8618eb33
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8618eb33
Branch: refs/heads/master
Commit: 8618eb336e4e84aa3aaf03f3418c9ec40be0ea1f
Parents: 69c78ca
Author: Alex Panov <al...@teradata.com>
Authored: Fri Aug 21 12:39:30 2015 +0200
Committer: Alex Panov <al...@teradata.com>
Committed: Fri Aug 21 12:39:30 2015 +0200
----------------------------------------------------------------------
external/storm-elasticsearch/README.md | 7 ++++++-
.../storm/elasticsearch/ElasticsearchGetRequest.java | 11 ++++++++++-
.../storm/elasticsearch/EsLookupResultOutput.java | 13 ++++++++++++-
3 files changed, 28 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/8618eb33/external/storm-elasticsearch/README.md
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/README.md b/external/storm-elasticsearch/README.md
index 04a3673..1b18019 100644
--- a/external/storm-elasticsearch/README.md
+++ b/external/storm-elasticsearch/README.md
@@ -36,7 +36,12 @@ for each Percolate.Match in PercolateResponse.
## EsLookupBolt (org.apache.storm.elasticsearch.bolt.EsLookupBolt)
-EsLookupBolt performs a get request to Elasticsearch. Incoming tuple is passed to provided GetRequest creator and the result of that execution is passed to Elasticsearch client.
+EsLookupBolt performs a get request to Elasticsearch.
+In order to do that, three dependencies need to be satisfied. Apart from usual EsConfig, two other dependencies must be provided:
+ ElasticsearchGetRequest is used to convert the incoming Tuple to the GetRequest that will be executed against Elasticsearch.
+ EsLookupResultOutput is used to declare the output fields and convert the GetResponse to values that are emited by the bolt.
+
+Incoming tuple is passed to provided GetRequest creator and the result of that execution is passed to Elasticsearch client.
The bolt then uses the provider output adapter (EsLookupResultOutput) to convert the GetResponse to Values to emit.
The output fields are also specified by the user of the bolt via the output adapter (EsLookupResultOutput).
http://git-wip-us.apache.org/repos/asf/storm/blob/8618eb33/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/ElasticsearchGetRequest.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/ElasticsearchGetRequest.java b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/ElasticsearchGetRequest.java
index 098c993..6a7ce61 100644
--- a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/ElasticsearchGetRequest.java
+++ b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/ElasticsearchGetRequest.java
@@ -17,11 +17,20 @@
*/
package org.apache.storm.elasticsearch;
+import java.io.Serializable;
+
import org.elasticsearch.action.get.GetRequest;
import backtype.storm.tuple.ITuple;
-public interface ElasticsearchGetRequest {
+/**
+ * @since 0.11
+ * The adapter to convert the incoming tuple to Elasticsearch GetRequest.
+ */
+public interface ElasticsearchGetRequest extends Serializable {
+ /**
+ * @return GetRequest to perform against Elasticsearch.
+ */
GetRequest extractFrom(ITuple tuple);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8618eb33/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/EsLookupResultOutput.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/EsLookupResultOutput.java b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/EsLookupResultOutput.java
index 68d7dba..9ccb9e6 100644
--- a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/EsLookupResultOutput.java
+++ b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/EsLookupResultOutput.java
@@ -17,6 +17,7 @@
*/
package org.apache.storm.elasticsearch;
+import java.io.Serializable;
import java.util.Collection;
import org.elasticsearch.action.get.GetResponse;
@@ -24,9 +25,19 @@ import org.elasticsearch.action.get.GetResponse;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
-public interface EsLookupResultOutput {
+/**
+ * @since 0.11
+ * The adapter to convert the results fetched from Elasticsearch to values.
+ */
+public interface EsLookupResultOutput extends Serializable {
+ /**
+ * @return collection of values to emit.
+ */
Collection<Values> toValues(GetResponse response);
+ /**
+ * @return output fields to declare.
+ */
Fields fields();
}
[02/13] storm git commit: Manage dependencies in storm-elasticsearch
pom
Posted by ka...@apache.org.
Manage dependencies in storm-elasticsearch pom
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e0bee05a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e0bee05a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e0bee05a
Branch: refs/heads/master
Commit: e0bee05a26d3c306a44998b9a409a4d44e5dcde7
Parents: 642ab9d
Author: Alex Panov <al...@teradata.com>
Authored: Thu Aug 20 12:26:00 2015 +0200
Committer: Alex Panov <al...@teradata.com>
Committed: Thu Aug 20 12:26:00 2015 +0200
----------------------------------------------------------------------
external/storm-elasticsearch/pom.xml | 4 ----
1 file changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e0bee05a/external/storm-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/pom.xml b/external/storm-elasticsearch/pom.xml
index 6cc06d1..ecc1bd2 100644
--- a/external/storm-elasticsearch/pom.xml
+++ b/external/storm-elasticsearch/pom.xml
@@ -55,8 +55,6 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.11</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
@@ -67,8 +65,6 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
- <version>1.9.0</version>
- <scope>test</scope>
</dependency>
</dependencies>
[04/13] storm git commit: Introduce a unit test superclass for ES
bolts
Posted by ka...@apache.org.
Introduce a unit test superclass for ES bolts
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3094baf2
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3094baf2
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3094baf2
Branch: refs/heads/master
Commit: 3094baf200beb513d98cea3a3ce94605e6760fe2
Parents: 3400c7d
Author: Alex Panov <al...@teradata.com>
Authored: Thu Aug 20 12:29:08 2015 +0200
Committer: Alex Panov <al...@teradata.com>
Committed: Thu Aug 20 12:29:08 2015 +0200
----------------------------------------------------------------------
.../bolt/AbstractEsBoltIntegrationTest.java | 91 ++++++++++++++++++++
.../elasticsearch/bolt/AbstractEsBoltTest.java | 76 ++++++----------
.../elasticsearch/bolt/EsIndexBoltTest.java | 33 ++++---
.../elasticsearch/bolt/EsPercolateBoltTest.java | 30 +++----
4 files changed, 143 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/3094baf2/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/AbstractEsBoltIntegrationTest.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/AbstractEsBoltIntegrationTest.java b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/AbstractEsBoltIntegrationTest.java
new file mode 100644
index 0000000..9bed459
--- /dev/null
+++ b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/AbstractEsBoltIntegrationTest.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.elasticsearch.bolt;
+
+import java.io.File;
+
+import org.apache.commons.io.FileUtils;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.client.Requests;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.node.NodeBuilder;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+public abstract class AbstractEsBoltIntegrationTest<Bolt extends AbstractEsBolt> extends AbstractEsBoltTest<Bolt> {
+
+ protected static Node node;
+
+ @BeforeClass
+ public static void startElasticSearchNode() throws Exception {
+ node = NodeBuilder.nodeBuilder().data(true).settings(createSettings()).build();
+ node.start();
+ ensureEsGreen(node);
+ ClusterHealthResponse clusterHealth = node.client()
+ .admin()
+ .cluster()
+ .health(Requests.clusterHealthRequest()
+ .timeout(TimeValue.timeValueSeconds(30))
+ .waitForGreenStatus()
+ .waitForRelocatingShards(0))
+ .actionGet();
+ Thread.sleep(1000);
+ }
+
+ private static ImmutableSettings.Builder createSettings() {
+ return ImmutableSettings.builder()
+ .put(ClusterName.SETTING, "test-cluster")
+ .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+ .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+ .put(EsExecutors.PROCESSORS, 1)
+ .put("http.enabled", false)
+ .put("index.percolator.map_unmapped_fields_as_string", true)
+ .put("index.store.type", "memory");
+ }
+
+ @AfterClass
+ public static void closeElasticSearchNode() throws Exception {
+ node.stop();
+ node.close();
+ FileUtils.deleteDirectory(new File("./data"));
+ }
+
+ private static void ensureEsGreen(Node node) {
+ ClusterHealthResponse chr = node.client()
+ .admin()
+ .cluster()
+ .health(Requests.clusterHealthRequest()
+ .timeout(TimeValue.timeValueSeconds(30))
+ .waitForGreenStatus()
+ .waitForEvents(Priority.LANGUID)
+ .waitForRelocatingShards(0))
+ .actionGet();
+ assertThat("cluster status is green", chr.getStatus(), equalTo(ClusterHealthStatus.GREEN));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3094baf2/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/AbstractEsBoltTest.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/AbstractEsBoltTest.java b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/AbstractEsBoltTest.java
index ae6b321..8c5ea55 100644
--- a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/AbstractEsBoltTest.java
+++ b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/AbstractEsBoltTest.java
@@ -17,65 +17,43 @@
*/
package org.apache.storm.elasticsearch.bolt;
-import backtype.storm.Config;
-import backtype.storm.task.OutputCollector;
-import org.apache.commons.io.FileUtils;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.settings.ImmutableSettings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.node.Node;
-import org.elasticsearch.node.NodeBuilder;
+import org.apache.storm.elasticsearch.common.EsConfig;
import org.junit.After;
-import org.junit.AfterClass;
import org.junit.Before;
-import org.junit.BeforeClass;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
-import java.io.File;
+import backtype.storm.Config;
+import backtype.storm.task.OutputCollector;
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.mockito.Mockito.mock;
+@RunWith(MockitoJUnitRunner.class)
+abstract class AbstractEsBoltTest<Bolt extends AbstractEsBolt> {
-public class AbstractEsBoltTest {
protected static Config config = new Config();
- protected static OutputCollector collector = mock(OutputCollector.class);
- protected static Node node;
- @BeforeClass
- public static void setup() throws Exception {
- node = NodeBuilder.nodeBuilder().data(true).settings(
- ImmutableSettings.builder()
- .put(ClusterName.SETTING, "test-cluster")
- .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
- .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
- .put(EsExecutors.PROCESSORS, 1)
- .put("http.enabled", false)
- .put("index.percolator.map_unmapped_fields_as_string", true)
- .put("index.store.type", "memory")
- ).build();
- node.start();
- ensureEsGreen(node);
- ClusterHealthResponse chr = node.client().admin().cluster()
- .health(Requests.clusterHealthRequest().timeout(TimeValue.timeValueSeconds(30)).waitForGreenStatus().waitForRelocatingShards(0)).actionGet();
- Thread.sleep(1000);
+ @Mock
+ protected OutputCollector outputCollector;
+
+ protected Bolt bolt;
+
+ @Before
+ public void createBolt() throws Exception {
+ bolt = createBolt(esConfig());
+ bolt.prepare(config, null, outputCollector);
}
- @AfterClass
- public static void cleanup() throws Exception {
- node.stop();
- node.close();
- FileUtils.deleteDirectory(new File("./data"));
+ protected abstract Bolt createBolt(EsConfig esConfig);
+
+ protected EsConfig esConfig() {
+ EsConfig esConfig = new EsConfig();
+ esConfig.setClusterName("test-cluster");
+ esConfig.setNodes(new String[] {"127.0.0.1:9300"});
+ return esConfig;
}
- private static void ensureEsGreen(Node node) {
- ClusterHealthResponse chr = node.client().admin().cluster()
- .health(Requests.clusterHealthRequest().timeout(TimeValue.timeValueSeconds(30)).waitForGreenStatus().waitForEvents(Priority.LANGUID).waitForRelocatingShards(0)).actionGet();
- assertThat("cluster status is green", chr.getStatus(), equalTo(ClusterHealthStatus.GREEN));
+ @After
+ public void cleanupBolt() throws Exception {
+ bolt.cleanup();
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/3094baf2/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsIndexBoltTest.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsIndexBoltTest.java b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsIndexBoltTest.java
index dd4b088..d2def5d 100644
--- a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsIndexBoltTest.java
+++ b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsIndexBoltTest.java
@@ -20,10 +20,7 @@ package org.apache.storm.elasticsearch.bolt;
import backtype.storm.tuple.Tuple;
import org.apache.storm.elasticsearch.common.EsConfig;
import org.apache.storm.elasticsearch.common.EsTestUtil;
-import org.elasticsearch.action.count.CountRequest;
-import org.elasticsearch.action.count.CountRequestBuilder;
import org.elasticsearch.action.count.CountResponse;
-import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.junit.Assert;
import org.junit.Test;
@@ -32,29 +29,20 @@ import org.slf4j.LoggerFactory;
import static org.mockito.Mockito.verify;
-public class EsIndexBoltTest extends AbstractEsBoltTest{
+public class EsIndexBoltTest extends AbstractEsBoltIntegrationTest<EsIndexBolt> {
private static final Logger LOG = LoggerFactory.getLogger(EsIndexBoltTest.class);
- private EsIndexBolt bolt;
@Test
public void testEsIndexBolt()
throws Exception {
- EsConfig esConfig = new EsConfig();
- esConfig.setClusterName("test-cluster");
- esConfig.setNodes(new String[]{"127.0.0.1:9300"});
-
- bolt = new EsIndexBolt(esConfig);
- bolt.prepare(config, null, collector);
-
- String source = "{\"user\":\"user1\"}";
String index = "index1";
String type = "type1";
- String id = "docId";
- Tuple tuple = EsTestUtil.generateTestTuple(source, index, type, id);
+
+ Tuple tuple = createTestTuple(index, type);
bolt.execute(tuple);
- verify(collector).ack(tuple);
+ verify(outputCollector).ack(tuple);
node.client().admin().indices().prepareRefresh(index).execute().actionGet();
CountResponse resp = node.client().prepareCount(index)
@@ -62,7 +50,16 @@ public class EsIndexBoltTest extends AbstractEsBoltTest{
.execute().actionGet();
Assert.assertEquals(1, resp.getCount());
+ }
+
+ private Tuple createTestTuple(String index, String type) {
+ String source = "{\"user\":\"user1\"}";
+ String id = "docId";
+ return EsTestUtil.generateTestTuple(source, index, type, id);
+ }
- bolt.cleanup();
+ @Override
+ protected EsIndexBolt createBolt(EsConfig esConfig) {
+ return new EsIndexBolt(esConfig);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3094baf2/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsPercolateBoltTest.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsPercolateBoltTest.java b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsPercolateBoltTest.java
index fd4fa4f..d68cc89 100644
--- a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsPercolateBoltTest.java
+++ b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsPercolateBoltTest.java
@@ -21,35 +21,27 @@ import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
import org.apache.storm.elasticsearch.common.EsConfig;
import org.apache.storm.elasticsearch.common.EsTestUtil;
-import org.elasticsearch.action.count.CountResponse;
import org.elasticsearch.action.percolate.PercolateResponse;
-import org.elasticsearch.index.query.TermQueryBuilder;
-import org.junit.Assert;
import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import static org.mockito.Mockito.verify;
import static org.mockito.Matchers.any;
-public class EsPercolateBoltTest extends AbstractEsBoltTest {
- private static final Logger LOG = LoggerFactory.getLogger(EsIndexBoltTest.class);
- private EsPercolateBolt bolt;
+public class EsPercolateBoltTest extends AbstractEsBoltIntegrationTest<EsPercolateBolt> {
+
+ @Override
+ protected EsPercolateBolt createBolt(EsConfig esConfig) {
+ return new EsPercolateBolt(esConfig);
+ }
@Test
public void testEsPercolateBolt()
throws Exception {
- EsConfig esConfig = new EsConfig();
- esConfig.setClusterName("test-cluster");
- esConfig.setNodes(new String[]{"localhost:9300"});
- bolt = new EsPercolateBolt(esConfig);
- bolt.prepare(config, null, collector);
-
String source = "{\"user\":\"user1\"}";
String index = "index1";
String type = ".percolator";
- node.client().prepareIndex("index1",".percolator")
+ node.client().prepareIndex("index1", ".percolator")
.setId("1")
.setSource("{\"query\":{\"match\":{\"user\":\"user1\"}}}").
execute().actionGet();
@@ -57,9 +49,7 @@ public class EsPercolateBoltTest extends AbstractEsBoltTest {
bolt.execute(tuple);
- verify(collector).ack(tuple);
- verify(collector).emit(new Values(source, any(PercolateResponse.Match.class)));
-
- bolt.cleanup();
+ verify(outputCollector).ack(tuple);
+ verify(outputCollector).emit(new Values(source, any(PercolateResponse.Match.class)));
}
-}
\ No newline at end of file
+}
[05/13] storm git commit: Implement EsLookupBolt
Posted by ka...@apache.org.
Implement EsLookupBolt
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/59999924
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/59999924
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/59999924
Branch: refs/heads/master
Commit: 599999246f4ce251d44c50a64423ccf8ea464b34
Parents: 3094baf
Author: Alex Panov <al...@teradata.com>
Authored: Thu Aug 20 12:30:17 2015 +0200
Committer: Alex Panov <al...@teradata.com>
Committed: Thu Aug 20 12:30:17 2015 +0200
----------------------------------------------------------------------
.../elasticsearch/ElasticsearchGetRequest.java | 10 ++
.../storm/elasticsearch/bolt/EsLookupBolt.java | 70 ++++++++++
.../bolt/EsLookupResultOutput.java | 15 +++
.../bolt/EsLookupBoltIntegrationTest.java | 130 +++++++++++++++++++
.../elasticsearch/bolt/EsLookupBoltTest.java | 102 +++++++++++++++
5 files changed, 327 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/59999924/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/ElasticsearchGetRequest.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/ElasticsearchGetRequest.java b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/ElasticsearchGetRequest.java
new file mode 100644
index 0000000..442af3c
--- /dev/null
+++ b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/ElasticsearchGetRequest.java
@@ -0,0 +1,10 @@
+package org.apache.storm.elasticsearch;
+
+import org.elasticsearch.action.get.GetRequest;
+
+import backtype.storm.tuple.Tuple;
+
+public interface ElasticsearchGetRequest {
+
+ GetRequest extractFrom(Tuple tuple);
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/59999924/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupBolt.java b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupBolt.java
new file mode 100644
index 0000000..b6063d4
--- /dev/null
+++ b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupBolt.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.elasticsearch.bolt;
+
+import java.util.Collection;
+
+import org.apache.storm.elasticsearch.ElasticsearchGetRequest;
+import org.apache.storm.elasticsearch.common.EsConfig;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
+
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Values;
+
+public class EsLookupBolt extends AbstractEsBolt {
+
+ private final ElasticsearchGetRequest getRequest;
+ private final EsLookupResultOutput output;
+
+ public EsLookupBolt(EsConfig esConfig, ElasticsearchGetRequest getRequest, EsLookupResultOutput output) {
+ super(esConfig);
+ this.getRequest = getRequest;
+ this.output = output;
+ }
+
+ @Override
+ public void execute(Tuple tuple) {
+ try {
+ Collection<Values> values = lookupValuesInEs(tuple);
+ tryEmitAndAck(values, tuple);
+ } catch (Exception e) {
+ collector.reportError(e);
+ collector.fail(tuple);
+ }
+ }
+
+ private Collection<Values> lookupValuesInEs(Tuple tuple) {
+ GetRequest request = getRequest.extractFrom(tuple);
+ GetResponse response = client.get(request).actionGet();
+ return output.toValues(response);
+ }
+
+ private void tryEmitAndAck(Collection<Values> values, Tuple tuple) {
+ for (Values value : values) {
+ collector.emit(tuple, value);
+ }
+ collector.ack(tuple);
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
+ outputFieldsDeclarer.declare(output.fields());
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/59999924/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupResultOutput.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupResultOutput.java b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupResultOutput.java
new file mode 100644
index 0000000..8696c6b
--- /dev/null
+++ b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupResultOutput.java
@@ -0,0 +1,15 @@
+package org.apache.storm.elasticsearch.bolt;
+
+import java.util.Collection;
+
+import org.elasticsearch.action.get.GetResponse;
+
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Values;
+
+public interface EsLookupResultOutput {
+
+ Collection<Values> toValues(GetResponse response);
+
+ Fields fields();
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/59999924/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltIntegrationTest.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltIntegrationTest.java b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltIntegrationTest.java
new file mode 100644
index 0000000..2a544f9
--- /dev/null
+++ b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltIntegrationTest.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.elasticsearch.bolt;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.UUID;
+
+import org.apache.storm.elasticsearch.ElasticsearchGetRequest;
+import org.apache.storm.elasticsearch.common.EsConfig;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Values;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.verify;
+
+@RunWith(MockitoJUnitRunner.class)
+public class EsLookupBoltIntegrationTest extends AbstractEsBoltIntegrationTest<EsLookupBolt> {
+
+ private final String documentId = UUID.randomUUID().toString();
+ private final String indexName = "index";
+ private final String typeName = "type";
+ private final String source = "{\"user\":\"user1\"}";
+
+ private ElasticsearchGetRequest getRequest = new TestElasticsearchGetRequest();
+ private EsLookupResultOutput output = new TestEsLookupResultOutput();
+
+ @Captor
+ private ArgumentCaptor<Tuple> anchor;
+
+ @Captor
+ private ArgumentCaptor<Values> emmitedValues;
+
+ @Mock
+ private Tuple tuple;
+
+ @Override
+ protected EsLookupBolt createBolt(EsConfig esConfig) {
+ return new EsLookupBolt(esConfig, getRequest, output);
+ }
+
+ @Before
+ public void populateIndexWithTestData() throws Exception {
+ node.client().prepareIndex(indexName, typeName, documentId).setSource(source).execute().actionGet();
+ }
+
+ @Before
+ public void clearIndex() throws Exception {
+ node.client().delete(new DeleteRequest(indexName, typeName, documentId)).actionGet();
+ }
+
+ @Test
+ public void anchorsTheTuple() throws Exception {
+ bolt.execute(tuple);
+
+ verify(outputCollector).emit(anchor.capture(), emmitedValues.capture());
+ assertThat(anchor.getValue(), is(tuple));
+ }
+
+ @Test
+ public void emitsExpectedValues() throws Exception {
+ Values expectedValues = expectedValues();
+
+ bolt.execute(tuple);
+
+ verify(outputCollector).emit(anchor.capture(), emmitedValues.capture());
+ assertThat(emmitedValues.getValue(), is(expectedValues));
+ }
+
+ @Test
+ public void acksTuple() throws Exception {
+ bolt.execute(tuple);
+
+ verify(outputCollector).ack(anchor.capture());
+ assertThat(anchor.getValue(), is(tuple));
+ }
+
+ private Values expectedValues() {
+ return new Values(source);
+ }
+
+ private class TestElasticsearchGetRequest implements ElasticsearchGetRequest {
+
+ @Override
+ public GetRequest extractFrom(Tuple tuple) {
+ return node.client().prepareGet().setId(documentId).setIndex(indexName).setType(typeName).request();
+ }
+ }
+
+ private class TestEsLookupResultOutput implements EsLookupResultOutput {
+
+ @Override
+ public Collection<Values> toValues(GetResponse response) {
+ return Collections.singleton(expectedValues());
+ }
+
+ @Override
+ public Fields fields() {
+ return new Fields("data");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/59999924/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltTest.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltTest.java b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltTest.java
new file mode 100644
index 0000000..602ed6d
--- /dev/null
+++ b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltTest.java
@@ -0,0 +1,102 @@
+package org.apache.storm.elasticsearch.bolt;
+
+import java.util.Collections;
+import java.util.UUID;
+
+import org.apache.storm.elasticsearch.ElasticsearchGetRequest;
+import org.apache.storm.elasticsearch.common.EsConfig;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.client.Client;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Values;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class EsLookupBoltTest extends AbstractEsBoltTest<EsLookupBolt> {
+
+ @Mock
+ private EsConfig esConfig;
+
+ @Mock
+ private ElasticsearchGetRequest getRequest;
+
+ @Mock
+ private EsLookupResultOutput output;
+
+ @Mock
+ private Tuple tuple;
+
+ @Mock
+ private GetRequest request;
+
+ @Mock
+ private Client client;
+
+ private Client originalClient;
+
+ @Override
+ protected EsLookupBolt createBolt(EsConfig esConfig) {
+ originalClient = EsLookupBolt.getClient();
+ EsLookupBolt.replaceClient(this.client);
+ return new EsLookupBolt(esConfig, getRequest, output);
+ }
+
+ @After
+ public void replaceClientWithOriginal() throws Exception {
+ EsLookupBolt.replaceClient(originalClient);
+ }
+
+ @Before
+ public void configureBoltDependencies() throws Exception {
+ when(getRequest.extractFrom(tuple)).thenReturn(request);
+ when(output.toValues(any(GetResponse.class))).thenReturn(Collections.singleton(new Values("")));
+ }
+
+ @Test
+ public void failsTupleWhenClientThrows() throws Exception {
+ when(client.get(request)).thenThrow(ElasticsearchException.class);
+ bolt.execute(tuple);
+
+ verify(outputCollector).fail(tuple);
+ }
+
+ @Test
+ public void reportsExceptionWhenClientThrows() throws Exception {
+ ElasticsearchException elasticsearchException = new ElasticsearchException("dummy");
+ when(client.get(request)).thenThrow(elasticsearchException);
+ bolt.execute(tuple);
+
+ verify(outputCollector).reportError(elasticsearchException);
+ }
+
+ @Test
+ public void fieldsAreDeclaredThroughProvidedOutput() throws Exception {
+ Fields fields = new Fields(UUID.randomUUID().toString());
+ when(output.fields()).thenReturn(fields);
+ OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class);
+ bolt.declareOutputFields(declarer);
+
+ ArgumentCaptor<Fields> declaredFields = ArgumentCaptor.forClass(Fields.class);
+ verify(declarer).declare(declaredFields.capture());
+
+ assertThat(declaredFields.getValue(), is(fields));
+ }
+}
[08/13] storm git commit: Weaken the ES get request extractor
dependency to ITuple
Posted by ka...@apache.org.
Weaken the ES get request extractor dependency to ITuple
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a2db49f1
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a2db49f1
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a2db49f1
Branch: refs/heads/master
Commit: a2db49f1b476c5cbd0253a6cba4a29c0e3c5584e
Parents: 7b52754
Author: Alex Panov <al...@teradata.com>
Authored: Thu Aug 20 17:53:52 2015 +0200
Committer: Alex Panov <al...@teradata.com>
Committed: Thu Aug 20 17:53:52 2015 +0200
----------------------------------------------------------------------
.../org/apache/storm/elasticsearch/ElasticsearchGetRequest.java | 4 ++--
.../storm/elasticsearch/bolt/EsLookupBoltIntegrationTest.java | 3 ++-
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/a2db49f1/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/ElasticsearchGetRequest.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/ElasticsearchGetRequest.java b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/ElasticsearchGetRequest.java
index 2fcbbf1..098c993 100644
--- a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/ElasticsearchGetRequest.java
+++ b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/ElasticsearchGetRequest.java
@@ -19,9 +19,9 @@ package org.apache.storm.elasticsearch;
import org.elasticsearch.action.get.GetRequest;
-import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.ITuple;
public interface ElasticsearchGetRequest {
- GetRequest extractFrom(Tuple tuple);
+ GetRequest extractFrom(ITuple tuple);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/a2db49f1/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltIntegrationTest.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltIntegrationTest.java b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltIntegrationTest.java
index 2a544f9..a869281 100644
--- a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltIntegrationTest.java
+++ b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltIntegrationTest.java
@@ -35,6 +35,7 @@ import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.ITuple;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
@@ -110,7 +111,7 @@ public class EsLookupBoltIntegrationTest extends AbstractEsBoltIntegrationTest<E
private class TestElasticsearchGetRequest implements ElasticsearchGetRequest {
@Override
- public GetRequest extractFrom(Tuple tuple) {
+ public GetRequest extractFrom(ITuple tuple) {
return node.client().prepareGet().setId(documentId).setIndex(indexName).setType(typeName).request();
}
}
[03/13] storm git commit: Extract ES client creator from ES bolts
Posted by ka...@apache.org.
Extract ES client creator from ES bolts
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3400c7db
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3400c7db
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3400c7db
Branch: refs/heads/master
Commit: 3400c7db4b70b6a61750f3d4ad1f948c49e9a34d
Parents: e0bee05
Author: Alex Panov <al...@teradata.com>
Authored: Thu Aug 20 12:28:08 2015 +0200
Committer: Alex Panov <al...@teradata.com>
Committed: Thu Aug 20 12:28:08 2015 +0200
----------------------------------------------------------------------
.../elasticsearch/bolt/AbstractEsBolt.java | 50 +++++++-------
.../elasticsearch/bolt/ElasticSearchClient.java | 57 ++++++++++++++++
.../elasticsearch/bolt/TransportAddresses.java | 72 ++++++++++++++++++++
.../bolt/TransportAddressesTest.java | 64 +++++++++++++++++
4 files changed, 216 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/3400c7db/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/AbstractEsBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/AbstractEsBolt.java b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/AbstractEsBolt.java
index ff1b543..a19a660 100644
--- a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/AbstractEsBolt.java
+++ b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/AbstractEsBolt.java
@@ -17,27 +17,26 @@
*/
package org.apache.storm.elasticsearch.bolt;
-import backtype.storm.task.OutputCollector;
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.topology.base.BaseRichBolt;
-import backtype.storm.tuple.Tuple;
+import java.util.Map;
+
import org.apache.storm.elasticsearch.common.EsConfig;
import org.elasticsearch.client.Client;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.settings.ImmutableSettings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.base.BaseRichBolt;
+import backtype.storm.tuple.Tuple;
public abstract class AbstractEsBolt extends BaseRichBolt {
+
private static final Logger LOG = LoggerFactory.getLogger(AbstractEsBolt.class);
+
protected static Client client;
+
protected OutputCollector collector;
private EsConfig esConfig;
@@ -51,19 +50,7 @@ public abstract class AbstractEsBolt extends BaseRichBolt {
this.collector = outputCollector;
synchronized (AbstractEsBolt.class) {
if (client == null) {
- Settings settings =
- ImmutableSettings.settingsBuilder().put("cluster.name", esConfig.getClusterName())
- .put("client.transport.sniff", "true").build();
- List<InetSocketTransportAddress> transportAddressList = new ArrayList<InetSocketTransportAddress>();
- for (String node : esConfig.getNodes()) {
- String[] hostAndPort = node.split(":");
- if(hostAndPort.length != 2){
- throw new IllegalArgumentException("incorrect Elasticsearch node format, should follow {host}:{port} pattern");
- }
- transportAddressList.add(new InetSocketTransportAddress(hostAndPort[0], Integer.parseInt(hostAndPort[1])));
- }
- client = new TransportClient(settings)
- .addTransportAddresses(transportAddressList.toArray(new InetSocketTransportAddress[transportAddressList.size()]));
+ client = new ElasticSearchClient(esConfig).construct();
}
}
} catch (Exception e) {
@@ -72,10 +59,19 @@ public abstract class AbstractEsBolt extends BaseRichBolt {
}
@Override
- public void execute(Tuple tuple) {
- }
+ public abstract void execute(Tuple tuple);
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
}
+
+ @VisibleForTesting
+ static Client getClient() {
+ return AbstractEsBolt.client;
+ }
+
+ @VisibleForTesting
+ static void replaceClient(Client client) {
+ AbstractEsBolt.client = client;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/3400c7db/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/ElasticSearchClient.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/ElasticSearchClient.java b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/ElasticSearchClient.java
new file mode 100644
index 0000000..0a9f4ea
--- /dev/null
+++ b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/ElasticSearchClient.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.elasticsearch.bolt;
+
+import java.io.Serializable;
+
+import org.apache.storm.elasticsearch.common.EsConfig;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+
+final class ElasticSearchClient implements Serializable {
+
+ private final EsConfig esConfig;
+
+ ElasticSearchClient(EsConfig esConfig) {
+ this.esConfig = esConfig;
+ }
+
+ Client construct() {
+ Settings settings = createBasicSettings();
+ TransportClient transportClient = new TransportClient(settings);
+ addTransportAddresses(transportClient);
+ return transportClient;
+ }
+
+ private Settings createBasicSettings() {
+ return ImmutableSettings.settingsBuilder()
+ .put("cluster.name", esConfig.getClusterName())
+ .put("client.transport.sniff", "true")
+ .build();
+ }
+
+ private void addTransportAddresses(TransportClient transportClient) {
+ Iterable<InetSocketTransportAddress> transportAddresses = new TransportAddresses(esConfig.getNodes());
+ for (InetSocketTransportAddress transportAddress : transportAddresses) {
+ transportClient.addTransportAddress(transportAddress);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3400c7db/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/TransportAddresses.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/TransportAddresses.java b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/TransportAddresses.java
new file mode 100644
index 0000000..f4479df
--- /dev/null
+++ b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/TransportAddresses.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.elasticsearch.bolt;
+
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+
+final class TransportAddresses implements Iterable<InetSocketTransportAddress> {
+
+ static final String DELIMETER = ":";
+
+ private final String[] nodes;
+
+ TransportAddresses(String[] nodes) {
+ if (nodes == null) {
+ throw new IllegalArgumentException("Elasticsearch hosts cannot be null");
+ }
+ if (nodes.length == 0) {
+ throw new IllegalArgumentException("At least one Elasticsearch host must be specified");
+ }
+
+ this.nodes = nodes;
+ }
+
+ @Override
+ public Iterator<InetSocketTransportAddress> iterator() {
+ List<InetSocketTransportAddress> result = new LinkedList<>();
+
+ for (String node : nodes) {
+ InetSocketTransportAddress transportAddress = transformToInetAddress(node);
+ result.add(transportAddress);
+ }
+
+ return result.iterator();
+ }
+
+ private InetSocketTransportAddress transformToInetAddress(String node) {
+ String[] hostAndPort = node.split(DELIMETER);
+ if (hostAndPort.length != 2) {
+ throw new IllegalArgumentException(
+ "Incorrect Elasticsearch node format, should follow {host}" + DELIMETER + "{port} pattern");
+ }
+ String hostname = hostname(hostAndPort[0]);
+ return new InetSocketTransportAddress(hostname, port(hostAndPort[1]));
+ }
+
+ private String hostname(String input) {
+ return input.trim();
+ }
+
+ private int port(String input) {
+ return Integer.parseInt(input.trim());
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3400c7db/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/TransportAddressesTest.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/TransportAddressesTest.java b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/TransportAddressesTest.java
new file mode 100644
index 0000000..ea07131
--- /dev/null
+++ b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/TransportAddressesTest.java
@@ -0,0 +1,64 @@
+package org.apache.storm.elasticsearch.bolt;
+
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.junit.Test;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.junit.Assert.assertThat;
+
+public class TransportAddressesTest {
+
+ @Test
+ public void readsMultipleHosts() throws Exception {
+ String[] hosts = new String[] {"h1:1000", "h2:10003"};
+ TransportAddresses addresses = new TransportAddresses(hosts);
+ assertThat(addresses, containsInAnyOrder(new InetSocketTransportAddress("h1", 1000),
+ new InetSocketTransportAddress("h2", 10003)));
+ }
+
+ @Test
+ public void stripsSpaces() throws Exception {
+ String[] hosts = new String[] {"h1:1000", " h2:10003 "};
+ TransportAddresses addresses = new TransportAddresses(hosts);
+ assertThat(addresses, containsInAnyOrder(new InetSocketTransportAddress("h1", 1000),
+ new InetSocketTransportAddress("h2", 10003)));
+ }
+
+ @Test
+ public void readsOneHost() throws Exception {
+ String[] hosts = new String[] {"h1:1000"};
+ TransportAddresses addresses = new TransportAddresses(hosts);
+ assertThat(addresses, containsInAnyOrder(new InetSocketTransportAddress("h1", 1000)));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void throwsOnNullHosts() throws Exception {
+ new TransportAddresses(null);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void throwsOnEmptyArray() throws Exception {
+ new TransportAddresses(new String[] {}).iterator();
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void throwsOnInvalidHostAndPortPair() throws Exception {
+ new TransportAddresses(new String[] {"h1:1000", "h2"}).iterator();
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void throwsOnInvalidPortValue() throws Exception {
+ new TransportAddresses(new String[] {"h1:-1000"}).iterator();
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void throwsOnPortNotANumber() throws Exception {
+ new TransportAddresses(new String[] {"h1:dummy"}).iterator();
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void throwsOnInvalidHostAndPortFormat() throws Exception {
+ new TransportAddresses(new String[] {"h1:dummy:231"}).iterator();
+ }
+
+}
[06/13] storm git commit: Make AbstractEsBoltTest public
Posted by ka...@apache.org.
Make AbstractEsBoltTest public
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7996697a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7996697a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7996697a
Branch: refs/heads/master
Commit: 7996697af5a1e8aa4bc63e80e04829b173d08d2d
Parents: 5999992
Author: Alex Panov <al...@teradata.com>
Authored: Thu Aug 20 13:07:31 2015 +0200
Committer: Alex Panov <al...@teradata.com>
Committed: Thu Aug 20 13:07:31 2015 +0200
----------------------------------------------------------------------
.../org/apache/storm/elasticsearch/bolt/AbstractEsBoltTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/7996697a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/AbstractEsBoltTest.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/AbstractEsBoltTest.java b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/AbstractEsBoltTest.java
index 8c5ea55..d9c09d0 100644
--- a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/AbstractEsBoltTest.java
+++ b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/AbstractEsBoltTest.java
@@ -28,7 +28,7 @@ import backtype.storm.Config;
import backtype.storm.task.OutputCollector;
@RunWith(MockitoJUnitRunner.class)
-abstract class AbstractEsBoltTest<Bolt extends AbstractEsBolt> {
+public abstract class AbstractEsBoltTest<Bolt extends AbstractEsBolt> {
protected static Config config = new Config();
[10/13] storm git commit: Add README entry for EsLookupBolt
Posted by ka...@apache.org.
Add README entry for EsLookupBolt
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/69c78ca1
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/69c78ca1
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/69c78ca1
Branch: refs/heads/master
Commit: 69c78ca18c62fd9142b8052c20952ed72fc42216
Parents: 4be31c5
Author: Alex Panov <al...@teradata.com>
Authored: Thu Aug 20 18:01:45 2015 +0200
Committer: Alex Panov <al...@teradata.com>
Committed: Thu Aug 20 18:01:45 2015 +0200
----------------------------------------------------------------------
external/storm-elasticsearch/README.md | 15 ++++++++++++++-
1 file changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/69c78ca1/external/storm-elasticsearch/README.md
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/README.md b/external/storm-elasticsearch/README.md
index 4fa1098..04a3673 100644
--- a/external/storm-elasticsearch/README.md
+++ b/external/storm-elasticsearch/README.md
@@ -34,6 +34,19 @@ EsPercolateBolt percolateBolt = new EsPercolateBolt(esConfig);
If there exists non-empty percolate response, EsPercolateBolt will emit tuple with original source and Percolate.Match
for each Percolate.Match in PercolateResponse.
+## EsLookupBolt (org.apache.storm.elasticsearch.bolt.EsLookupBolt)
+
+EsLookupBolt performs a get request to Elasticsearch. Incoming tuple is passed to provided GetRequest creator and the result of that execution is passed to Elasticsearch client.
+The bolt then uses the provider output adapter (EsLookupResultOutput) to convert the GetResponse to Values to emit.
+The output fields are also specified by the user of the bolt via the output adapter (EsLookupResultOutput).
+
+```java
+EsConfig esConfig = createEsConfig();
+ElasticsearchGetRequest getRequestAdapter = createElasticsearchGetRequest();
+EsLookupResultOutput output = createOutput();
+EsLookupBolt lookupBolt = new EsLookupBolt(esConfig, getRequestAdapter, output);
+```
+
## EsConfig (org.apache.storm.elasticsearch.common.EsConfig)
Two bolts above takes in EsConfig as a constructor arg.
@@ -69,4 +82,4 @@ Elasticsearch Trident state also follows similar pattern to EsBolts. It takes in
## Committer Sponsors
* Sriharsha Chintalapani ([@harshach](https://github.com/harshach))
- * Jungtaek Lim ([@HeartSaVioR](https://github.com/HeartSaVioR))
\ No newline at end of file
+ * Jungtaek Lim ([@HeartSaVioR](https://github.com/HeartSaVioR))
[12/13] storm git commit: Merge branch 'master' of
https://github.com/alexpanov/storm into STORM-978
Posted by ka...@apache.org.
Merge branch 'master' of https://github.com/alexpanov/storm into STORM-978
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/95985c58
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/95985c58
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/95985c58
Branch: refs/heads/master
Commit: 95985c58149369da63718cd05ecf62a5c19f4302
Parents: eb6107b 8618eb3
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Fri Aug 28 07:09:05 2015 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Fri Aug 28 07:09:05 2015 +0900
----------------------------------------------------------------------
external/storm-elasticsearch/README.md | 20 ++-
external/storm-elasticsearch/pom.xml | 10 +-
.../elasticsearch/ElasticsearchGetRequest.java | 36 +++++
.../elasticsearch/EsLookupResultOutput.java | 43 ++++++
.../elasticsearch/bolt/AbstractEsBolt.java | 50 ++++---
.../elasticsearch/bolt/ElasticSearchClient.java | 57 ++++++++
.../storm/elasticsearch/bolt/EsLookupBolt.java | 71 ++++++++++
.../elasticsearch/bolt/TransportAddresses.java | 72 ++++++++++
.../bolt/AbstractEsBoltIntegrationTest.java | 91 +++++++++++++
.../elasticsearch/bolt/AbstractEsBoltTest.java | 76 ++++-------
.../elasticsearch/bolt/EsIndexBoltTest.java | 33 +++--
.../bolt/EsLookupBoltIntegrationTest.java | 132 +++++++++++++++++++
.../elasticsearch/bolt/EsLookupBoltTest.java | 120 +++++++++++++++++
.../elasticsearch/bolt/EsPercolateBoltTest.java | 30 ++---
.../bolt/TransportAddressesTest.java | 81 ++++++++++++
15 files changed, 803 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
[07/13] storm git commit: Add missing Apache Licence headers
Posted by ka...@apache.org.
Add missing Apache Licence headers
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7b527548
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7b527548
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7b527548
Branch: refs/heads/master
Commit: 7b527548574719f2af7abd835cc30f097d320247
Parents: 7996697
Author: Alex Panov <al...@teradata.com>
Authored: Thu Aug 20 17:53:16 2015 +0200
Committer: Alex Panov <al...@teradata.com>
Committed: Thu Aug 20 17:53:16 2015 +0200
----------------------------------------------------------------------
.../elasticsearch/ElasticsearchGetRequest.java | 17 +++++++++++++++++
.../elasticsearch/bolt/EsLookupResultOutput.java | 17 +++++++++++++++++
.../storm/elasticsearch/bolt/EsLookupBoltTest.java | 17 +++++++++++++++++
.../elasticsearch/bolt/TransportAddressesTest.java | 17 +++++++++++++++++
4 files changed, 68 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/7b527548/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/ElasticsearchGetRequest.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/ElasticsearchGetRequest.java b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/ElasticsearchGetRequest.java
index 442af3c..2fcbbf1 100644
--- a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/ElasticsearchGetRequest.java
+++ b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/ElasticsearchGetRequest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.storm.elasticsearch;
import org.elasticsearch.action.get.GetRequest;
http://git-wip-us.apache.org/repos/asf/storm/blob/7b527548/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupResultOutput.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupResultOutput.java b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupResultOutput.java
index 8696c6b..c21fba6 100644
--- a/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupResultOutput.java
+++ b/external/storm-elasticsearch/src/main/java/org/apache/storm/elasticsearch/bolt/EsLookupResultOutput.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.storm.elasticsearch.bolt;
import java.util.Collection;
http://git-wip-us.apache.org/repos/asf/storm/blob/7b527548/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltTest.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltTest.java b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltTest.java
index 602ed6d..9f19036 100644
--- a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltTest.java
+++ b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/EsLookupBoltTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.storm.elasticsearch.bolt;
import java.util.Collections;
http://git-wip-us.apache.org/repos/asf/storm/blob/7b527548/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/TransportAddressesTest.java
----------------------------------------------------------------------
diff --git a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/TransportAddressesTest.java b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/TransportAddressesTest.java
index ea07131..411f187 100644
--- a/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/TransportAddressesTest.java
+++ b/external/storm-elasticsearch/src/test/java/org/apache/storm/elasticsearch/bolt/TransportAddressesTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.storm.elasticsearch.bolt;
import org.elasticsearch.common.transport.InetSocketTransportAddress;