You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by sr...@apache.org on 2015/08/27 19:03:52 UTC
[3/5] storm git commit: STORM-851: Storm Solr Connector 1. Address
code review README.md Javadocs Apache Copyright 2. Fixed Multivalue fields
bug
STORM-851: Storm Solr Connector
1. Address code review
README.md
Javadocs
Apache Copyright
2. Fixed Multivalue fields bug
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8552f43c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8552f43c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8552f43c
Branch: refs/heads/master
Commit: 8552f43c4704a83488c235a0a8c1cb6da883304d
Parents: 4ab6e0c
Author: Hugo Louro <hm...@gmail.com>
Authored: Fri Aug 21 16:51:53 2015 -0700
Committer: Hugo Louro <hm...@gmail.com>
Committed: Tue Aug 25 16:23:51 2015 -0700
----------------------------------------------------------------------
external/storm-solr/README.md | 201 +++++++++++++++++++
external/storm-solr/pom.xml | 48 ++---
.../java/org/apache/storm/solr/bolt/.gitignore | 1 -
.../storm/solr/bolt/AbstractSolrBolt.java | 33 ---
.../apache/storm/solr/bolt/SolrUpdateBolt.java | 83 +++++---
.../storm/solr/config/CountBasedCommit.java | 30 ++-
.../storm/solr/config/SolrCommitStrategy.java | 20 +-
.../apache/storm/solr/config/SolrConfig.java | 24 ++-
.../org/apache/storm/solr/mapper/.gitignore | 1 -
.../storm/solr/mapper/SolrFieldsMapper.java | 74 +++++--
.../storm/solr/mapper/SolrJsonMapper.java | 101 ++++++++--
.../apache/storm/solr/mapper/SolrMapper.java | 21 +-
.../storm/solr/mapper/SolrMapperException.java | 21 +-
.../org/apache/storm/solr/schema/CopyField.java | 21 +-
.../org/apache/storm/solr/schema/Field.java | 31 +--
.../org/apache/storm/solr/schema/FieldType.java | 21 +-
.../org/apache/storm/solr/schema/Schema.java | 23 ++-
.../storm/solr/schema/SolrFieldTypeFinder.java | 43 ++--
.../schema/builder/RestJsonSchemaBuilder.java | 22 +-
.../solr/schema/builder/SchemaBuilder.java | 21 +-
.../apache/storm/solr/trident/SolrState.java | 1 +
.../storm/solr/trident/SolrStateFactory.java | 2 +-
.../apache/storm/solr/trident/SolrUpdater.java | 1 +
.../storm/solr/spout/SolrFieldsSpout.java | 26 ++-
.../apache/storm/solr/spout/SolrJsonSpout.java | 60 +++---
.../org/apache/storm/solr/topology/.gitignore | 1 -
.../storm/solr/topology/SolrFieldsTopology.java | 28 ++-
.../storm/solr/topology/SolrJsonTopology.java | 23 ++-
.../storm/solr/topology/SolrTopology.java | 23 ++-
.../solr/trident/SolrFieldsTridentTopology.java | 31 ++-
.../solr/trident/SolrJsonTridentTopology.java | 25 ++-
.../org/apache/storm/solr/util/TestUtil.java | 25 ++-
pom.xml | 5 +-
33 files changed, 834 insertions(+), 257 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/README.md
----------------------------------------------------------------------
diff --git a/external/storm-solr/README.md b/external/storm-solr/README.md
new file mode 100644
index 0000000..915258e
--- /dev/null
+++ b/external/storm-solr/README.md
@@ -0,0 +1,201 @@
+# Storm Solr
+Storm and Trident integration for Apache Solr. This package includes a bolt and a trident state that enable a Storm topology
+stream the contents of storm tuples to index Solr collections.
+
+# Index Storm tuples into a Solr collection
+The The bolt and trident state provided use one of the supplied mappers to build a `SolrRequest` object that is
+responsible for making the update calls to Solr, thus updating the index of the collection specified.
+
+# Usage Examples
+In this section we provide some simple code snippets on how to build Storm and Trident topologies to index Solr. In subsequent sections we
+describe in detail the two key components of the Storm Solr integration, the `SolrUpdateBolt`, and the `Mappers`, `SolrFieldsMapper`, and `SolrJsonMapper`.
+
+## Storm Bolt With JSON Mapper and Count Based Commit Strategy
+
+```java
+ new SolrUpdateBolt(solrConfig, solrMapper, solrCommitStgy)
+
+ // zkHostString for Solr 'gettingstarted' example
+ SolrConfig solrConfig = new SolrConfig("127.0.0.1:9983");
+
+ // JSON Mapper used to generate 'SolrRequest' requests to update the "gettingstarted" Solr collection with JSON content declared the tuple field with name "JSON"
+ SolrMapper solrMapper = new SolrJsonMapper.Builder("gettingstarted", "JSON").build();
+
+ // Acks every other five tuples. Setting to null acks every tuple
+ SolrCommitStrategy solrCommitStgy = new CountBasedCommit(5);
+```
+
+## Trident Topology With Fields Mapper
+```java
+ new SolrStateFactory(solrConfig, solrMapper);
+
+ // zkHostString for Solr 'gettingstarted' example
+ SolrConfig solrConfig = new SolrConfig("127.0.0.1:9983");
+
+ /* Solr Fields Mapper used to generate 'SolrRequest' requests to update the "gettingstarted" Solr collection. The Solr index is updated using the field values of the tuple fields that match static or dynamic fields declared in the schema object build using schemaBuilder */
+ SolrMapper solrMapper = new SolrFieldsMapper.Builder(schemaBuilder, "gettingstarted").build();
+
+ // builds the Schema object from the JSON representation of the schema as returned by the URL http://localhost:8983/solr/gettingstarted/schema/
+ SchemaBuilder schemaBuilder = new RestJsonSchemaBuilder("localhost", "8983", "gettingstarted")
+```
+
+## SolrUpdateBolt
+ `SolrUpdateBolt` streams tuples directly into Apache Solr. The Solr index is updated using`SolrRequest` requests.
+ The `SolrUpdateBolt` is configurable using implementations of `SolrConfig`, `SolrMapper`, and optionally `SolrCommitStrategy`.
+
+ The data to stream onto Solr is extracted from the tuples using the strategy defined in the `SolrMapper` implementation.
+
+ The `SolrRquest` can be sent every tuple, or according to a strategy defined by `SolrCommitStrategy` implementations.
+ If a `SolrCommitStrategy` is in place and one of the tuples in the batch fails, the batch is not committed, and all the tuples in that
+ batch are marked as Fail, and retried. On the other hand, if all tuples succeed, the `SolrRequest` is committed and all tuples are successfully acked.
+
+ `SolrConfig` is the class containing Solr configuration to be made available to Storm Solr bolts. Any configuration needed in the bolts should be put in this class.
+
+
+## SolrMapper
+`SorlMapper` implementations define the strategy to extract information from the tuples. The public method
+`toSolrRequest` receives a tuple or a list of tuples and returns a `SolrRequest` object that is used to update the Solr index.
+
+
+### SolrJsonMapper
+The `SolrJsonMapper` creates a Solr update request that is sent to the URL endpoint defined by Solr as the resource
+destination for requests in JSON format.
+
+To create a `SolrJsonMapper` the client must specify the name of the collection to update as well as the
+tuple field that contains the JSON object used to update the Solr index. If the tuple does not contain the field specified,
+a `SolrMapperException` will be thrown when the method `toSolrRequest`is called. If the field exists, its value can either
+be a String with the contents in JSON format, or a Java object that will be serialized to JSON
+
+Code snippet illustrating how to create a `SolrJsonMapper` object to update the `gettingstarted` Solr collection with JSON content
+declared in the tuple field with name "JSON"
+``` java
+ SolrMapper solrMapper = new SolrJsonMapper.Builder("gettingstarted", "JSON").build();
+```
+
+
+### SolrFieldsMapper
+The `SolrFieldsMapper` creates a Solr update request that is sent to the Solr URL endpoint that handles the updates of `SolrInputDocument` objects.
+
+To create a `SolrFieldsMapper` the client must specify the name of the collection to update as well as the `SolrSchemaBuilder`.
+The Solr `Schema` is used to extract information about the Solr schema fields and corresponding types. This metadata is used
+to get the information from the tuples. Only tuple fields that match a static or dynamic Solr fields are added to the document. Tuple fields
+that do not match the schema are not added to the `SolrInputDocument` being prepared for indexing. A debug log message is printed for the
+ tuple fields that do not match the schema and hence are not indexed.
+
+
+The `SolrFieldsMapper` supports multivalue fields. A multivalue tuple field must be tokenized. The default token is |. Any
+arbitrary token can be specified by calling the method `org.apache.storm.solr.mapper.SolrFieldsMapper.Builder.setMultiValueFieldToken`
+that is part of the `SolrFieldsMapper.Builder` builder class.
+
+Code snippet illustrating how to create a `SolrFieldsMapper` object to update the `gettingstarted` Solr collection. The multivalue
+field separates each value with the token % instead of the default | . To use the default token you can ommit the call to the method
+`setMultiValueFieldToken`.
+
+``` java
+ new SolrFieldsMapper.Builder(
+ new RestJsonSchemaBuilder("localhost", "8983", "gettingstarted"), "gettingstarted")
+ .setMultiValueFieldToken("%").build();
+```
+
+# Build And Run Bundled Examples
+To be able to run the examples you must first build the java code in the package `storm-solr`,
+and then generate an uber jar with all the dependencies.
+
+
+## Build the Storm Apache Solr Integration Code
+
+`mvn clean install -f REPO_HOME/storm/external/storm-solr/pom.xml`
+
+## Use the Maven Shade Plugin to Build the Uber Jar
+
+ Add the following to `REPO_HOME/storm/external/storm-solr/pom.xml`
+
+ ```
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.4.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.apache.storm.solr.topology.SolrJsonTopology</mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+</plugin>
+ ```
+
+create the uber jar by running the commmand:
+
+`mvn package -f REPO_HOME/storm/external/storm-solr/pom.xml`
+
+This will create the uber jar file with the name and location matching the following pattern:
+
+`REPO_HOME/storm/external/storm/target/storm-solr-0.11.0-SNAPSHOT.jar`
+
+## Run Examples
+Copy the file `REPO_HOME/storm/external/storm-solr/target/storm-solr-0.11.0-SNAPSHOT.jar` to `STORM_HOME/extlib`
+
+**The code examples provided require that you first run the [Solr gettingstarted](http://lucene.apache.org/solr/quickstart.html) example**
+
+### Run Storm Topology
+
+STORM_HOME/bin/storm jar REPO_HOME/storm/external/storm-solr/target/storm-solr-0.11.0-SNAPSHOT-tests.jar org.apache.storm.solr.topology.SolrFieldsTopology
+
+STORM_HOME/bin/storm jar REPO_HOME/storm/external/storm-solr/target/storm-solr-0.11.0-SNAPSHOT-tests.jar org.apache.storm.solr.topology.SolrJsonTopology
+
+### Run Trident Topology
+
+STORM_HOME/bin/storm jar REPO_HOME/storm/external/storm-solr/target/storm-solr-0.11.0-SNAPSHOT-tests.jar org.apache.storm.solr.trident.SolrFieldsTridentTopology
+
+STORM_HOME/bin/storm jar REPO_HOME/storm/external/storm-solr/target/storm-solr-0.11.0-SNAPSHOT-tests.jar org.apache.storm.solr.trident.SolrJsonTridentTopology
+
+
+### Verify Results
+
+The aforementioned Storm and Trident topologies index the Solr `gettingstarted` collection with objects that have the following `id` pattern:
+
+\*id_fields_test_val\* for `SolrFieldsTopology` and `SolrFieldsTridentTopology`
+
+\*json_test_val\* for `SolrJsonTopology` and `SolrJsonTridentTopology`
+
+Querying Solr for these patterns, you will see the values that have been indexed by the Storm Apache Solr integration:
+
+curl -X GET -H "Content-type:application/json" -H "Accept:application/json" http://localhost:8983/solr/gettingstarted_shard1_replica2/select?q=*id_fields_test_val*&wt=json&indent=true
+
+curl -X GET -H "Content-type: application/json" -H "Accept: application/json" http://localhost:8983/solr/gettingstarted_shard1_replica2/select?q=*id_fields_test_val*&wt=json&indent=true
+
+You can also see the results by opening the Apache Solr UI and pasting the `id` pattern in the `q` textbox in the queries page
+
+http://localhost:8983/solr/#/gettingstarted_shard1_replica2/query
+
+## License
+
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+
+## Committer Sponsors
+ * Sriharsha Chintalapani ([sriharsha@apache.org](mailto:sriharsha@apache.org))
+ * P. Taylor Goetz ([ptgoetz@apache.org](mailto:ptgoetz@apache.org))
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-solr/pom.xml b/external/storm-solr/pom.xml
index 4213e0c..2b5c742 100644
--- a/external/storm-solr/pom.xml
+++ b/external/storm-solr/pom.xml
@@ -35,18 +35,6 @@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>org.apache.solr</groupId>
- <artifactId>solr-core</artifactId>
- <version>5.2.1</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.solr</groupId>
- <artifactId>solr-test-framework</artifactId>
- <version>5.2.1</version>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.3</version>
@@ -61,7 +49,24 @@
<artifactId>commons-io</artifactId>
<version>1.4</version>
</dependency>
- <!--test-->
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-core</artifactId>
+ <version>5.2.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-test-framework</artifactId>
+ <version>5.2.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>17.0</version>
+ </dependency>
+ <!--test dependencies -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
@@ -73,10 +78,6 @@
<artifactId>gson</artifactId>
<version>2.3.1</version>
</dependency>
- <!--<dependency>-->
- <!--<groupId>com.googlecode.json-simple</groupId>-->
- <!--<artifactId>json-simple</artifactId>-->
- <!--</dependency>-->
</dependencies>
<build>
<plugins>
@@ -92,19 +93,6 @@
</execution>
</executions>
</plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <archive>
- <manifest>
- <mainClass>org.apache.storm.solr.topology.SolrFieldsTopology</mainClass>
- </manifest>
- </archive>
- <descriptorRefs>
- <descriptorRef>jar-with-dependencies</descriptorRef>
- </descriptorRefs>
- </configuration>
- </plugin>
</plugins>
</build>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/.gitignore
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/.gitignore b/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/.gitignore
deleted file mode 100644
index 1adc831..0000000
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-# Created by .ignore support plugin (hsz.mobi)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/AbstractSolrBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/AbstractSolrBolt.java b/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/AbstractSolrBolt.java
deleted file mode 100644
index cd3261f..0000000
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/AbstractSolrBolt.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package org.apache.storm.solr.bolt;
-
-import backtype.storm.task.OutputCollector;
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.topology.base.BaseRichBolt;
-import backtype.storm.tuple.Tuple;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.storm.solr.config.SolrConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-
-/**
- * Created by hlouro on 7/17/15.
- */
-public abstract class AbstractSolrBolt extends BaseRichBolt {
- protected OutputCollector collector;
- protected SolrConfig solrConfig;
- protected SolrClient solrClient;
-
- public AbstractSolrBolt(SolrConfig solrConfig) {
- this.solrConfig = solrConfig;
- }
-
- @Override
- public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
- this.collector = collector;
- solrClient = new CloudSolrClient(solrConfig.getZkHostString());
- }
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/SolrUpdateBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/SolrUpdateBolt.java b/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/SolrUpdateBolt.java
index fe90b73..4a9599e 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/SolrUpdateBolt.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/SolrUpdateBolt.java
@@ -1,11 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.bolt;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Tuple;
+import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.storm.solr.config.CountBasedCommit;
import org.apache.storm.solr.config.SolrCommitStrategy;
import org.apache.storm.solr.config.SolrConfig;
import org.apache.storm.solr.mapper.SolrMapper;
@@ -13,41 +35,49 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-/**
- * Created by hlouro on 7/19/15.
- */
-public class SolrUpdateBolt extends AbstractSolrBolt {
- private final Logger logger = LoggerFactory.getLogger(SolrUpdateBolt.class);
+public class SolrUpdateBolt extends BaseRichBolt {
+ private static final Logger logger = LoggerFactory.getLogger(SolrUpdateBolt.class);
+
+ private final SolrConfig solrConfig;
private final SolrMapper solrMapper;
- private final SolrCommitStrategy commitStgy;
- private List<Tuple> toCommitTuples;
- private final String ackFailLock = "LOCK"; //serializable lock
+ private final SolrCommitStrategy commitStgy; // if null, acks every tuple
+ private SolrClient solrClient;
+ private OutputCollector collector;
+ private List<Tuple> toCommitTuples;
public SolrUpdateBolt(SolrConfig solrConfig, SolrMapper solrMapper) {
this(solrConfig, solrMapper, null);
}
public SolrUpdateBolt(SolrConfig solrConfig, SolrMapper solrMapper, SolrCommitStrategy commitStgy) {
- super(solrConfig);
+ this.solrConfig = solrConfig;
this.solrMapper = solrMapper;
this.commitStgy = commitStgy;
- logger.info("Created {} with the following configuration: " +
+ logger.debug("Created {} with the following configuration: " +
"[SolrConfig = {}], [SolrMapper = {}], [CommitStgy = {}]",
this.getClass().getSimpleName(), solrConfig, solrMapper, commitStgy);
}
- @Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
- super.prepare(stormConf, context, collector);
- this.toCommitTuples = new LinkedList<>();
+ this.collector = collector;
+ this.solrClient = new CloudSolrClient(solrConfig.getZkHostString());
+ this.toCommitTuples = new ArrayList<>(capacity());
+
+ }
+
+ private int capacity() {
+ final int defArrListCpcty = 10;
+ return (commitStgy instanceof CountBasedCommit) ?
+ ((CountBasedCommit)commitStgy).getThreshold() :
+ defArrListCpcty;
}
- @Override
public void execute(Tuple tuple) {
try {
SolrRequest request = solrMapper.toSolrRequest(tuple);
@@ -62,10 +92,8 @@ public class SolrUpdateBolt extends AbstractSolrBolt {
if (commitStgy == null) {
collector.ack(tuple);
} else {
- synchronized(ackFailLock) {
- toCommitTuples.add(tuple);
- commitStgy.update();
- }
+ toCommitTuples.add(tuple);
+ commitStgy.update();
if (commitStgy.commit()) {
solrClient.commit(solrMapper.getCollection());
ackCommittedTuples();
@@ -87,21 +115,22 @@ public class SolrUpdateBolt extends AbstractSolrBolt {
collector.fail(tuple);
} else {
List<Tuple> failedTuples = getQueuedTuples();
- for (Tuple failedTuple : failedTuples) {
- collector.fail(failedTuple);
- }
+ failQueuedTuples(failedTuples);
}
}
- private List<Tuple> getQueuedTuples() {
- List<Tuple> queuedTuples;
- synchronized(ackFailLock) {
- queuedTuples = toCommitTuples;
- toCommitTuples = new LinkedList<>();
+ private void failQueuedTuples(List<Tuple> failedTuples) {
+ for (Tuple failedTuple : failedTuples) {
+ collector.fail(failedTuple);
}
+ }
+
+ private List<Tuple> getQueuedTuples() {
+ List<Tuple> queuedTuples = toCommitTuples;
+ toCommitTuples = new ArrayList<>(capacity());
return queuedTuples;
}
- @Override
public void declareOutputFields(OutputFieldsDeclarer declarer) { }
+
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/config/CountBasedCommit.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/config/CountBasedCommit.java b/external/storm-solr/src/main/java/org/apache/storm/solr/config/CountBasedCommit.java
index 3877980..6bd1220 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/config/CountBasedCommit.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/config/CountBasedCommit.java
@@ -1,13 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.config;
/**
* Class defining a count based commit strategy. When the count reaches the commit threshold,
* SolrInputDocuments are committed to Solr.
- *
- * Created by hlouro on 7/29/15.
*/
public class CountBasedCommit implements SolrCommitStrategy {
- private int threshHold;
+ private int threshold;
private int count;
/**
@@ -19,12 +35,12 @@ public class CountBasedCommit implements SolrCommitStrategy {
if (threshold < 1) {
throw new IllegalArgumentException("Threshold must be a positive integer: " + threshold);
}
- this.threshHold = threshold;
+ this.threshold = threshold;
}
@Override
public boolean commit() {
- return count != 0 && count % threshHold == 0;
+ return count != 0 && count % threshold == 0;
}
@@ -36,4 +52,8 @@ public class CountBasedCommit implements SolrCommitStrategy {
public int getCount() {
return count;
}
+
+ public int getThreshold() {
+ return threshold;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrCommitStrategy.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrCommitStrategy.java b/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrCommitStrategy.java
index 0ec9149..77acb95 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrCommitStrategy.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrCommitStrategy.java
@@ -1,11 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.config;
import java.io.Serializable;
/**
* Strategy definining when the Solr Bolt should commit the request to Solr.
- * <p></p>
- * Created by hlouro on 7/29/15.
*/
public interface SolrCommitStrategy extends Serializable {
boolean commit();
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrConfig.java b/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrConfig.java
index 6a75ae7..009430c 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrConfig.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrConfig.java
@@ -1,20 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.config;
-import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
import java.io.Serializable;
/**
* Class containing Solr configuration to be made available to Storm Solr bolts. Any configuration needed in
* the bolts should be put in this class.
- * <p></p>
- * Created by hlouro on 7/29/15.
*/
public class SolrConfig implements Serializable {
private String zkHostString;
/**
- * @param zkHostString Zookeeper host string as defined in the {@link SolrClient} constructor
+ * @param zkHostString Zookeeper host string as defined in the {@link CloudSolrClient} constructor
* */
public SolrConfig(String zkHostString) {
this.zkHostString = zkHostString;
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/.gitignore
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/.gitignore b/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/.gitignore
deleted file mode 100644
index 1adc831..0000000
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-# Created by .ignore support plugin (hsz.mobi)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrFieldsMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrFieldsMapper.java b/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrFieldsMapper.java
index df89a47..9fa38cd 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrFieldsMapper.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrFieldsMapper.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.mapper;
import static org.apache.storm.solr.schema.SolrFieldTypeFinder.FieldTypeWrapper;
@@ -8,7 +26,7 @@ import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrInputDocument;
-import org.apache.storm.solr.schema.Field;
+import org.apache.storm.solr.schema.FieldType;
import org.apache.storm.solr.schema.Schema;
import org.apache.storm.solr.schema.builder.SchemaBuilder;
import org.apache.storm.solr.schema.SolrFieldTypeFinder;
@@ -18,11 +36,8 @@ import org.slf4j.LoggerFactory;
import java.util.LinkedList;
import java.util.List;
-/**
- * Created by hlouro on 7/24/15.
- */
public class SolrFieldsMapper implements SolrMapper {
- private static final Logger logger = LoggerFactory.getLogger(SolrFieldsMapper.class);
+ private static final Logger log = LoggerFactory.getLogger(SolrFieldsMapper.class);
private String collection;
private SolrFieldTypeFinder typeFinder;
private String multiValueFieldToken;
@@ -32,7 +47,22 @@ public class SolrFieldsMapper implements SolrMapper {
private SolrFieldTypeFinder typeFinder;
private String multiValueFieldToken = "|";
- public Builder(SchemaBuilder schemaBuilder) {
+ /**
+ * {@link SolrFieldsMapper} builder class.
+ * @param schemaBuilder Solr {@link Schema} builder class
+ * @param collection Name of the Solr collection to update using the SolrRequest
+ */
+ public Builder(SchemaBuilder schemaBuilder, String collection) {
+ setTypeFinder(schemaBuilder);
+ this.collection = collection;
+ }
+
+ /**
+ * {@link SolrFieldsMapper} builder class.
+ * @param schemaBuilder Solr {@link Schema} builder class
+ * @param solrClient {@link SolrClient} implementation from where to extract the default Solr collection, if any defined.
+ */
+ public Builder(SchemaBuilder schemaBuilder, SolrClient solrClient) {
setTypeFinder(schemaBuilder);
}
@@ -42,18 +72,13 @@ public class SolrFieldsMapper implements SolrMapper {
typeFinder = new SolrFieldTypeFinder(schema);
}
- public Builder setCollection(String collection) {
- this.collection = collection;
- return this;
- }
-
- public Builder setDefaultCollection(SolrClient solrClient) {
+ // Sets {@link SolrFieldsMapper} to use the default Solr collection if there is one defined
+ private void setDefaultCollection(SolrClient solrClient) {
String defaultCollection = null;
if (solrClient instanceof CloudSolrClient) {
defaultCollection = ((CloudSolrClient) solrClient).getDefaultCollection();
}
this.collection = defaultCollection;
- return this;
}
/**
@@ -73,6 +98,8 @@ public class SolrFieldsMapper implements SolrMapper {
this.collection = builder.collection;
this.typeFinder = builder.typeFinder;
this.multiValueFieldToken = builder.multiValueFieldToken;
+ log.debug("Created {} with the following configuration: [{}] "
+ + this.getClass().getSimpleName(), this.toString());
}
@Override
@@ -88,6 +115,7 @@ public class SolrFieldsMapper implements SolrMapper {
}
UpdateRequest request = new UpdateRequest();
request.add(docs);
+ log.debug("Created SolrRequest with content: {}", docs);
return request;
}
@@ -96,6 +124,7 @@ public class SolrFieldsMapper implements SolrMapper {
SolrInputDocument doc = buildDocument(tuple);
UpdateRequest request = new UpdateRequest();
request.add(doc);
+ log.debug("Created SolrRequest with content: {}", doc);
return request;
}
@@ -105,14 +134,14 @@ public class SolrFieldsMapper implements SolrMapper {
for (String tupleField : tuple.getFields()) {
FieldTypeWrapper fieldTypeWrapper = typeFinder.getFieldTypeWrapper(tupleField);
if (fieldTypeWrapper != null) {
- Field field = fieldTypeWrapper.getField();
- if (field.isMultiValued()) {
+ FieldType fieldType = fieldTypeWrapper.getType();
+ if (fieldType.isMultiValued()) {
addMultivalueFieldToDoc(doc, tupleField, tuple);
} else {
addFieldToDoc(doc, tupleField, tuple);
}
} else {
- logger.info("Field [{}] does NOT match static or dynamic field declared in schema. Not added to document", tupleField);
+ log.debug("Field [{}] does NOT match static or dynamic field declared in schema. Not added to document", tupleField);
}
}
return doc;
@@ -120,14 +149,14 @@ public class SolrFieldsMapper implements SolrMapper {
private void addFieldToDoc(SolrInputDocument doc, String tupleField, ITuple tuple) {
Object val = getValue(tupleField, tuple);
- logger.info("Adding to document (field, val) = ({}, {})", tupleField, val);
+ log.debug("Adding to document (field, val) = ({}, {})", tupleField, val);
doc.addField(tupleField, val);
}
private void addMultivalueFieldToDoc(SolrInputDocument doc, String tupleField, ITuple tuple) {
String[] values = getValues(tupleField, tuple);
for (String value : values) {
- logger.info("Adding {} to multivalue field document {}", value, tupleField);
+ log.debug("Adding {} to multivalue field document {}", value, tupleField);
doc.addField(tupleField, value);
}
}
@@ -141,4 +170,13 @@ public class SolrFieldsMapper implements SolrMapper {
String[] values = multiValueField.split(multiValueFieldToken);
return values;
}
+
+ @Override
+ public String toString() {
+ return "SolrFieldsMapper{" +
+ "collection='" + collection + '\'' +
+ ", typeFinder=" + typeFinder +
+ ", multiValueFieldToken='" + multiValueFieldToken + '\'' +
+ '}';
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrJsonMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrJsonMapper.java b/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrJsonMapper.java
index ef5bb3c..f9d6e9b 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrJsonMapper.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrJsonMapper.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.mapper;
import backtype.storm.tuple.ITuple;
@@ -13,30 +31,66 @@ import org.slf4j.LoggerFactory;
import java.util.List;
-/**
- * Created by hlouro on 7/24/15.
- */
public class SolrJsonMapper implements SolrMapper {
private static final Logger logger = LoggerFactory.getLogger(SolrJsonMapper.class);
- private static final String JSON_UPDATE_URL = "/update/json/docs";
- private static final String CONTENT_TYPE = "application/json";;
+ private static final String CONTENT_TYPE = "application/json";
+ private final String jsonUpdateUrl;
private final String jsonTupleField;
private final String collection;
- public SolrJsonMapper(String collection, String jsonTupleField) {
- this.collection = collection;
- this.jsonTupleField = jsonTupleField;
- }
+ public static class Builder {
+ private String jsonUpdateUrl = "/update/json/docs";
+ private final String jsonTupleField;
+ private final String collection;
+
+ /**
+ * {@link SolrJsonMapper} builder class.
+ * @param collection Name of the Solr collection to update using the SolrRequest
+ * @param jsonTupleField Name of the tuple field that contains the JSON object used to update the Solr index
+ */
+ public Builder(String collection, String jsonTupleField) {
+ this.collection = collection;
+ this.jsonTupleField = jsonTupleField;
+ }
- /** Uses default collection */
- public SolrJsonMapper(SolrClient solrClient, String jsonTupleField) {
- String defaultCollection = null;
- if (solrClient instanceof CloudSolrClient) {
- defaultCollection = ((CloudSolrClient) solrClient).getDefaultCollection();
+ /**
+ * {@link SolrJsonMapper} builder class.
+ * @param solrClient {@link SolrClient} implementation from where to extract the default Solr collection, if any defined.
+ * @param jsonTupleField Name of the tuple field that contains the JSON object used to update the Solr index
+ */
+ public Builder(SolrClient solrClient, String jsonTupleField) {
+ String defaultCollection = null;
+ if (solrClient instanceof CloudSolrClient) {
+ defaultCollection = ((CloudSolrClient) solrClient).getDefaultCollection();
+ }
+ this.collection = defaultCollection;
+ this.jsonTupleField = jsonTupleField;
+ }
+
+ /**
+ * Sets the URL endpoint where to send {@link SolrRequest} requests in JSON format. This URL is defined
+ * by Solr and for the most part you will not have to call this method to to override this value.
+ * Currently the default value is /update/json/docs.
+ * This method is available to support future evolutions of the Solr API.
+ *
+ * @param jsonUpdateUrl URL endpoint to where send {@link SolrRequest}
+ * @return {@link Builder object} used to build instances of {@link SolrJsonMapper}
+ */
+ public Builder setJsonUpdateUrl(String jsonUpdateUrl) {
+ this.jsonUpdateUrl = jsonUpdateUrl;
+ return this;
+ }
+
+ public SolrJsonMapper build() {
+ return new SolrJsonMapper(this);
}
- this.collection = defaultCollection;
- this.jsonTupleField = jsonTupleField;
+ }
+
+ private SolrJsonMapper(Builder builder) {
+ jsonTupleField = builder.jsonTupleField ;
+ collection = builder.collection;
+ jsonUpdateUrl = builder.jsonUpdateUrl;
}
@Override
@@ -44,6 +98,12 @@ public class SolrJsonMapper implements SolrMapper {
return collection;
}
+ /**
+ *
+ * @param tuples
+ * @return
+ * @throws SolrMapperException if the tuple does not contain the
+ */
@Override
public SolrRequest toSolrRequest(List<? extends ITuple> tuples) throws SolrMapperException {
final String jsonList = getJsonFromTuples(tuples);
@@ -57,13 +117,16 @@ public class SolrJsonMapper implements SolrMapper {
}
private SolrRequest createtSolrRequest(String json) {
- ContentStreamUpdateRequest request = new ContentStreamUpdateRequest(JSON_UPDATE_URL);
- ContentStream cs = new ContentStreamBase.StringStream(json, CONTENT_TYPE);
+ final ContentStreamUpdateRequest request = new ContentStreamUpdateRequest(jsonUpdateUrl);
+ final ContentStream cs = new ContentStreamBase.StringStream(json, CONTENT_TYPE);
request.addContentStream(cs);
- logger.info("Request generated with JSON: " + json);
+ if (logger.isDebugEnabled()) {
+ logger.debug("Request generated with JSON: " + json);
+ }
return request;
}
+ // Builds a JSON list
private String getJsonFromTuples(List<? extends ITuple> tuples) throws SolrMapperException {
final StringBuilder jsonListBuilder = new StringBuilder("[");
for (ITuple tuple : tuples) {
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapper.java b/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapper.java
index 165498d..96242d1 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapper.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapper.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.mapper;
import backtype.storm.tuple.ITuple;
@@ -7,9 +25,6 @@ import org.apache.solr.client.solrj.SolrRequest;
import java.io.Serializable;
import java.util.List;
-/**
- * Created by hlouro on 7/22/15.
- */
public interface SolrMapper extends Serializable {
String getCollection();
SolrRequest toSolrRequest(ITuple tuple) throws SolrMapperException;
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapperException.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapperException.java b/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapperException.java
index b9ad5a3..654ee21 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapperException.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapperException.java
@@ -1,8 +1,23 @@
-package org.apache.storm.solr.mapper;
-
/**
- * Created by hlouro on 7/24/15.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
+
+package org.apache.storm.solr.mapper;
+
public class SolrMapperException extends Exception {
public SolrMapperException(String msg) {
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/schema/CopyField.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/CopyField.java b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/CopyField.java
index 9fd291e..f07584a 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/CopyField.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/CopyField.java
@@ -1,10 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.schema;
import java.io.Serializable;
-/**
- * Created by hlouro on 7/27/15.
- */
public class CopyField implements Serializable {
private String source;
private String dest;
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Field.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Field.java b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Field.java
index 55f9589..99b14a2 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Field.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Field.java
@@ -1,14 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.schema;
import java.io.Serializable;
-/**
- * Created by hlouro on 7/27/15.
- */
public class Field implements Serializable {
private String name;
private String type;
- private boolean multiValued;
public String getName() {
return name;
@@ -18,10 +32,6 @@ public class Field implements Serializable {
return type;
}
- public boolean isMultiValued() {
- return multiValued;
- }
-
public void setName(String name) {
this.name = name;
}
@@ -30,16 +40,11 @@ public class Field implements Serializable {
this.type = type;
}
- public void setMultiValued(boolean multiValued) {
- this.multiValued = multiValued;
- }
-
@Override
public String toString() {
return "Field{" +
"name='" + name + '\'' +
", type='" + type + '\'' +
- ", multiValued=" + multiValued +
'}';
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/schema/FieldType.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/FieldType.java b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/FieldType.java
index 23bce92..e5d60bf 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/FieldType.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/FieldType.java
@@ -1,12 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.schema;
import com.google.gson.annotations.SerializedName;
import java.io.Serializable;
-/**
- * Created by hlouro on 7/27/15.
- */
public class FieldType implements Serializable {
private String name;
@SerializedName("class")
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Schema.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Schema.java b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Schema.java
index 91e1e09..d8071dd 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Schema.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Schema.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.schema;
import java.io.Serializable;
@@ -8,8 +26,6 @@ import java.util.List;
* Class representing the SolrSchema as returned by the REST call to the URL of the form
* http://localhost:8983/solr/gettingstarted/schema. This particular URL returns the schema in JSON format for the
* <a href="http://lucene.apache.org/solr/quickstart.html">Solr quickstart</a> example running locally.
- * <p></p>
- * Created by hlouro on 7/27/15.
*/
public class Schema implements Serializable {
private String name;
@@ -88,7 +104,8 @@ public class Schema implements Serializable {
", copyFields=" + copyFields +
'}';
}
-
+
+ // Wrapper class handy for the client code to use the JSON parser to build to use with JSON parser
public static class SchemaWrapper implements Serializable {
Schema schema;
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/schema/SolrFieldTypeFinder.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/SolrFieldTypeFinder.java b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/SolrFieldTypeFinder.java
index 2cebcf5..f2337e6 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/SolrFieldTypeFinder.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/SolrFieldTypeFinder.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.schema;
import org.slf4j.Logger;
@@ -14,15 +32,12 @@ import java.util.TreeSet;
/**
* Class containing all the information relating fields with their types. This information is wrapped in the class
* {@link FieldTypeWrapper}
- * <p></p>
- * Created by hlouro on 7/27/15.
*/
public class SolrFieldTypeFinder implements Serializable {
- private static final Logger logger = LoggerFactory.getLogger(SolrFieldTypeFinder.class);
+ private static final Logger log = LoggerFactory.getLogger(SolrFieldTypeFinder.class);
private Schema schema;
private Map<String, FieldTypeWrapper> fieldToWrapper;
-
/**
* Class wrapping all the information for fields and types
* */
@@ -72,9 +87,8 @@ public class SolrFieldTypeFinder implements Serializable {
buildMapForFields(fieldTypes, schema.getFields());
// dynamic fields
buildMapForFields(fieldTypes, schema.getDynamicFields());
- System.out.println("Completed building Field/Type Map: " + fieldToWrapper);
- if (logger.isDebugEnabled()) {
- logger.debug("Completed building Field/Type Map: " + fieldToWrapper);
+ if (log.isDebugEnabled()) {
+ log.debug("Completed building FieldType Map: " + fieldToWrapper);
}
}
@@ -115,7 +129,7 @@ public class SolrFieldTypeFinder implements Serializable {
}
}
}
- logger.debug("Solr Field and Type info: {}, {}", fieldName, typeWrapper);
+ log.debug("Solr Field Name = {}, Solr Type = {}", fieldName, typeWrapper);
return typeWrapper;
}
@@ -125,7 +139,7 @@ public class SolrFieldTypeFinder implements Serializable {
for (FieldTypeWrapper typeWrapper : typeWrappers) {
fieldTypeClasses.add(typeWrapper.getType().getClazz());
}
- logger.debug("Field type classes present in schema: {}", fieldTypeClasses);
+ log.debug("Field type classes present in schema: {}", fieldTypeClasses);
return fieldTypeClasses;
}
@@ -139,8 +153,8 @@ public class SolrFieldTypeFinder implements Serializable {
return true;
}
}
- if (logger.isDebugEnabled()) {
- logger.debug("Field [{}] did NOT match any dynamic field present in {}", fieldName, fieldToWrapper.keySet());
+ if (log.isDebugEnabled()) {
+ log.debug("Field [{}] did NOT match any dynamic field present in {}", fieldName, fieldToWrapper.keySet());
}
return false;
}
@@ -151,15 +165,18 @@ public class SolrFieldTypeFinder implements Serializable {
}
if (pattern.startsWith("*")) {
if (fieldName.endsWith(pattern.substring(1))) {
- logger.debug("Field [{}] MATCHES dynamic field {}", fieldName, pattern);
+ log.debug("Field [{}] MATCHES dynamic field {}", fieldName, pattern);
return true;
}
} else if (pattern.endsWith("*")) {
if (fieldName.startsWith(pattern.substring(0, pattern.length()-1))) {
- logger.debug("Field [{}] MATCHES dynamic field {}", fieldName, pattern);
+ log.debug("Field [{}] MATCHES dynamic field {}", fieldName, pattern);
return true;
}
}
+ if (log.isDebugEnabled()) {
+ log.debug("Field [{}] did NOT match any dynamic field present in {}", fieldName, fieldToWrapper.keySet());
+ }
return false;
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilder.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilder.java b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilder.java
index 52bc16d..662c768 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilder.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilder.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.schema.builder;
import com.google.gson.Gson;
@@ -10,11 +28,9 @@ import java.net.URL;
import java.util.Scanner;
/**
- * Class that buils the {@link Schema} object from the JSON representation of the schema as returned by the
+ * Class that builds the {@link Schema} object from the JSON representation of the schema as returned by the
* URL of the form http://localhost:8983/solr/gettingstarted/schema/ . This particular URL returns the schema
* in JSON format for the gettingstarted example running locally.
- * <p></p>
- * Created by hlouro on 7/28/15.
*/
public class RestJsonSchemaBuilder implements SchemaBuilder {
private static final Logger logger = LoggerFactory.getLogger(RestJsonSchemaBuilder.class);
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/SchemaBuilder.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/SchemaBuilder.java b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/SchemaBuilder.java
index a3456e0..a1a7f54 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/SchemaBuilder.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/SchemaBuilder.java
@@ -1,12 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.schema.builder;
import org.apache.storm.solr.schema.Schema;
import java.io.Serializable;
-/**
- * Created by hlouro on 7/28/15.
- */
public interface SchemaBuilder extends Serializable {
Schema getSchema();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrState.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrState.java b/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrState.java
index 3ec8b70..8187d11 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrState.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrState.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.solr.trident;
import backtype.storm.topology.FailedException;
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrStateFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrStateFactory.java b/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrStateFactory.java
index d3bbdc4..a1f815d 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrStateFactory.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrStateFactory.java
@@ -15,10 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.solr.trident;
import backtype.storm.task.IMetricsContext;
-import org.apache.storm.solr.config.SolrCommitStrategy;
import org.apache.storm.solr.config.SolrConfig;
import org.apache.storm.solr.mapper.SolrMapper;
import storm.trident.state.State;
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrUpdater.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrUpdater.java b/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrUpdater.java
index 3736cb3..db7b995 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrUpdater.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrUpdater.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.solr.trident;
import storm.trident.operation.TridentCollector;
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/test/java/org/apache/storm/solr/spout/SolrFieldsSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/test/java/org/apache/storm/solr/spout/SolrFieldsSpout.java b/external/storm-solr/src/test/java/org/apache/storm/solr/spout/SolrFieldsSpout.java
index 0d85f75..7d2357c 100644
--- a/external/storm-solr/src/test/java/org/apache/storm/solr/spout/SolrFieldsSpout.java
+++ b/external/storm-solr/src/test/java/org/apache/storm/solr/spout/SolrFieldsSpout.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.spout;
import backtype.storm.spout.SpoutOutputCollector;
@@ -13,20 +31,18 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
-/**
- * Created by hlouro on 7/24/15.
- */
public class SolrFieldsSpout extends BaseRichSpout {
private SpoutOutputCollector collector;
public static final List<Values> listValues = Lists.newArrayList(
getValues("1"), getValues("2"), getValues("3"));
private static Values getValues(String suf) {
- String suffix = "_hmcl_fields_test_val_" + suf;
+ String suffix = "_fields_test_val_" + suf;
return new Values(
"id" + suffix,
TestUtil.getDate(),
"dc_title" + suffix,
+ "Hugo%Miguel%Louro" + suffix, // Multivalue field split by non default token %
"dynamic_field" + suffix + "_txt", // to match dynamic fields of the form "*_txt"
"non_matching_field" + suffix); // this field won't be indexed by solr
}
@@ -50,7 +66,7 @@ public class SolrFieldsSpout extends BaseRichSpout {
}
public Fields getOutputFields() {
- return new Fields("id","date","dc_title","dynamic_field_txt","non_matching_field");
+ return new Fields("id","date","dc_title","author","dynamic_field_txt","non_matching_field");
}
@Override
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/test/java/org/apache/storm/solr/spout/SolrJsonSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/test/java/org/apache/storm/solr/spout/SolrJsonSpout.java b/external/storm-solr/src/test/java/org/apache/storm/solr/spout/SolrJsonSpout.java
index 7352614..6afed2c 100644
--- a/external/storm-solr/src/test/java/org/apache/storm/solr/spout/SolrJsonSpout.java
+++ b/external/storm-solr/src/test/java/org/apache/storm/solr/spout/SolrJsonSpout.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.spout;
import backtype.storm.spout.SpoutOutputCollector;
@@ -8,6 +26,7 @@ import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
import com.google.common.collect.Lists;
import com.google.gson.Gson;
+import org.apache.storm.solr.util.TestUtil;
import org.junit.Test;
import java.text.DateFormat;
@@ -17,17 +36,11 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
-/**
- * Created by hlouro on 7/24/15.
- */
public class SolrJsonSpout extends BaseRichSpout {
private SpoutOutputCollector collector;
private static final List<Values> listValues = Lists.newArrayList(
- new Values((new JsonSchema("_hmcl_json_test_1")).toJson()),
- new Values((new JsonSchema("_hmcl_json_test_2")).toJson()),
- new Values((new JsonSchema("_hmcl_json_test_3")).toJson()),
- new Values(new JsonSchema("_hmcl_json_test_4")),
- new Values(new JsonSchema("_hmcl_json_test_5")));
+ getJsonValues("1"), getJsonValues("2"), getJsonValues("3"), // Tuple contains String Object in JSON format
+ getPojoValues("1"), getPojoValues("2")); // Tuple contains Java object that must be serialized to JSON by SolrJsonMapper
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
@@ -56,6 +69,18 @@ public class SolrJsonSpout extends BaseRichSpout {
super.close();
}
+ // ====
+
+ private static Values getJsonValues(String suf) {
+ String suffix = "_json_test_val_" + suf;
+ return new Values((new JsonSchema(suffix)).toJson());
+ }
+
+ private static Values getPojoValues(String suf) {
+ String suffix = "_json_test_val_" + suf;
+ return new Values(new JsonSchema(suffix));
+ }
+
public static class JsonSchema {
private String id;
private String date;
@@ -65,7 +90,7 @@ public class SolrJsonSpout extends BaseRichSpout {
public JsonSchema(String suffix) {
this.id = "id" + suffix;
- this.date = getDate();
+ this.date = TestUtil.getDate();
this.dc_title = "dc_title" + suffix;
}
@@ -88,25 +113,8 @@ public class SolrJsonSpout extends BaseRichSpout {
return json;
}
- //TODO: clean this up
public static JsonSchema fromJson(String jsonStr) {
return new JsonSchema(gson.fromJson(jsonStr, JsonSchema.class));
}
-
- private String getDate() {
- DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
- String date = df.format(new Date());
- System.out.println(date);
- return date;
- }
}
-
- //TODO Delete
- @Test
- public void test() {
- SolrJsonSpout solrJsonSpout = new SolrJsonSpout();
- System.out.println("stop");
- }
-
-
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/test/java/org/apache/storm/solr/topology/.gitignore
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/test/java/org/apache/storm/solr/topology/.gitignore b/external/storm-solr/src/test/java/org/apache/storm/solr/topology/.gitignore
deleted file mode 100644
index 1adc831..0000000
--- a/external/storm-solr/src/test/java/org/apache/storm/solr/topology/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-# Created by .ignore support plugin (hsz.mobi)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/test/java/org/apache/storm/solr/topology/SolrFieldsTopology.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/test/java/org/apache/storm/solr/topology/SolrFieldsTopology.java b/external/storm-solr/src/test/java/org/apache/storm/solr/topology/SolrFieldsTopology.java
index 27d57db..809f434 100644
--- a/external/storm-solr/src/test/java/org/apache/storm/solr/topology/SolrFieldsTopology.java
+++ b/external/storm-solr/src/test/java/org/apache/storm/solr/topology/SolrFieldsTopology.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.topology;
import backtype.storm.generated.StormTopology;
@@ -12,18 +30,16 @@ import org.apache.storm.solr.spout.SolrFieldsSpout;
import java.io.IOException;
-/**
- * Created by hlouro on 7/31/15.
- */
public class SolrFieldsTopology extends SolrTopology {
public static void main(String[] args) throws Exception {
- SolrFieldsTopology solrJsonTopology = new SolrFieldsTopology();
- solrJsonTopology.run(args);
+ SolrFieldsTopology solrFieldsTopology = new SolrFieldsTopology();
+ solrFieldsTopology.run(args);
}
protected SolrMapper getSolrMapper() throws IOException {
return new SolrFieldsMapper.Builder(
- new RestJsonSchemaBuilder("localhost", "8983", COLLECTION)).setCollection(COLLECTION).build();
+ new RestJsonSchemaBuilder("localhost", "8983", COLLECTION), COLLECTION)
+ .setMultiValueFieldToken("%").build();
}
protected SolrCommitStrategy getSolrCommitStgy() {
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/test/java/org/apache/storm/solr/topology/SolrJsonTopology.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/test/java/org/apache/storm/solr/topology/SolrJsonTopology.java b/external/storm-solr/src/test/java/org/apache/storm/solr/topology/SolrJsonTopology.java
index bcb3f55..02e6d6f 100644
--- a/external/storm-solr/src/test/java/org/apache/storm/solr/topology/SolrJsonTopology.java
+++ b/external/storm-solr/src/test/java/org/apache/storm/solr/topology/SolrJsonTopology.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.topology;
import backtype.storm.generated.StormTopology;
@@ -9,9 +27,6 @@ import org.apache.storm.solr.spout.SolrJsonSpout;
import java.io.IOException;
-/**
- * Created by hlouro on 7/31/15.
- */
public class SolrJsonTopology extends SolrTopology {
public static void main(String[] args) throws Exception {
SolrJsonTopology solrJsonTopology = new SolrJsonTopology();
@@ -20,7 +35,7 @@ public class SolrJsonTopology extends SolrTopology {
protected SolrMapper getSolrMapper() throws IOException {
final String jsonTupleField = "JSON";
- return new SolrJsonMapper(COLLECTION, jsonTupleField);
+ return new SolrJsonMapper.Builder(COLLECTION, jsonTupleField).build();
}
protected StormTopology getTopology() throws IOException {
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/test/java/org/apache/storm/solr/topology/SolrTopology.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/test/java/org/apache/storm/solr/topology/SolrTopology.java b/external/storm-solr/src/test/java/org/apache/storm/solr/topology/SolrTopology.java
index 38aae80..607cf98 100644
--- a/external/storm-solr/src/test/java/org/apache/storm/solr/topology/SolrTopology.java
+++ b/external/storm-solr/src/test/java/org/apache/storm/solr/topology/SolrTopology.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.topology;
import backtype.storm.Config;
@@ -11,12 +29,8 @@ import org.apache.storm.solr.config.SolrConfig;
import java.io.IOException;
-/**
- * Created by hlouro on 7/24/15.
- */
public abstract class SolrTopology {
protected static String COLLECTION = "gettingstarted";
- protected static SolrClient solrClient = getSolrClient();
public void run(String[] args) throws Exception {
final StormTopology topology = getTopology();
@@ -39,6 +53,7 @@ public abstract class SolrTopology {
LocalCluster cluster = new LocalCluster();
cluster.submitTopology("test", config, topology);
Thread.sleep(10000);
+ System.out.println("Killing topology per client's request");
cluster.killTopology("test");
cluster.shutdown();
System.exit(0);
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/test/java/org/apache/storm/solr/trident/SolrFieldsTridentTopology.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/test/java/org/apache/storm/solr/trident/SolrFieldsTridentTopology.java b/external/storm-solr/src/test/java/org/apache/storm/solr/trident/SolrFieldsTridentTopology.java
index 86dabe3..4884c82 100644
--- a/external/storm-solr/src/test/java/org/apache/storm/solr/trident/SolrFieldsTridentTopology.java
+++ b/external/storm-solr/src/test/java/org/apache/storm/solr/trident/SolrFieldsTridentTopology.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.trident;
import backtype.storm.generated.StormTopology;
@@ -10,21 +28,18 @@ import storm.trident.state.StateFactory;
import java.io.IOException;
-/**
- * Created by hlouro on 7/31/15.
- */
public class SolrFieldsTridentTopology extends SolrFieldsTopology {
public static void main(String[] args) throws Exception {
- SolrFieldsTridentTopology solrJsonTopology = new SolrFieldsTridentTopology();
- solrJsonTopology.run(args);
+ SolrFieldsTridentTopology solrFieldsTridentTopology = new SolrFieldsTridentTopology();
+ solrFieldsTridentTopology.run(args);
}
protected StormTopology getTopology() throws IOException {
- final TridentTopology topology = new TridentTopology();
+ final TridentTopology tridentTopology = new TridentTopology();
final SolrFieldsSpout spout = new SolrFieldsSpout();
- final Stream stream = topology.newStream("SolrFieldsSpout", spout);
+ final Stream stream = tridentTopology.newStream("SolrFieldsSpout", spout);
final StateFactory solrStateFactory = new SolrStateFactory(getSolrConfig(), getSolrMapper());
stream.partitionPersist(solrStateFactory, spout.getOutputFields(), new SolrUpdater(), new Fields());
- return topology.build();
+ return tridentTopology.build();
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8552f43c/external/storm-solr/src/test/java/org/apache/storm/solr/trident/SolrJsonTridentTopology.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/test/java/org/apache/storm/solr/trident/SolrJsonTridentTopology.java b/external/storm-solr/src/test/java/org/apache/storm/solr/trident/SolrJsonTridentTopology.java
index 8f28909..d03b1dd 100644
--- a/external/storm-solr/src/test/java/org/apache/storm/solr/trident/SolrJsonTridentTopology.java
+++ b/external/storm-solr/src/test/java/org/apache/storm/solr/trident/SolrJsonTridentTopology.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.solr.trident;
import backtype.storm.generated.StormTopology;
@@ -10,13 +28,10 @@ import storm.trident.state.StateFactory;
import java.io.IOException;
-/**
- * Created by hlouro on 7/31/15.
- */
public class SolrJsonTridentTopology extends SolrJsonTopology {
public static void main(String[] args) throws Exception {
- SolrJsonTridentTopology solrJsonTopology = new SolrJsonTridentTopology();
- solrJsonTopology.run(args);
+ SolrJsonTridentTopology solrJsonTridentTopology = new SolrJsonTridentTopology();
+ solrJsonTridentTopology.run(args);
}
protected StormTopology getTopology() throws IOException {