You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gora.apache.org by le...@apache.org on 2016/05/23 15:43:27 UTC
[2/4] gora git commit: GORA-477 Add support for Solr 5.x
GORA-477 Add support for Solr 5.x
Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/348e020e
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/348e020e
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/348e020e
Branch: refs/heads/master
Commit: 348e020e1df6c0b3a5b8c38b220746617320fb43
Parents: 0495551
Author: Lewis John McGibbney <le...@gmail.com>
Authored: Tue May 17 16:11:35 2016 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Tue May 17 16:11:35 2016 -0700
----------------------------------------------------------------------
gora-solr-5/pom.xml | 275 +++++++
.../org/apache/gora/solr/query/SolrQuery.java | 62 ++
.../org/apache/gora/solr/query/SolrResult.java | 103 +++
.../org/apache/gora/solr/store/SolrMapping.java | 55 ++
.../org/apache/gora/solr/store/SolrStore.java | 794 +++++++++++++++++++
gora-solr-5/src/test/conf/gora-solr-mapping.xml | 40 +
gora-solr-5/src/test/conf/gora.properties | 21 +
gora-solr-5/src/test/conf/log4j.properties | 35 +
.../solr/Employee/conf/lang/stopwords_en.txt | 54 ++
.../test/conf/solr/Employee/conf/protwords.txt | 21 +
.../src/test/conf/solr/Employee/conf/schema.xml | 45 ++
.../test/conf/solr/Employee/conf/solrconfig.xml | 107 +++
.../test/conf/solr/Employee/conf/stopwords.txt | 14 +
.../test/conf/solr/Employee/conf/synonyms.txt | 29 +
.../solr/WebPage/conf/lang/stopwords_en.txt | 54 ++
.../test/conf/solr/WebPage/conf/protwords.txt | 21 +
.../src/test/conf/solr/WebPage/conf/schema.xml | 46 ++
.../test/conf/solr/WebPage/conf/solrconfig.xml | 107 +++
.../test/conf/solr/WebPage/conf/stopwords.txt | 14 +
.../test/conf/solr/WebPage/conf/synonyms.txt | 29 +
.../test/conf/solr/collection1/conf/schema.xml | 32 +
.../conf/solr/collection1/conf/solrconfig.xml | 113 +++
.../test/conf/solr/collection1/core.properties | 1 +
gora-solr-5/src/test/conf/solr/solr.xml | 69 ++
gora-solr-5/src/test/conf/solr/zoo.cfg | 17 +
.../apache/gora/solr/GoraSolrTestDriver.java | 101 +++
.../apache/gora/solr/store/TestSolrStore.java | 56 ++
gora-solr/pom.xml | 10 +-
pom.xml | 114 ++-
29 files changed, 2376 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/pom.xml
----------------------------------------------------------------------
diff --git a/gora-solr-5/pom.xml b/gora-solr-5/pom.xml
new file mode 100644
index 0000000..a98a304
--- /dev/null
+++ b/gora-solr-5/pom.xml
@@ -0,0 +1,275 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.gora</groupId>
+ <artifactId>gora</artifactId>
+ <version>0.7-SNAPSHOT</version>
+ <relativePath>../</relativePath>
+ </parent>
+ <artifactId>gora-solr-5</artifactId>
+ <packaging>bundle</packaging>
+
+ <name>Apache Gora :: Solr5</name>
+
+ <properties>
+ <osgi.import>*</osgi.import>
+ <osgi.export>org.apache.gora.solr*;version="${project.version}";-noimport:=true</osgi.export>
+ <jetty.version>9.2.13.v20150730</jetty.version>
+ </properties>
+
+ <build>
+ <directory>target</directory>
+ <outputDirectory>target/classes</outputDirectory>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ <testOutputDirectory>target/test-classes</testOutputDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testResources>
+ <testResource>
+ <directory>${project.basedir}/src/test/conf/</directory>
+ <includes>
+ <include>**</include>
+ </includes>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>${build-helper-maven-plugin.version}</version>
+ <executions>
+ <execution>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/examples/java</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <repositories>
+ <repository>
+ <id>maven-restlet</id>
+ <name>Public online Restlet repository</name>
+ <url>http://maven.restlet.org</url>
+ </repository>
+ </repositories>
+
+ <dependencies>
+ <!-- Gora Internal Dependencies -->
+ <dependency>
+ <groupId>org.apache.gora</groupId>
+ <artifactId>gora-core</artifactId>
+ <exclusions>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>org.mortbay.jetty</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>servlet-api-2.5</artifactId>
+ <groupId>org.mortbay.jetty</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jetty-util</artifactId>
+ <groupId>org.mortbay.jetty</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jetty</artifactId>
+ <groupId>org.mortbay.jetty</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.gora</groupId>
+ <artifactId>gora-core</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- Logging Dependencies -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.jms</groupId>
+ <artifactId>jms</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- Solr Dependencies -->
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-core</artifactId>
+ <version>${lucene-solr-5.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.eclipse.jetty.orbit</groupId>
+ <artifactId>javax.servlet</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <version>${solr-solrj-5.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-parsers</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.adobe.xmp</groupId>
+ <artifactId>xmpcore</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.geronimo.specs</groupId>
+ <artifactId>geronimo-stax-api_1.0_spec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.gagravarr</groupId>
+ <artifactId>vorbis-java-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>asm</groupId>
+ <artifactId>asm</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.aspectj</groupId>
+ <artifactId>aspectjrt</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.eclipse.jetty.orbit</groupId>
+ <artifactId>javax.servlet</artifactId>
+ </exclusion>
+ </exclusions>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-webapp</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+
+ <!-- ADDED TO AVOID PROBLEMS WITH JAVAX -->
+ <dependency>
+ <groupId>javax</groupId>
+ <artifactId>javaee-api</artifactId>
+ <version>7.0</version>
+ </dependency>
+
+ <!-- Testing Dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+<!-- <dependency> -->
+<!-- <groupId>org.apache.solr</groupId> -->
+<!-- <artifactId>solr-test-framework</artifactId> -->
+<!-- <version>${lucene-solr-5.version}</version> -->
+<!-- <exclusions> -->
+<!-- <exclusion> -->
+<!-- <groupId>org.eclipse.jetty</groupId> -->
+<!-- <artifactId>jetty-servlet</artifactId> -->
+<!-- </exclusion> -->
+<!-- </exclusions> -->
+<!-- <scope>test</scope> -->
+<!-- </dependency> -->
+<!-- <dependency> -->
+<!-- <groupId>org.apache.lucene</groupId> -->
+<!-- <artifactId>lucene-test-framework</artifactId> -->
+<!-- <version>${lucene-solr-5.version}</version> -->
+<!-- <scope>test</scope> -->
+<!-- </dependency> -->
+ </dependencies>
+
+ <url>http://gora.apache.org</url>
+ <description>The Apache Gora open source framework provides an in-memory data model and
+ persistence for big data. Gora supports persisting to column stores, key value stores,
+ document stores and RDBMSs, and analyzing the data with extensive Apache Hadoop MapReduce
+ support.</description>
+ <inceptionYear>2010</inceptionYear>
+ <organization>
+ <name>The Apache Software Foundation</name>
+ <url>http://apache.org</url>
+ </organization>
+ <issueManagement>
+ <system>JIRA</system>
+ <url>https://issues.apache.org/jira/browse/GORA</url>
+ </issueManagement>
+ <ciManagement>
+ <system>Jenkins</system>
+ <url>https://builds.apache.org/job/Gora-trunk/</url>
+ </ciManagement>
+</project>
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrQuery.java
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrQuery.java b/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrQuery.java
new file mode 100644
index 0000000..b19e384
--- /dev/null
+++ b/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrQuery.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.solr.query;
+
+import org.apache.gora.persistency.impl.PersistentBase;
+import org.apache.gora.query.impl.QueryBase;
+import org.apache.gora.solr.store.SolrMapping;
+import org.apache.gora.solr.store.SolrStore;
+import org.apache.gora.store.DataStore;
+
+public class SolrQuery<K, T extends PersistentBase> extends QueryBase<K, T> {
+ SolrStore<K, T> store;
+
+ public SolrQuery() {
+ super(null);
+ store = null;
+ }
+
+ public SolrQuery(DataStore<K, T> dataStore) {
+ super(dataStore);
+ store = (SolrStore<K, T>)dataStore;
+ }
+
+ public String toSolrQuery() {
+ SolrMapping mapping = store.getMapping();
+ String fld = mapping.getPrimaryKey();
+ String q;
+ if (getKey() != null) {
+ q = fld + ":" + SolrStore.escapeQueryKey(getKey().toString());
+ } else {
+ q = fld + ":[";
+ if (getStartKey() != null) {
+ q += SolrStore.escapeQueryKey(getStartKey().toString());
+ } else {
+ q += "*";
+ }
+ q += " TO ";
+ if (getEndKey() != null) {
+ q += SolrStore.escapeQueryKey(getEndKey().toString());
+ } else {
+ q += "*";
+ }
+ q += "]";
+ }
+ return q;
+ }
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrResult.java
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrResult.java b/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrResult.java
new file mode 100644
index 0000000..7006e2d
--- /dev/null
+++ b/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrResult.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.solr.query;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+
+import org.apache.gora.persistency.impl.PersistentBase;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.impl.PartitionQueryImpl;
+import org.apache.gora.query.impl.ResultBase;
+import org.apache.gora.solr.store.SolrStore;
+import org.apache.gora.store.DataStore;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+
+public class SolrResult<K, T extends PersistentBase> extends ResultBase<K, T> {
+
+ SolrDocumentList list = null;
+ SolrStore<K, T> store;
+ String[] fields;
+ int pos = 0;
+
+ public SolrResult(DataStore<K, T> dataStore, Query<K, T> query,
+ SolrClient server, int resultsSize) throws IOException {
+ super(dataStore, query);
+ store = (SolrStore<K, T>)dataStore;
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ if (query instanceof PartitionQueryImpl) {
+ query = ((PartitionQueryImpl<K, T>)query).getBaseQuery();
+ }
+ String q = ((SolrQuery<K, T>)query).toSolrQuery();
+ params.set(CommonParams.Q, q);
+ fields = query.getFields();
+ if (fields == null) {
+ params.set(CommonParams.FL, "*");
+ } else {
+ HashSet<String> uniqFields = new HashSet<>(Arrays.asList(fields));
+ String keyFld = ((SolrStore<K, T>)dataStore).getMapping().getPrimaryKey();
+ uniqFields.add(keyFld); // return also primary key
+ StringBuilder sb = new StringBuilder();
+ for (String f : uniqFields) {
+ if (sb.length() > 0) sb.append(',');
+ sb.append(f);
+ }
+ params.set(CommonParams.FL, sb.toString());
+ }
+ params.set(CommonParams.ROWS, resultsSize);
+ try {
+ QueryResponse rsp = server.query(params);
+ list = rsp.getResults();
+ } catch (SolrServerException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ protected boolean nextInner() throws IOException {
+ if (list == null || pos >= list.size()) {
+ return false;
+ }
+ SolrDocument doc = list.get(pos++);
+ key = (K) doc.get(store.getMapping().getPrimaryKey());
+ persistent = store.newInstance(doc, fields);
+ return true;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (list != null) list.clear();
+ }
+
+ @Override
+ public float getProgress() throws IOException {
+ if (list != null && list.size() > 0) {
+ return (float)pos / (float)list.size();
+ } else {
+ return 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrMapping.java
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrMapping.java b/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrMapping.java
new file mode 100644
index 0000000..424ae71
--- /dev/null
+++ b/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrMapping.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.solr.store;
+
+import java.util.HashMap;
+
+public class SolrMapping {
+ HashMap<String,String> mapping;
+ String coreName;
+ String primaryKey;
+
+ public SolrMapping() {
+ mapping = new HashMap<>();
+ }
+
+ public void addField(String goraField, String solrField) {
+ mapping.put(goraField, solrField);
+ }
+
+ public void setPrimaryKey(String solrKey) {
+ primaryKey = solrKey;
+ }
+
+ public void setCoreName(String coreName) {
+ this.coreName = coreName;
+ }
+
+ public String getCoreName() {
+ return coreName;
+ }
+
+ public String getPrimaryKey() {
+ return primaryKey;
+ }
+
+ public String getSolrField(String goraField) {
+ return mapping.get(goraField);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java b/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java
new file mode 100644
index 0000000..8c8d2f7
--- /dev/null
+++ b/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java
@@ -0,0 +1,794 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.gora.solr.store;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.avro.Schema;
+import org.apache.avro.Schema.Field;
+import org.apache.avro.Schema.Type;
+import org.apache.avro.specific.SpecificDatumReader;
+import org.apache.avro.specific.SpecificDatumWriter;
+import org.apache.avro.util.Utf8;
+import org.apache.gora.persistency.Persistent;
+import org.apache.gora.persistency.impl.PersistentBase;
+import org.apache.gora.query.PartitionQuery;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.Result;
+import org.apache.gora.query.impl.PartitionQueryImpl;
+import org.apache.gora.solr.query.SolrQuery;
+import org.apache.gora.solr.query.SolrResult;
+import org.apache.gora.store.DataStoreFactory;
+import org.apache.gora.store.impl.DataStoreBase;
+import org.apache.gora.util.AvroUtils;
+import org.apache.gora.util.IOUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
+import org.apache.solr.client.solrj.request.CoreAdminRequest;
+import org.apache.solr.client.solrj.response.CoreAdminResponse;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.jdom.Document;
+import org.jdom.Element;
+import org.jdom.input.SAXBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SolrStore<K, T extends PersistentBase> extends DataStoreBase<K, T> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SolrStore.class);
+
+ /** The default file name value to be used for obtaining the Solr object field mapping's */
+ protected static final String DEFAULT_MAPPING_FILE = "gora-solr-mapping.xml";
+
+ /** The URL of the Solr server - defined in <code>gora.properties</code> */
+ protected static final String SOLR_URL_PROPERTY = "solr.url";
+
+ /** The <code>solrconfig.xml</code> file to be used - defined in <code>gora.properties</code>*/
+ protected static final String SOLR_CONFIG_PROPERTY = "solr.config";
+
+ /** The <code>schema.xml</code> file to be used - defined in <code>gora.properties</code>*/
+ protected static final String SOLR_SCHEMA_PROPERTY = "solr.schema";
+
+ /** A batch size unit (ArrayList) of SolrDocument's to be used for writing to Solr.
+ * Should be defined in <code>gora.properties</code>.
+ * A default value of 100 is used if this value is absent. This value must be of type <b>Integer</b>.
+ */
+ protected static final String SOLR_BATCH_SIZE_PROPERTY = "solr.batch_size";
+
+ /** The solrj implementation to use. This has a default value of <i>http</i> for HttpSolrClient.
+ * Available options include <b>http</b>, <b>cloud</b>, <b>concurrent</b> and <b>loadbalance</b>.
+ * Defined in <code>gora.properties</code>
+ * This value must be of type <b>String</b>.
+ */
+ protected static final String SOLR_SOLRJSERVER_IMPL = "solr.solrjserver";
+
+ /** Whether to use secured Solr client or not.
+ * Available options include <b>true</b>, and <b>false</b>.
+ * Defined in <code>gora.properties</code>
+ * This value must be of type <b>boolean</b>.
+ */
+ protected static final String SOLR_SERVER_USER_AUTH = "solr.solrjserver.user_auth";
+
+ /** Solr client username.
+ * Solr client user authentication should be enabled for this property.
+ * Defined in <code>gora.properties</code>
+ * This value must be of type <b>String</b>.
+ */
+ protected static final String SOLR_SERVER_USERNAME = "solr.solrjserver.username";
+
+ /** Solr client password.
+ * Solr client user authentication should be enabled for this property.
+ * Defined in <code>gora.properties</code>
+ * This value must be of type <b>String</b>.
+ */
+ protected static final String SOLR_SERVER_PASSWORD = "solr.solrjserver.password";
+
+ /** A batch commit unit for SolrDocument's used when making (commit) calls to Solr.
+ * Should be defined in <code>gora.properties</code>.
+ * A default value of 1000 is used if this value is absent. This value must be of type <b>Integer</b>.
+ */
+ protected static final String SOLR_COMMIT_WITHIN_PROPERTY = "solr.commit_within";
+
+ /** The maximum number of result to return when we make a call to
+ * {@link org.apache.gora.solr.store.SolrStore#execute(Query)}. This should be
+ * defined in <code>gora.properties</code>. This value must be of type <b>Integer</b>.
+ */
+ protected static final String SOLR_RESULTS_SIZE_PROPERTY = "solr.results_size";
+
+ /** The default batch size (ArrayList) of SolrDocuments to be used in the event of an absent
+ * value for <code>solr.batchSize</code>.
+ * Set to 100 by default.
+ */
+ protected static final int DEFAULT_BATCH_SIZE = 100;
+
+ /** The default commit size of SolrDocuments to be used in the event of an absent
+ * value for <code>solr.commitSize</code>.
+ * Set to 1000 by default.
+ */
+ protected static final int DEFAULT_COMMIT_WITHIN = 1000;
+
+ /** The default results size of SolrDocuments to be used in the event of an absent
+ * value for <code>solr.resultsSize</code>.
+ * Set to 100 by default.
+ */
+ protected static final int DEFAULT_RESULTS_SIZE = 100;
+
+ private SolrMapping mapping;
+
+ private String SolrClientUrl, solrConfig, solrSchema, solrJServerImpl;
+
+ private SolrClient server, adminServer;
+
+ private boolean serverUserAuth;
+
+ private String serverUsername;
+
+ private String serverPassword;
+
+ private ArrayList<SolrInputDocument> batch;
+
+ private int batchSize = DEFAULT_BATCH_SIZE;
+
+ private int commitWithin = DEFAULT_COMMIT_WITHIN;
+
+ private int resultsSize = DEFAULT_RESULTS_SIZE;
+
+ /**
+ * Default schema index with value "0" used when AVRO Union data types are
+ * stored
+ */
+ public static final int DEFAULT_UNION_SCHEMA = 0;
+
+ /*
+ * Create a threadlocal map for the datum readers and writers, because they
+ * are not thread safe, at least not before Avro 1.4.0 (See AVRO-650). When
+ * they are thread safe, it is possible to maintain a single reader and writer
+ * pair for every schema, instead of one for every thread.
+ */
+
+ public static final ConcurrentHashMap<Schema, SpecificDatumReader<?>> readerMap = new ConcurrentHashMap<>();
+
+ public static final ConcurrentHashMap<Schema, SpecificDatumWriter<?>> writerMap = new ConcurrentHashMap<>();
+
+ @Override
+ public void initialize(Class<K> keyClass, Class<T> persistentClass,
+ Properties properties) {
+ super.initialize(keyClass, persistentClass, properties);
+ try {
+ String mappingFile = DataStoreFactory.getMappingFile(properties, this,
+ DEFAULT_MAPPING_FILE);
+ mapping = readMapping(mappingFile);
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ }
+
+ SolrClientUrl = DataStoreFactory.findProperty(properties, this,
+ SOLR_URL_PROPERTY, null);
+ solrConfig = DataStoreFactory.findProperty(properties, this,
+ SOLR_CONFIG_PROPERTY, null);
+ solrSchema = DataStoreFactory.findProperty(properties, this,
+ SOLR_SCHEMA_PROPERTY, null);
+ solrJServerImpl = DataStoreFactory.findProperty(properties, this,
+ SOLR_SOLRJSERVER_IMPL, "http");
+ serverUserAuth = DataStoreFactory.findBooleanProperty(properties, this,
+ SOLR_SERVER_USER_AUTH, "false");
+ if (serverUserAuth) {
+ serverUsername = DataStoreFactory.findProperty(properties, this,
+ SOLR_SERVER_USERNAME, null);
+ serverPassword = DataStoreFactory.findProperty(properties, this,
+ SOLR_SERVER_PASSWORD, null);
+ }
+ LOG.info("Using Solr server at " + SolrClientUrl);
+ String solrJServerType = ((solrJServerImpl == null || solrJServerImpl.equals(""))?"http":solrJServerImpl);
+ // HttpSolrClient - denoted by "http" in properties
+ if (solrJServerType.toLowerCase(Locale.getDefault()).equals("http")) {
+ LOG.info("Using HttpSolrClient Solrj implementation.");
+ this.adminServer = new HttpSolrClient(SolrClientUrl);
+ this.server = new HttpSolrClient( SolrClientUrl + "/" + mapping.getCoreName() );
+ if (serverUserAuth) {
+ HttpClientUtil.setBasicAuth(
+ (DefaultHttpClient) ((HttpSolrClient) adminServer).getHttpClient(),
+ serverUsername, serverPassword);
+ HttpClientUtil.setBasicAuth(
+ (DefaultHttpClient) ((HttpSolrClient) server).getHttpClient(),
+ serverUsername, serverPassword);
+ }
+ // CloudSolrClient - denoted by "cloud" in properties
+ } else if (solrJServerType.toLowerCase(Locale.getDefault()).equals("cloud")) {
+ LOG.info("Using CloudSolrClient Solrj implementation.");
+ this.adminServer = new CloudSolrClient(SolrClientUrl);
+ this.server = new CloudSolrClient( SolrClientUrl + "/" + mapping.getCoreName() );
+ if (serverUserAuth) {
+ HttpClientUtil.setBasicAuth(
+ (DefaultHttpClient) ((CloudSolrClient) adminServer).getLbClient().getHttpClient(),
+ serverUsername, serverPassword);
+ HttpClientUtil.setBasicAuth(
+ (DefaultHttpClient) ((CloudSolrClient) server).getLbClient().getHttpClient(),
+ serverUsername, serverPassword);
+ }
+ } else if (solrJServerType.toLowerCase(Locale.getDefault()).equals("concurrent")) {
+ LOG.info("Using ConcurrentUpdateSolrClient Solrj implementation.");
+ this.adminServer = new ConcurrentUpdateSolrClient(SolrClientUrl, 1000, 10);
+ this.server = new ConcurrentUpdateSolrClient( SolrClientUrl + "/" + mapping.getCoreName(), 1000, 10);
+ // LBHttpSolrClient - denoted by "loadbalance" in properties
+ } else if (solrJServerType.toLowerCase(Locale.getDefault()).equals("loadbalance")) {
+ LOG.info("Using LBHttpSolrClient Solrj implementation.");
+ String[] solrUrlElements = StringUtils.split(SolrClientUrl);
+ try {
+ this.adminServer = new LBHttpSolrClient(solrUrlElements);
+ } catch (MalformedURLException e) {
+ LOG.error(e.getMessage());
+ throw new RuntimeException(e);
+ }
+ try {
+ this.server = new LBHttpSolrClient( solrUrlElements + "/" + mapping.getCoreName() );
+ } catch (MalformedURLException e) {
+ LOG.error(e.getMessage());
+ throw new RuntimeException(e);
+ }
+ if (serverUserAuth) {
+ HttpClientUtil.setBasicAuth(
+ (DefaultHttpClient) ((LBHttpSolrClient) adminServer).getHttpClient(),
+ serverUsername, serverPassword);
+ HttpClientUtil.setBasicAuth(
+ (DefaultHttpClient) ((LBHttpSolrClient) server).getHttpClient(),
+ serverUsername, serverPassword);
+ }
+ }
+ if (autoCreateSchema) {
+ createSchema();
+ }
+ String batchSizeString = DataStoreFactory.findProperty(properties, this,
+ SOLR_BATCH_SIZE_PROPERTY, null);
+ if (batchSizeString != null) {
+ try {
+ batchSize = Integer.parseInt(batchSizeString);
+ } catch (NumberFormatException nfe) {
+ LOG.warn("Invalid batch size '{}', using default {}", batchSizeString, DEFAULT_BATCH_SIZE);
+ }
+ }
+ batch = new ArrayList<>(batchSize);
+ String commitWithinString = DataStoreFactory.findProperty(properties, this,
+ SOLR_COMMIT_WITHIN_PROPERTY, null);
+ if (commitWithinString != null) {
+ try {
+ commitWithin = Integer.parseInt(commitWithinString);
+ } catch (NumberFormatException nfe) {
+ LOG.warn("Invalid commit within '{}' , using default {}", commitWithinString, DEFAULT_COMMIT_WITHIN);
+ }
+ }
+ String resultsSizeString = DataStoreFactory.findProperty(properties, this,
+ SOLR_RESULTS_SIZE_PROPERTY, null);
+ if (resultsSizeString != null) {
+ try {
+ resultsSize = Integer.parseInt(resultsSizeString);
+ } catch (NumberFormatException nfe) {
+ LOG.warn("Invalid results size '{}' , using default {}", resultsSizeString, DEFAULT_RESULTS_SIZE);
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private SolrMapping readMapping(String filename) throws IOException {
+ SolrMapping map = new SolrMapping();
+ try {
+ SAXBuilder builder = new SAXBuilder();
+ Document doc = builder.build(getClass().getClassLoader()
+ .getResourceAsStream(filename));
+
+ List<Element> classes = doc.getRootElement().getChildren("class");
+
+ for (Element classElement : classes) {
+ if (classElement.getAttributeValue("keyClass").equals(
+ keyClass.getCanonicalName())
+ && classElement.getAttributeValue("name").equals(
+ persistentClass.getCanonicalName())) {
+
+ String tableName = getSchemaName(
+ classElement.getAttributeValue("table"), persistentClass);
+ map.setCoreName(tableName);
+
+ Element primaryKeyEl = classElement.getChild("primarykey");
+ map.setPrimaryKey(primaryKeyEl.getAttributeValue("column"));
+
+ List<Element> fields = classElement.getChildren("field");
+
+ for (Element field : fields) {
+ String fieldName = field.getAttributeValue("name");
+ String columnName = field.getAttributeValue("column");
+ map.addField(fieldName, columnName);
+ }
+ break;
+ }
+ LOG.warn("Check that 'keyClass' and 'name' parameters in gora-solr-mapping.xml "
+ + "match with intended values. A mapping mismatch has been found therefore "
+ + "no mapping has been initialized for class mapping at position "
+ + " {} in mapping file.", classes.indexOf(classElement));
+ }
+ } catch (Exception ex) {
+ throw new IOException(ex);
+ }
+
+ return map;
+ }
+
+ public SolrMapping getMapping() {
+ return mapping;
+ }
+
+ @Override
+ public String getSchemaName() {
+ return mapping.getCoreName();
+ }
+
+ @Override
+ public void createSchema() {
+ try {
+ if (!schemaExists())
+ CoreAdminRequest.createCore(mapping.getCoreName(),
+ mapping.getCoreName(), adminServer, solrConfig, solrSchema);
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ /** Default implementation deletes and recreates the schema*/
+ public void truncateSchema() {
+ try {
+ server.deleteByQuery("*:*");
+ server.commit();
+ } catch (Exception e) {
+ // ignore?
+ LOG.error(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public void deleteSchema() {
+ // XXX should this be only in truncateSchema ???
+ try {
+ server.deleteByQuery("*:*");
+ server.commit();
+ } catch (Exception e) {
+ // ignore?
+ // LOG.error(e.getMessage(), e);
+ }
+ try {
+ CoreAdminRequest.unloadCore(mapping.getCoreName(), adminServer);
+ } catch (Exception e) {
+ if (e.getMessage().contains("No such core")) {
+ return; // it's ok, the core is not there
+ } else {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+ }
+
+ @Override
+ public boolean schemaExists() {
+ boolean exists = false;
+ try {
+ CoreAdminResponse rsp = CoreAdminRequest.getStatus(mapping.getCoreName(),
+ adminServer);
+ exists = rsp.getUptime(mapping.getCoreName()) != null;
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ }
+ return exists;
+ }
+
+ private static final String toDelimitedString(String[] arr, String sep) {
+ if (arr == null || arr.length == 0) {
+ return "";
+ }
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < arr.length; i++) {
+ if (i > 0)
+ sb.append(sep);
+ sb.append(arr[i]);
+ }
+ return sb.toString();
+ }
+
+ public static String escapeQueryKey(String key) {
+ if (key == null) {
+ return null;
+ }
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < key.length(); i++) {
+ char c = key.charAt(i);
+ switch (c) {
+ case ':':
+ case '*':
+ sb.append("\\").append(c);
+ break;
+ default:
+ sb.append(c);
+ }
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public T get(K key, String[] fields) {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(CommonParams.QT, "/get");
+ params.set(CommonParams.FL, toDelimitedString(fields, ","));
+ params.set("id", key.toString());
+ try {
+ QueryResponse rsp = server.query(params);
+ Object o = rsp.getResponse().get("doc");
+ if (o == null) {
+ return null;
+ }
+ return newInstance((SolrDocument) o, fields);
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ }
+ return null;
+ }
+
+ public T newInstance(SolrDocument doc, String[] fields) throws IOException {
+ T persistent = newPersistent();
+ if (fields == null) {
+ fields = fieldMap.keySet().toArray(new String[fieldMap.size()]);
+ }
+ String pk = mapping.getPrimaryKey();
+ for (String f : fields) {
+ Field field = fieldMap.get(f);
+ Schema fieldSchema = field.schema();
+ String sf = null;
+ if (pk.equals(f)) {
+ sf = f;
+ } else {
+ sf = mapping.getSolrField(f);
+ }
+ Object sv = doc.get(sf);
+ if (sv == null) {
+ continue;
+ }
+
+ Object v = deserializeFieldValue(field, fieldSchema, sv, persistent);
+ persistent.put(field.pos(), v);
+ persistent.setDirty(field.pos());
+
+ }
+ persistent.clearDirty();
+ return persistent;
+ }
+
+ @SuppressWarnings("rawtypes")
+ private SpecificDatumReader getDatumReader(Schema fieldSchema) {
+ SpecificDatumReader<?> reader = readerMap.get(fieldSchema);
+ if (reader == null) {
+ reader = new SpecificDatumReader(fieldSchema);// ignore dirty bits
+ SpecificDatumReader localReader = null;
+ if ((localReader = readerMap.putIfAbsent(fieldSchema, reader)) != null) {
+ reader = localReader;
+ }
+ }
+ return reader;
+ }
+
+ @SuppressWarnings("rawtypes")
+ private SpecificDatumWriter getDatumWriter(Schema fieldSchema) {
+ SpecificDatumWriter writer = writerMap.get(fieldSchema);
+ if (writer == null) {
+ writer = new SpecificDatumWriter(fieldSchema);// ignore dirty bits
+ writerMap.put(fieldSchema, writer);
+ }
+
+ return writer;
+ }
+
+ @SuppressWarnings("unchecked")
+ private Object deserializeFieldValue(Field field, Schema fieldSchema,
+ Object solrValue, T persistent) throws IOException {
+ Object fieldValue = null;
+ switch (fieldSchema.getType()) {
+ case MAP:
+ case ARRAY:
+ case RECORD:
+ @SuppressWarnings("rawtypes")
+ SpecificDatumReader reader = getDatumReader(fieldSchema);
+ fieldValue = IOUtils.deserialize((byte[]) solrValue, reader,
+ persistent.get(field.pos()));
+ break;
+ case ENUM:
+ fieldValue = AvroUtils.getEnumValue(fieldSchema, (String) solrValue);
+ break;
+ case FIXED:
+ throw new IOException("???");
+ // break;
+ case BYTES:
+ fieldValue = ByteBuffer.wrap((byte[]) solrValue);
+ break;
+ case STRING:
+ fieldValue = new Utf8(solrValue.toString());
+ break;
+ case UNION:
+ if (fieldSchema.getTypes().size() == 2 && isNullable(fieldSchema)) {
+ // schema [type0, type1]
+ Type type0 = fieldSchema.getTypes().get(0).getType();
+ Type type1 = fieldSchema.getTypes().get(1).getType();
+
+ // Check if types are different and there's a "null", like
+ // ["null","type"] or ["type","null"]
+ if (!type0.equals(type1)) {
+ if (type0.equals(Schema.Type.NULL))
+ fieldSchema = fieldSchema.getTypes().get(1);
+ else
+ fieldSchema = fieldSchema.getTypes().get(0);
+ } else {
+ fieldSchema = fieldSchema.getTypes().get(0);
+ }
+ fieldValue = deserializeFieldValue(field, fieldSchema, solrValue,
+ persistent);
+ } else {
+ @SuppressWarnings("rawtypes")
+ SpecificDatumReader unionReader = getDatumReader(fieldSchema);
+ fieldValue = IOUtils.deserialize((byte[]) solrValue, unionReader,
+ persistent.get(field.pos()));
+ break;
+ }
+ break;
+ default:
+ fieldValue = solrValue;
+ }
+ return fieldValue;
+ }
+
+ @Override
+ public void put(K key, T persistent) {
+ Schema schema = persistent.getSchema();
+ if (!persistent.isDirty()) {
+ // nothing to do
+ return;
+ }
+ SolrInputDocument doc = new SolrInputDocument();
+ // add primary key
+ doc.addField(mapping.getPrimaryKey(), key);
+ // populate the doc
+ List<Field> fields = schema.getFields();
+ for (Field field : fields) {
+ String sf = mapping.getSolrField(field.name());
+ // Solr will append values to fields in a SolrInputDocument, even the key
+ // mapping won't find the primary
+ if (sf == null) {
+ continue;
+ }
+ Schema fieldSchema = field.schema();
+ Object v = persistent.get(field.pos());
+ if (v == null) {
+ continue;
+ }
+ v = serializeFieldValue(fieldSchema, v);
+ doc.addField(sf, v);
+
+ }
+ LOG.info("Putting DOCUMENT: " + doc);
+ batch.add(doc);
+ if (batch.size() >= batchSize) {
+ try {
+ add(batch, commitWithin);
+ batch.clear();
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private Object serializeFieldValue(Schema fieldSchema, Object fieldValue) {
+ switch (fieldSchema.getType()) {
+ case MAP:
+ case ARRAY:
+ case RECORD:
+ byte[] data = null;
+ try {
+ @SuppressWarnings("rawtypes")
+ SpecificDatumWriter writer = getDatumWriter(fieldSchema);
+ data = IOUtils.serialize(writer, fieldValue);
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ }
+ fieldValue = data;
+ break;
+ case BYTES:
+ fieldValue = ((ByteBuffer) fieldValue).array();
+ break;
+ case ENUM:
+ case STRING:
+ fieldValue = fieldValue.toString();
+ break;
+ case UNION:
+ // If field's schema is null and one type, we do undertake serialization.
+ // All other types are serialized.
+ if (fieldSchema.getTypes().size() == 2 && isNullable(fieldSchema)) {
+ int schemaPos = getUnionSchema(fieldValue, fieldSchema);
+ Schema unionSchema = fieldSchema.getTypes().get(schemaPos);
+ fieldValue = serializeFieldValue(unionSchema, fieldValue);
+ } else {
+ byte[] serilazeData = null;
+ try {
+ @SuppressWarnings("rawtypes")
+ SpecificDatumWriter writer = getDatumWriter(fieldSchema);
+ serilazeData = IOUtils.serialize(writer, fieldValue);
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ }
+ fieldValue = serilazeData;
+ }
+ break;
+ default:
+ // LOG.error("Unknown field type: " + fieldSchema.getType());
+ break;
+ }
+ return fieldValue;
+ }
+
+ private boolean isNullable(Schema unionSchema) {
+ for (Schema innerSchema : unionSchema.getTypes()) {
+ if (innerSchema.getType().equals(Schema.Type.NULL)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Given an object and the object schema this function obtains, from within
+ * the UNION schema, the position of the type used. If no data type can be
+ * inferred then we return a default value of position 0.
+ *
+ * @param pValue
+ * @param pUnionSchema
+ * @return the unionSchemaPosition.
+ */
+ private int getUnionSchema(Object pValue, Schema pUnionSchema) {
+ int unionSchemaPos = 0;
+ for (Schema currentSchema : pUnionSchema.getTypes()) {
+ Type schemaType = currentSchema.getType();
+ if (pValue instanceof Utf8 && schemaType.equals(Type.STRING))
+ return unionSchemaPos;
+ else if (pValue instanceof ByteBuffer && schemaType.equals(Type.BYTES))
+ return unionSchemaPos;
+ else if (pValue instanceof Integer && schemaType.equals(Type.INT))
+ return unionSchemaPos;
+ else if (pValue instanceof Long && schemaType.equals(Type.LONG))
+ return unionSchemaPos;
+ else if (pValue instanceof Double && schemaType.equals(Type.DOUBLE))
+ return unionSchemaPos;
+ else if (pValue instanceof Float && schemaType.equals(Type.FLOAT))
+ return unionSchemaPos;
+ else if (pValue instanceof Boolean && schemaType.equals(Type.BOOLEAN))
+ return unionSchemaPos;
+ else if (pValue instanceof Map && schemaType.equals(Type.MAP))
+ return unionSchemaPos;
+ else if (pValue instanceof List && schemaType.equals(Type.ARRAY))
+ return unionSchemaPos;
+ else if (pValue instanceof Persistent && schemaType.equals(Type.RECORD))
+ return unionSchemaPos;
+ unionSchemaPos++;
+ }
+ // if we weren't able to determine which data type it is, then we return the
+ // default
+ return DEFAULT_UNION_SCHEMA;
+ }
+
+ @Override
+ public boolean delete(K key) {
+ String keyField = mapping.getPrimaryKey();
+ try {
+ UpdateResponse rsp = server.deleteByQuery(keyField + ":"
+ + escapeQueryKey(key.toString()));
+ server.commit();
+ LOG.info(rsp.toString());
+ return true;
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ }
+ return false;
+ }
+
+ @Override
+ public long deleteByQuery(Query<K, T> query) {
+ String q = ((SolrQuery<K, T>) query).toSolrQuery();
+ try {
+ UpdateResponse rsp = server.deleteByQuery(q);
+ server.commit();
+ LOG.info(rsp.toString());
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ }
+ return 0;
+ }
+
+ @Override
+ public Result<K, T> execute(Query<K, T> query) {
+ try {
+ return new SolrResult<>(this, query, server, resultsSize);
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ }
+ return null;
+ }
+
+ @Override
+ public Query<K, T> newQuery() {
+ return new SolrQuery<>(this);
+ }
+
+ @Override
+ public List<PartitionQuery<K, T>> getPartitions(Query<K, T> query)
+ throws IOException {
+ // TODO: implement this using Hadoop DB support
+
+ ArrayList<PartitionQuery<K, T>> partitions = new ArrayList<>();
+ PartitionQueryImpl<K, T> pqi = new PartitionQueryImpl<>(query);
+ pqi.setConf(getConf());
+ partitions.add(pqi);
+
+ return partitions;
+ }
+
+ @Override
+ public void flush() {
+ try {
+ if (batch.size() > 0) {
+ add(batch, commitWithin);
+ batch.clear();
+ }
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public void close() {
+ flush();
+ }
+
+ private void add(ArrayList<SolrInputDocument> batch, int commitWithin)
+ throws SolrServerException, IOException {
+ if (commitWithin == 0) {
+ server.add(batch);
+ server.commit(false, true, true);
+ } else {
+ server.add(batch, commitWithin);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/gora-solr-mapping.xml
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/test/conf/gora-solr-mapping.xml b/gora-solr-5/src/test/conf/gora-solr-mapping.xml
new file mode 100644
index 0000000..78ca39b
--- /dev/null
+++ b/gora-solr-5/src/test/conf/gora-solr-mapping.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<gora-otd>
+ <class name="org.apache.gora.examples.generated.Employee" keyClass="java.lang.String" table="Employee">
+ <primarykey column="ssn"/>
+ <field name="name" column="name"/>
+ <field name="dateOfBirth" column="dateOfBirth"/>
+ <field name="salary" column="salary"/>
+ <field name="boss" column="boss"/>
+ <field name="webpage" column="webpage"/>
+ </class>
+
+ <class name="org.apache.gora.examples.generated.WebPage" keyClass="java.lang.String" table="WebPage">
+ <primarykey column="url"/>
+ <field name="content" column="content"/>
+ <field name="parsedContent" column="parsedContent"/>
+ <field name="outlinks" column="outlinks"/>
+ <field name="headers" column="headers"/>
+ <field name="metadata" column="metadata"/>
+ <field name="byteData" column="byteData"/>
+ <field name="stringData" column="stringData"/>
+ </class>
+</gora-otd>
+
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/gora.properties
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/test/conf/gora.properties b/gora-solr-5/src/test/conf/gora.properties
new file mode 100644
index 0000000..27403e2
--- /dev/null
+++ b/gora-solr-5/src/test/conf/gora.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+gora.solrstore.solr.url=http://localhost:9876/solr
+gora.datastore.solr.commit_within=0
+gora.solrstore.solr.solrjserver=http
+gora.solrstore.solr.solrjserver.user_auth=false
+#gora.solrstore.solr.solrjserver.username=solr-user
+#gora.solrstore.solr.solrjserver.password=solr-password
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/test/conf/log4j.properties b/gora-solr-5/src/test/conf/log4j.properties
new file mode 100644
index 0000000..3094e6b
--- /dev/null
+++ b/gora-solr-5/src/test/conf/log4j.properties
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Set root logger level to error
+log4j.rootLogger=DEBUG, Console
+
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.hadoop=WARN
+
+###### Console appender definition #######
+
+# All outputs currently set to be a ConsoleAppender.
+log4j.appender.Console=org.apache.log4j.ConsoleAppender
+log4j.appender.Console.layout=org.apache.log4j.PatternLayout
+
+log4j.appender.Console.layout.ConversionPattern=%d{ISO8601} %x %-5p [%c{3}] [%t] %m%n
+
+###### File appender definition #######
+log4j.appender.File=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.File.Append=true
+log4j.appender.File.DatePattern='.'yyyy-MM-dd
+log4j.appender.File.layout=org.apache.log4j.PatternLayout
+log4j.appender.File.layout.ConversionPattern=%d{ABSOLUTE} %-5p [%c] %m%n
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/Employee/conf/lang/stopwords_en.txt
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/test/conf/solr/Employee/conf/lang/stopwords_en.txt b/gora-solr-5/src/test/conf/solr/Employee/conf/lang/stopwords_en.txt
new file mode 100644
index 0000000..224230c
--- /dev/null
+++ b/gora-solr-5/src/test/conf/solr/Employee/conf/lang/stopwords_en.txt
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# a couple of test stopwords to test that the words are really being
+# configured from this file:
+stopworda
+stopwordb
+
+# Standard english stop words taken from Lucene's StopAnalyzer
+a
+an
+and
+are
+as
+at
+be
+but
+by
+for
+if
+in
+into
+is
+it
+no
+not
+of
+on
+or
+such
+that
+the
+their
+then
+there
+these
+they
+this
+to
+was
+will
+with
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/Employee/conf/protwords.txt
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/test/conf/solr/Employee/conf/protwords.txt b/gora-solr-5/src/test/conf/solr/Employee/conf/protwords.txt
new file mode 100644
index 0000000..5a32e50
--- /dev/null
+++ b/gora-solr-5/src/test/conf/solr/Employee/conf/protwords.txt
@@ -0,0 +1,21 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#-----------------------------------------------------------------------
+# Use a protected word file to protect against the stemmer reducing two
+# unrelated words to the same base word.
+
+# Some non-words that normally won't be encountered,
+# just to test that they won't be stemmed.
+dontstems
+zwhacky
+
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/Employee/conf/schema.xml
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/test/conf/solr/Employee/conf/schema.xml b/gora-solr-5/src/test/conf/solr/Employee/conf/schema.xml
new file mode 100644
index 0000000..3cb2e24
--- /dev/null
+++ b/gora-solr-5/src/test/conf/solr/Employee/conf/schema.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<schema name="test example Employee" version="1.5">
+
+ <fields>
+
+ <!-- Common Fields -->
+ <field name="_version_" type="long" indexed="true" stored="true"/>
+
+ <!-- Employee Fields -->
+ <field name="ssn" type="string" indexed="true" stored="true" required="true" multiValued="false" />
+ <field name="name" type="string" indexed="true" stored="true" />
+ <field name="dateOfBirth" type="long" stored="true" />
+ <field name="salary" type="int" stored="true" />
+ <field name="boss" type="binary" stored="true" />
+ <field name="webpage" type="binary" stored="true" />
+
+ </fields>
+
+ <uniqueKey>ssn</uniqueKey>
+
+ <types>
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true" />
+ <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldtype name="binary" class="solr.BinaryField"/>
+ </types>
+
+</schema>
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/Employee/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/test/conf/solr/Employee/conf/solrconfig.xml b/gora-solr-5/src/test/conf/solr/Employee/conf/solrconfig.xml
new file mode 100644
index 0000000..81ec4ba
--- /dev/null
+++ b/gora-solr-5/src/test/conf/solr/Employee/conf/solrconfig.xml
@@ -0,0 +1,107 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ For more details about configurations options that may appear in
+ this file, see http://wiki.apache.org/solr/SolrConfigXml.
+-->
+<config>
+ <luceneMatchVersion>4.8</luceneMatchVersion>
+ <dataDir>${solr.data.dir:}</dataDir>
+ <directoryFactory name="DirectoryFactory"
+ class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
+ <codecFactory class="solr.SchemaCodecFactory"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+ <indexConfig>
+ <lockType>${solr.lock.type:native}</lockType>
+ </indexConfig>
+
+ <jmx />
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+ <updateLog>
+ <str name="dir">${solr.ulog.dir:}</str>
+ </updateLog>
+ </updateHandler>
+
+ <query>
+ <maxBooleanClauses>1024</maxBooleanClauses>
+ <filterCache class="solr.FastLRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+ <queryResultCache class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+ <documentCache class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+ <enableLazyFieldLoading>true</enableLazyFieldLoading>
+ <queryResultWindowSize>20</queryResultWindowSize>
+ <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
+ <listener event="newSearcher" class="solr.QuerySenderListener">
+ <arr name="queries">
+ </arr>
+ </listener>
+ <listener event="firstSearcher" class="solr.QuerySenderListener">
+ <arr name="queries">
+ <lst>
+ <str name="q">static firstSearcher warming in solrconfig.xml</str>
+ </lst>
+ </arr>
+ </listener>
+ <useColdSearcher>false</useColdSearcher>
+ <maxWarmingSearchers>2</maxWarmingSearchers>
+ </query>
+
+ <requestDispatcher handleSelect="false" >
+ <requestParsers enableRemoteStreaming="true"
+ multipartUploadLimitInKB="2048000"
+ formdataUploadLimitInKB="2048"
+ addHttpRequestToContext="false"/>
+ <httpCaching never304="true" />
+ </requestDispatcher>
+
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <int name="rows">10</int>
+ <str name="df">ssn</str>
+ </lst>
+ </requestHandler>
+
+ <requestHandler name="/query" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="wt">json</str>
+ <str name="indent">true</str>
+ <str name="df">ssn</str>
+ </lst>
+ </requestHandler>
+
+ <requestHandler name="/get" class="solr.RealTimeGetHandler">
+ <lst name="defaults">
+ <str name="omitHeader">true</str>
+ </lst>
+ </requestHandler>
+
+ <requestHandler name="/update" class="solr.UpdateRequestHandler">
+ </requestHandler>
+</config>
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/Employee/conf/stopwords.txt
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/test/conf/solr/Employee/conf/stopwords.txt b/gora-solr-5/src/test/conf/solr/Employee/conf/stopwords.txt
new file mode 100644
index 0000000..25b47f6
--- /dev/null
+++ b/gora-solr-5/src/test/conf/solr/Employee/conf/stopwords.txt
@@ -0,0 +1,14 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/Employee/conf/synonyms.txt
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/test/conf/solr/Employee/conf/synonyms.txt b/gora-solr-5/src/test/conf/solr/Employee/conf/synonyms.txt
new file mode 100644
index 0000000..f00294b
--- /dev/null
+++ b/gora-solr-5/src/test/conf/solr/Employee/conf/synonyms.txt
@@ -0,0 +1,29 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#-----------------------------------------------------------------------
+#some test synonym mappings unlikely to appear in real input text
+aaafoo => aaabar
+bbbfoo => bbbfoo bbbbar
+cccfoo => cccbar cccbaz
+fooaaa,baraaa,bazaaa
+
+# Some synonym groups specific to this example
+GB,gib,gigabyte,gigabytes
+MB,mib,megabyte,megabytes
+Television, Televisions, TV, TVs
+#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming
+#after us won't split it into two words.
+
+# Synonym mappings can be used for spelling correction too
+pixima => pixma
+
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/WebPage/conf/lang/stopwords_en.txt
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/test/conf/solr/WebPage/conf/lang/stopwords_en.txt b/gora-solr-5/src/test/conf/solr/WebPage/conf/lang/stopwords_en.txt
new file mode 100644
index 0000000..224230c
--- /dev/null
+++ b/gora-solr-5/src/test/conf/solr/WebPage/conf/lang/stopwords_en.txt
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# a couple of test stopwords to test that the words are really being
+# configured from this file:
+stopworda
+stopwordb
+
+# Standard english stop words taken from Lucene's StopAnalyzer
+a
+an
+and
+are
+as
+at
+be
+but
+by
+for
+if
+in
+into
+is
+it
+no
+not
+of
+on
+or
+such
+that
+the
+their
+then
+there
+these
+they
+this
+to
+was
+will
+with
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/WebPage/conf/protwords.txt
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/test/conf/solr/WebPage/conf/protwords.txt b/gora-solr-5/src/test/conf/solr/WebPage/conf/protwords.txt
new file mode 100644
index 0000000..5a32e50
--- /dev/null
+++ b/gora-solr-5/src/test/conf/solr/WebPage/conf/protwords.txt
@@ -0,0 +1,21 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#-----------------------------------------------------------------------
+# Use a protected word file to protect against the stemmer reducing two
+# unrelated words to the same base word.
+
+# Some non-words that normally won't be encountered,
+# just to test that they won't be stemmed.
+dontstems
+zwhacky
+
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/WebPage/conf/schema.xml
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/test/conf/solr/WebPage/conf/schema.xml b/gora-solr-5/src/test/conf/solr/WebPage/conf/schema.xml
new file mode 100644
index 0000000..55765a3
--- /dev/null
+++ b/gora-solr-5/src/test/conf/solr/WebPage/conf/schema.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<schema name="test example WebPage" version="1.5">
+
+ <fields>
+
+ <!-- Common Fields -->
+ <field name="_version_" type="long" indexed="true" stored="true"/>
+
+ <!-- WebPage Fields -->
+ <field name="url" type="string" indexed="true" stored="true" required="true" multiValued="false" />
+ <field name="parsedContent" type="binary" stored="true" />
+ <field name="content" type="binary" stored="true" />
+ <field name="outlinks" type="binary" stored="true" />
+ <field name="headers" type="binary" stored="true" />
+ <field name="metadata" type="binary" stored="true" />
+ <field name="byteData" type="binary" stored="true" />
+ <field name="stringData" type="binary" stored="true" />
+
+ </fields>
+
+ <uniqueKey>url</uniqueKey>
+
+ <types>
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true" />
+ <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldtype name="binary" class="solr.BinaryField"/>
+ </types>
+
+</schema>
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/WebPage/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/test/conf/solr/WebPage/conf/solrconfig.xml b/gora-solr-5/src/test/conf/solr/WebPage/conf/solrconfig.xml
new file mode 100644
index 0000000..a88f393
--- /dev/null
+++ b/gora-solr-5/src/test/conf/solr/WebPage/conf/solrconfig.xml
@@ -0,0 +1,107 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ For more details about configurations options that may appear in
+ this file, see http://wiki.apache.org/solr/SolrConfigXml.
+-->
+<config>
+ <luceneMatchVersion>4.8</luceneMatchVersion>
+ <dataDir>${solr.data.dir:}</dataDir>
+ <directoryFactory name="DirectoryFactory"
+ class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
+ <codecFactory class="solr.SchemaCodecFactory"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+ <indexConfig>
+ <lockType>${solr.lock.type:native}</lockType>
+ </indexConfig>
+
+ <jmx />
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+ <updateLog>
+ <str name="dir">${solr.ulog.dir:}</str>
+ </updateLog>
+ </updateHandler>
+
+ <query>
+ <maxBooleanClauses>1024</maxBooleanClauses>
+ <filterCache class="solr.FastLRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+ <queryResultCache class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+ <documentCache class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+ <enableLazyFieldLoading>true</enableLazyFieldLoading>
+ <queryResultWindowSize>20</queryResultWindowSize>
+ <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
+ <listener event="newSearcher" class="solr.QuerySenderListener">
+ <arr name="queries">
+ </arr>
+ </listener>
+ <listener event="firstSearcher" class="solr.QuerySenderListener">
+ <arr name="queries">
+ <lst>
+ <str name="q">static firstSearcher warming in solrconfig.xml</str>
+ </lst>
+ </arr>
+ </listener>
+ <useColdSearcher>false</useColdSearcher>
+ <maxWarmingSearchers>2</maxWarmingSearchers>
+ </query>
+
+ <requestDispatcher handleSelect="false" >
+ <requestParsers enableRemoteStreaming="true"
+ multipartUploadLimitInKB="2048000"
+ formdataUploadLimitInKB="2048"
+ addHttpRequestToContext="false"/>
+ <httpCaching never304="true" />
+ </requestDispatcher>
+
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <int name="rows">10</int>
+ <str name="df">url</str>
+ </lst>
+ </requestHandler>
+
+ <requestHandler name="/query" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="wt">json</str>
+ <str name="indent">true</str>
+ <str name="df">url</str>
+ </lst>
+ </requestHandler>
+
+ <requestHandler name="/get" class="solr.RealTimeGetHandler">
+ <lst name="defaults">
+ <str name="omitHeader">true</str>
+ </lst>
+ </requestHandler>
+
+ <requestHandler name="/update" class="solr.UpdateRequestHandler">
+ </requestHandler>
+</config>
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/WebPage/conf/stopwords.txt
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/test/conf/solr/WebPage/conf/stopwords.txt b/gora-solr-5/src/test/conf/solr/WebPage/conf/stopwords.txt
new file mode 100644
index 0000000..25b47f6
--- /dev/null
+++ b/gora-solr-5/src/test/conf/solr/WebPage/conf/stopwords.txt
@@ -0,0 +1,14 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/WebPage/conf/synonyms.txt
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/test/conf/solr/WebPage/conf/synonyms.txt b/gora-solr-5/src/test/conf/solr/WebPage/conf/synonyms.txt
new file mode 100644
index 0000000..f00294b
--- /dev/null
+++ b/gora-solr-5/src/test/conf/solr/WebPage/conf/synonyms.txt
@@ -0,0 +1,29 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#-----------------------------------------------------------------------
+#some test synonym mappings unlikely to appear in real input text
+aaafoo => aaabar
+bbbfoo => bbbfoo bbbbar
+cccfoo => cccbar cccbaz
+fooaaa,baraaa,bazaaa
+
+# Some synonym groups specific to this example
+GB,gib,gigabyte,gigabytes
+MB,mib,megabyte,megabytes
+Television, Televisions, TV, TVs
+#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming
+#after us won't split it into two words.
+
+# Synonym mappings can be used for spelling correction too
+pixima => pixma
+
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/collection1/conf/schema.xml
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/test/conf/solr/collection1/conf/schema.xml b/gora-solr-5/src/test/conf/solr/collection1/conf/schema.xml
new file mode 100644
index 0000000..77465c4
--- /dev/null
+++ b/gora-solr-5/src/test/conf/solr/collection1/conf/schema.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<schema name="testexample" version="1.5">
+ <fields>
+ <!-- Common Fields -->
+ <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" />
+ <field name="_version_" type="long" indexed="true" stored="true"/>
+ </fields>
+
+ <uniqueKey>id</uniqueKey>
+
+ <types>
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true" />
+ <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
+ </types>
+</schema>
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/collection1/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/test/conf/solr/collection1/conf/solrconfig.xml b/gora-solr-5/src/test/conf/solr/collection1/conf/solrconfig.xml
new file mode 100644
index 0000000..5e5fc9a
--- /dev/null
+++ b/gora-solr-5/src/test/conf/solr/collection1/conf/solrconfig.xml
@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ For more details about configurations options that may appear in
+ this file, see http://wiki.apache.org/solr/SolrConfigXml.
+-->
+<config>
+ <luceneMatchVersion>4.8</luceneMatchVersion>
+ <dataDir>${solr.data.dir:}</dataDir>
+ <directoryFactory name="DirectoryFactory"
+ class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
+ <codecFactory class="solr.SchemaCodecFactory"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+ <indexConfig>
+ <lockType>${solr.lock.type:native}</lockType>
+ </indexConfig>
+
+ <jmx />
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+ <updateLog>
+ <str name="dir">${solr.ulog.dir:}</str>
+ </updateLog>
+ <autoCommit>
+ <maxTime>15000</maxTime>
+ <openSearcher>false</openSearcher>
+ </autoCommit>
+ </updateHandler>
+
+ <query>
+ <maxBooleanClauses>1024</maxBooleanClauses>
+ <filterCache class="solr.FastLRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+ <queryResultCache class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+ <documentCache class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+ <enableLazyFieldLoading>true</enableLazyFieldLoading>
+ <queryResultWindowSize>20</queryResultWindowSize>
+ <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
+ <listener event="newSearcher" class="solr.QuerySenderListener">
+ <arr name="queries">
+ </arr>
+ </listener>
+ <listener event="firstSearcher" class="solr.QuerySenderListener">
+ <arr name="queries">
+ <lst>
+ <str name="q">static firstSearcher warming in solrconfig.xml</str>
+ </lst>
+ </arr>
+ </listener>
+ <useColdSearcher>false</useColdSearcher>
+ <maxWarmingSearchers>2</maxWarmingSearchers>
+ </query>
+
+ <requestDispatcher handleSelect="false" >
+ <requestParsers enableRemoteStreaming="true"
+ multipartUploadLimitInKB="2048000"
+ formdataUploadLimitInKB="2048"
+ addHttpRequestToContext="false"/>
+ <httpCaching never304="true" />
+ </requestDispatcher>
+
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <int name="rows">10</int>
+ <str name="df">id</str>
+ </lst>
+ </requestHandler>
+
+ <requestHandler name="/query" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="wt">json</str>
+ <str name="indent">true</str>
+ <str name="df">id</str>
+ </lst>
+ </requestHandler>
+
+ <requestHandler name="/get" class="solr.RealTimeGetHandler">
+ <lst name="defaults">
+ <str name="omitHeader">true</str>
+ <str name="wt">json</str>
+ <str name="indent">true</str>
+ </lst>
+ </requestHandler>
+
+ <requestHandler name="/update" class="solr.UpdateRequestHandler">
+ </requestHandler>
+</config>
http://git-wip-us.apache.org/repos/asf/gora/blob/348e020e/gora-solr-5/src/test/conf/solr/collection1/core.properties
----------------------------------------------------------------------
diff --git a/gora-solr-5/src/test/conf/solr/collection1/core.properties b/gora-solr-5/src/test/conf/solr/collection1/core.properties
new file mode 100644
index 0000000..bc0cf7d
--- /dev/null
+++ b/gora-solr-5/src/test/conf/solr/collection1/core.properties
@@ -0,0 +1 @@
+name=collection1
\ No newline at end of file