You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ho...@apache.org on 2012/06/01 02:06:46 UTC
svn commit: r1344946 - in /lucene/dev/trunk/solr: ./
core/src/java/org/apache/solr/update/processor/
core/src/test-files/solr/conf/
core/src/test/org/apache/solr/update/processor/
Author: hossman
Date: Fri Jun 1 00:06:46 2012
New Revision: 1344946
URL: http://svn.apache.org/viewvc?rev=1344946&view=rev
Log:
SOLR-3495: new UpdateProcessors to add default values (constant, UUID, or Date) to documents w/o field values
Added:
lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/AbstractDefaultValueUpdateProcessorFactory.java (with props)
lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/DefaultValueUpdateProcessorFactory.java (with props)
lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/TimestampUpdateProcessorFactory.java (with props)
lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/UUIDUpdateProcessorFactory.java (with props)
lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/processor/DefaultValueUpdateProcessorTest.java (with props)
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-update-processor-chains.xml
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1344946&r1=1344945&r2=1344946&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Fri Jun 1 00:06:46 2012
@@ -320,6 +320,15 @@ New Features
prior "FieldName^boost" syntax is still accepted. In such cases the value on the
"ps" parameter serves as the default slop. (Ron Mayer via James Dyer)
+* SOLR-2796: New UpdateProcessors have been added to create default values for
+ configured fields. These works similarly to the <field default="..."/>
+ option in schema.xml, but are applied in the UpdateProcessorChain, so they
+ may be used prior to other UpdateProcessors, or to generate a uniqueKey field
+ value when using the DistributedUpdateProcessor (ie: SolrCloud)
+ TimestampUpdateProcessorFactory
+ UUIDUpdateProcessorFactory
+ DefaultValueUpdateProcessorFactory
+ (hossman)
Optimizations
----------------------
Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/AbstractDefaultValueUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/AbstractDefaultValueUpdateProcessorFactory.java?rev=1344946&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/AbstractDefaultValueUpdateProcessorFactory.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/AbstractDefaultValueUpdateProcessorFactory.java Fri Jun 1 00:06:46 2012
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update.processor;
+
+import java.io.IOException;
+
+import org.apache.solr.common.SolrException;
+import static org.apache.solr.common.SolrException.ErrorCode.*;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.AddUpdateCommand;
+
+/**
+ * <p>
+ * Base class that can be extended by any
+ * <code>UpdateRequestProcessorFactory</code> designed to add a default value
+ * to the document in an <code>AddUpdateCommand</code> when that field is not
+ * already specified.
+ * </p>
+ * <p>
+ * This base class handles initialization of the <code>fieldName</code> init
+ * param, and provides an {@link DefaultValueUpdateProcessor} that Factory
+ * subclasses may choose to return from their <code>getInstance</code>
+ * implementation.
+ * </p>
+ */
+public abstract class AbstractDefaultValueUpdateProcessorFactory
+ extends UpdateRequestProcessorFactory {
+
+ protected String fieldName = null;
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void init(NamedList args) {
+
+ Object obj = args.remove("fieldName");
+ if (null == obj && null == fieldName) {
+ throw new SolrException
+ (SERVER_ERROR, "'fieldName' init param must be specified and non-null");
+ } else {
+ fieldName = obj.toString();
+ }
+
+ if (0 < args.size()) {
+ throw new SolrException(SERVER_ERROR,
+ "Unexpected init param(s): '" +
+ args.getName(0) + "'");
+ }
+
+ super.init(args);
+ }
+
+ /**
+ * A simple processor that adds the results of {@link #getDefaultValue}
+ * to any document which does not already have a value in
+ * <code>fieldName</code>
+ */
+ protected static abstract class DefaultValueUpdateProcessor
+ extends UpdateRequestProcessor {
+
+ final String fieldName;
+
+ public DefaultValueUpdateProcessor(final String fieldName,
+ final UpdateRequestProcessor next) {
+ super(next);
+ this.fieldName = fieldName;
+ }
+
+ @Override
+ public void processAdd(AddUpdateCommand cmd) throws IOException {
+ final SolrInputDocument doc = cmd.getSolrInputDocument();
+
+ if (! doc.containsKey(fieldName)) {
+ doc.addField(fieldName, getDefaultValue());
+ }
+
+ super.processAdd(cmd);
+ }
+
+ public abstract Object getDefaultValue();
+ }
+}
+
+
+
Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/DefaultValueUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/DefaultValueUpdateProcessorFactory.java?rev=1344946&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/DefaultValueUpdateProcessorFactory.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/DefaultValueUpdateProcessorFactory.java Fri Jun 1 00:06:46 2012
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update.processor;
+
+import java.io.IOException;
+
+import org.apache.solr.common.SolrException;
+import static org.apache.solr.common.SolrException.ErrorCode.*;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.AddUpdateCommand;
+
+/**
+ * <p>
+ * An update processor that adds a constant default value to any document
+ * being added that does not already have a value in the specified field.
+ * </p>
+ *
+ * <p>
+ * In the example configuration below, if a document does not contain a value
+ * in the <code>price</code> and/or <code>type</code> fields, it will be given
+ * default values of <code>0.0</code> and/or <code>unknown</code>
+ * (respectively).
+ * <p>
+ *
+ * <pre class="prettyprint">
+ * <processor class="solr.DefaultValueUpdateProcessorFactory">
+ * <str name="fieldName">price</str>
+ * <float name="value">0.0</float>
+ * </processor>
+ * <processor class="solr.DefaultValueUpdateProcessorFactory">
+ * <str name="fieldName">type</str>
+ * <str name="value">unknown</str>
+ * </processor>
+ * </pre>
+ */
+public class DefaultValueUpdateProcessorFactory
+ extends AbstractDefaultValueUpdateProcessorFactory {
+
+ protected Object defaultValue = null;
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void init(NamedList args) {
+
+ Object obj = args.remove("value");
+ if (null == obj) {
+ throw new SolrException
+ (SERVER_ERROR, "'value' init param must be specified and non-null");
+ } else {
+ defaultValue = obj;
+ }
+
+ super.init(args);
+ }
+
+ public UpdateRequestProcessor getInstance(SolrQueryRequest req,
+ SolrQueryResponse rsp,
+ UpdateRequestProcessor next ) {
+ return new DefaultValueUpdateProcessor(fieldName, next) {
+ public Object getDefaultValue() { return defaultValue; }
+ };
+ }
+
+}
+
+
+
Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/TimestampUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/TimestampUpdateProcessorFactory.java?rev=1344946&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/TimestampUpdateProcessorFactory.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/TimestampUpdateProcessorFactory.java Fri Jun 1 00:06:46 2012
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update.processor;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.apache.solr.common.SolrException;
+import static org.apache.solr.common.SolrException.ErrorCode.*;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.AddUpdateCommand;
+
+import org.apache.solr.common.params.CommonParams; // javadoc
+
+/**
+ * <p>
+ * An update processor that adds a newly generated <code>Date</code> value
+ * of "NOW" to any document being added that does not already have a value
+ * in the specified field.
+ * </p>
+ *
+ * <p>
+ * In the example configuration below, if a document does not contain a value
+ * in the <code>timestamp</code> field, a new <code>Date</code> will be
+ * generated and added as the value of that field.
+ * <p>
+ *
+ * <pre class="prettyprint">
+ * <processor class="solr.TimestampUpdateProcessorFactory">
+ * <str name="fieldName">timestamp</str>
+ * </processor>
+ * </pre>
+ *
+ * @see Date
+ * @see CommonParams#NOW
+ */
+public class TimestampUpdateProcessorFactory
+ extends AbstractDefaultValueUpdateProcessorFactory {
+
+ public UpdateRequestProcessor getInstance(SolrQueryRequest req,
+ SolrQueryResponse rsp,
+ UpdateRequestProcessor next ) {
+ return new DefaultValueUpdateProcessor(fieldName, next) {
+ public Object getDefaultValue() {
+ return SolrRequestInfo.getRequestInfo().getNOW();
+ }
+ };
+ }
+}
+
+
+
Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/UUIDUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/UUIDUpdateProcessorFactory.java?rev=1344946&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/UUIDUpdateProcessorFactory.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/UUIDUpdateProcessorFactory.java Fri Jun 1 00:06:46 2012
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update.processor;
+
+import java.io.IOException;
+import java.util.UUID;
+import java.util.Locale;
+
+import org.apache.solr.common.SolrException;
+import static org.apache.solr.common.SolrException.ErrorCode.*;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.AddUpdateCommand;
+
+
+/**
+ * <p>
+ * An update processor that adds a newly generated <code>UUID</code> value
+ * to any document being added that does not already have a value in the
+ * specified field.
+ * </p>
+ *
+ * <p>
+ * In the example configuration below, if a document does not contain a value
+ * in the <code>id</code> field, a new <code>UUID</code> will be generated
+ * and added as the value of that field.
+ * <p>
+ *
+ * <pre class="prettyprint">
+ * <processor class="solr.UUIDUpdateProcessorFactory">
+ * <str name="fieldName">id</str>
+ * </processor>
+ * </pre>
+ *
+ * @see UUID
+ */
+public class UUIDUpdateProcessorFactory
+ extends AbstractDefaultValueUpdateProcessorFactory {
+
+ public UpdateRequestProcessor getInstance(SolrQueryRequest req,
+ SolrQueryResponse rsp,
+ UpdateRequestProcessor next ) {
+ return new DefaultValueUpdateProcessor(fieldName, next) {
+ public Object getDefaultValue() {
+ return UUID.randomUUID().toString().toLowerCase(Locale.ENGLISH);
+ }
+ };
+ }
+}
+
+
+
Modified: lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-update-processor-chains.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-update-processor-chains.xml?rev=1344946&r1=1344945&r2=1344946&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-update-processor-chains.xml (original)
+++ lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-update-processor-chains.xml Fri Jun 1 00:06:46 2012
@@ -233,4 +233,21 @@
</processor>
</updateRequestProcessorChain>
+ <updateRequestProcessorChain name="default-values">
+ <processor class="solr.DefaultValueUpdateProcessorFactory">
+ <str name="fieldName">processor_default_s</str>
+ <str name="value">X</str>
+ </processor>
+ <processor class="solr.DefaultValueUpdateProcessorFactory">
+ <str name="fieldName">processor_default_i</str>
+ <int name="value">42</int>
+ </processor>
+ <processor class="solr.UUIDUpdateProcessorFactory">
+ <str name="fieldName">uuid</str>
+ </processor>
+ <processor class="solr.TimestampUpdateProcessorFactory">
+ <str name="fieldName">timestamp</str>
+ </processor>
+ </updateRequestProcessorChain>
+
</config>
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/processor/DefaultValueUpdateProcessorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/processor/DefaultValueUpdateProcessorTest.java?rev=1344946&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/processor/DefaultValueUpdateProcessorTest.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/processor/DefaultValueUpdateProcessorTest.java Fri Jun 1 00:06:46 2012
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update.processor;
+
+import java.util.Date;
+import java.util.UUID;
+import java.util.Arrays;
+import java.io.IOException;
+
+import org.apache.solr.SolrTestCaseJ4;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.SolrInputField;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+
+import org.apache.solr.core.SolrCore;
+
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.response.SolrQueryResponse;
+
+import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+import org.apache.solr.update.processor.UpdateRequestProcessorChain;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class DefaultValueUpdateProcessorTest extends SolrTestCaseJ4 {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig-update-processor-chains.xml", "schema12.xml");
+ }
+
+ public void testDefaults() throws Exception {
+ SolrInputDocument d = null;
+
+ Date now = new Date();
+
+ // get all defaults
+ d = processAdd("default-values",
+ doc(f("id", "1111"),
+ f("name", "Existing", "Values")));
+
+ assertNotNull(d);
+
+ assertEquals("X", d.getFieldValue("processor_default_s"));
+ assertEquals(42, d.getFieldValue("processor_default_i"));
+ assertNotNull(d.getFieldValue("uuid"));
+ assertNotNull(UUID.fromString(d.getFieldValue("uuid").toString()));
+ assertNotNull(d.getFieldValue("timestamp"));
+ assertTrue("timestamp not a date: " +
+ d.getFieldValue("timestamp").getClass(),
+ d.getFieldValue("timestamp") instanceof Date);
+ assertEquals(Arrays.asList("Existing","Values"),
+ d.getFieldValues("name"));
+
+ // defaults already specified
+ d = processAdd("default-values",
+ doc(f("id", "1111"),
+ f("timestamp", now),
+ f("uuid", "550e8400-e29b-41d4-a716-446655440000"),
+ f("processor_default_s", "I HAVE A VALUE"),
+ f("processor_default_i", 12345),
+ f("name", "Existing", "Values")));
+
+ assertNotNull(d);
+
+ assertEquals("I HAVE A VALUE", d.getFieldValue("processor_default_s"));
+ assertEquals(12345, d.getFieldValue("processor_default_i"));
+ assertEquals("550e8400-e29b-41d4-a716-446655440000",
+ d.getFieldValue("uuid"));
+ assertEquals(now, d.getFieldValue("timestamp"));
+ assertEquals(Arrays.asList("Existing","Values"),
+ d.getFieldValues("name"));
+ }
+
+
+ /**
+ * Convenience method for building up SolrInputDocuments
+ */
+ SolrInputDocument doc(SolrInputField... fields) {
+ SolrInputDocument d = new SolrInputDocument();
+ for (SolrInputField f : fields) {
+ d.put(f.getName(), f);
+ }
+ return d;
+ }
+
+ /**
+ * Convenience method for building up SolrInputFields
+ */
+ SolrInputField field(String name, float boost, Object... values) {
+ SolrInputField f = new SolrInputField(name);
+ for (Object v : values) {
+ f.addValue(v, 1.0F);
+ }
+ f.setBoost(boost);
+ return f;
+ }
+
+ /**
+ * Convenience method for building up SolrInputFields with default boost
+ */
+ SolrInputField f(String name, Object... values) {
+ return field(name, 1.0F, values);
+ }
+
+
+ /**
+ * Runs a document through the specified chain, and returns the final
+ * document used when the chain is completed (NOTE: some chains may
+ * modify the document in place
+ */
+ SolrInputDocument processAdd(final String chain,
+ final SolrInputDocument docIn)
+ throws IOException {
+
+ SolrCore core = h.getCore();
+ UpdateRequestProcessorChain pc = core.getUpdateProcessingChain(chain);
+ assertNotNull("No Chain named: " + chain, pc);
+
+ SolrQueryResponse rsp = new SolrQueryResponse();
+
+ SolrQueryRequest req = new LocalSolrQueryRequest
+ (core, new ModifiableSolrParams());
+ try {
+ SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req,rsp));
+ AddUpdateCommand cmd = new AddUpdateCommand(req);
+ cmd.solrDoc = docIn;
+
+ UpdateRequestProcessor processor = pc.createProcessor(req, rsp);
+ processor.processAdd(cmd);
+
+ return cmd.solrDoc;
+ } finally {
+ SolrRequestInfo.clearRequestInfo();
+ req.close();
+ }
+ }
+}