You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2012/07/18 18:52:42 UTC
svn commit: r1363013 - in /lucene/dev/trunk/solr: ./
core/src/java/org/apache/solr/update/processor/
core/src/test/org/apache/solr/ solrj/src/java/org/apache/solr/common/
Author: markrmiller
Date: Wed Jul 18 16:52:42 2012
New Revision: 1363013
URL: http://svn.apache.org/viewvc?rev=1363013&view=rev
Log:
SOLR-3215: Clone SolrInputDocument when distrib indexing so that update processors after the distrib update process do not process the document twice.
Added:
lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestDocumentBuilder.java
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java
lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/SolrInputField.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1363013&r1=1363012&r2=1363013&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Wed Jul 18 16:52:42 2012
@@ -117,6 +117,9 @@ Other Changes
* SOLR-3600: Raise the default zkClientTimeout from 10 seconds to 15 seconds. (Mark Miller)
+* SOLR-3215: Clone SolrInputDocument when distrib indexing so that update processors after
+ the distrib update process do not process the document twice. (Mark Miller)
+
================== 4.0.0-ALPHA ==================
More information about this release, including any errata related to the
release notes, upgrade instructions, or other changes may be found online at:
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java?rev=1363013&r1=1363012&r2=1363013&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java Wed Jul 18 16:52:42 2012
@@ -285,7 +285,10 @@ public class DistributedUpdateProcessor
boolean dropCmd = false;
if (!forwardToLeader) {
- dropCmd = versionAdd(cmd);
+ // clone the original doc
+ SolrInputDocument clonedDoc = cmd.solrDoc.deepCopy();
+ dropCmd = versionAdd(cmd, clonedDoc);
+ cmd.solrDoc = clonedDoc;
}
if (dropCmd) {
@@ -393,10 +396,11 @@ public class DistributedUpdateProcessor
/**
* @param cmd
+ * @param cloneDoc needs the version if it's assigned
* @return whether or not to drop this cmd
* @throws IOException
*/
- private boolean versionAdd(AddUpdateCommand cmd) throws IOException {
+ private boolean versionAdd(AddUpdateCommand cmd, SolrInputDocument cloneDoc) throws IOException {
BytesRef idBytes = cmd.getIndexedId();
if (vinfo == null || idBytes == null) {
@@ -469,6 +473,7 @@ public class DistributedUpdateProcessor
long version = vinfo.getNewClock();
cmd.setVersion(version);
cmd.getSolrInputDocument().setField(VersionInfo.VERSION_FIELD, version);
+ cloneDoc.setField(VersionInfo.VERSION_FIELD, version);
bucket.updateHighest(version);
} else {
// The leader forwarded us this update.
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestDocumentBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestDocumentBuilder.java?rev=1363013&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestDocumentBuilder.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestDocumentBuilder.java Wed Jul 18 16:52:42 2012
@@ -0,0 +1,71 @@
+package org.apache.solr;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.solr.common.SolrInputDocument;
+import org.junit.Test;
+
+
+public class TestDocumentBuilder extends LuceneTestCase {
+
+ @Test
+ public void testDeepCopy() throws IOException {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("field1", "value1");
+ doc.addField("field2", "value1");
+ doc.addField("field3", "value2");
+ doc.addField("field4", 15);
+ List<Integer> list = new ArrayList<Integer>();
+ list.add(45);
+ list.add(33);
+ list.add(20);
+ doc.addField("field5", list);
+ doc.setDocumentBoost(5f);
+
+ SolrInputDocument clone = doc.deepCopy();
+
+ System.out.println("doc1: "+ doc);
+ System.out.println("clone: "+ clone);
+
+ assertNotSame(doc, clone);
+
+ Collection<String> fieldNames = doc.getFieldNames();
+ for (String name : fieldNames) {
+ Collection<Object> values = doc.getFieldValues(name);
+ Collection<Object> cloneValues = clone.getFieldValues(name);
+
+ assertEquals(values.size(), cloneValues.size());
+ assertNotSame(values, cloneValues);
+
+ Iterator<Object> cloneIt = cloneValues.iterator();
+ for (Object value : values) {
+ Object cloneValue = cloneIt.next();
+ assertSame(value, cloneValue);
+ }
+ }
+ }
+
+}
Modified: lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java?rev=1363013&r1=1363012&r2=1363013&view=diff
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java (original)
+++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java Wed Jul 18 16:52:42 2012
@@ -18,10 +18,10 @@
package org.apache.solr.common;
import java.io.Serializable;
+import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
-import java.util.Collection;
import java.util.Set;
/**
@@ -182,6 +182,15 @@ public class SolrInputDocument implement
return "SolrInputDocument" + _fields.values();
}
+ public SolrInputDocument deepCopy() {
+ SolrInputDocument clone = new SolrInputDocument();
+ Set<Entry<String,SolrInputField>> entries = _fields.entrySet();
+ for (Map.Entry<String,SolrInputField> fieldEntry : entries) {
+ clone._fields.put(fieldEntry.getKey(), fieldEntry.getValue().deepCopy());
+ }
+ clone._documentBoost = _documentBoost;
+ return clone;
+ }
//---------------------------------------------------
// MAP interface
Modified: lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/SolrInputField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/SolrInputField.java?rev=1363013&r1=1363012&r2=1363013&view=diff
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/SolrInputField.java (original)
+++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/SolrInputField.java Wed Jul 18 16:52:42 2012
@@ -199,4 +199,19 @@ public class SolrInputField implements I
{
return name + ((boost == 1.0) ? "=" : ("("+boost+")=")) + value;
}
+
+ public SolrInputField deepCopy() {
+ SolrInputField clone = new SolrInputField(name);
+ clone.boost = boost;
+ // We can't clone here, so we rely on simple primitives
+ if (value instanceof Collection) {
+ Collection<Object> values = (Collection<Object>) value;
+ Collection<Object> cloneValues = new ArrayList<Object>(values.size());
+ cloneValues.addAll(values);
+ clone.value = cloneValues;
+ } else {
+ clone.value = value;
+ }
+ return clone;
+ }
}