You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2016/04/28 21:52:53 UTC
lucene-solr:master: SOLR-9034: fix atomic updates for copyField w/
docValues
Repository: lucene-solr
Updated Branches:
refs/heads/master 2c66d4b04 -> c897917c7
SOLR-9034: fix atomic updates for copyField w/ docValues
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c897917c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c897917c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c897917c
Branch: refs/heads/master
Commit: c897917c718eef75d66c5d0006f409d5c95260c7
Parents: 2c66d4b
Author: yonik <yo...@apache.org>
Authored: Thu Apr 28 15:52:17 2016 -0400
Committer: yonik <yo...@apache.org>
Committed: Thu Apr 28 15:52:22 2016 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 3 ++
.../handler/component/RealTimeGetComponent.java | 2 +-
.../apache/solr/search/SolrIndexSearcher.java | 16 +++++++
.../test-files/solr/collection1/conf/schema.xml | 34 +++++++++------
.../update/processor/AtomicUpdatesTest.java | 44 ++++++++++++++++++++
5 files changed, 86 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c897917c/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 0d69bca..8ac956f 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -159,6 +159,9 @@ Bug Fixes
* SOLR-9046: Fix solr.cmd that wrongly assumes Jetty will always listen on 0.0.0.0.
(Bram Van Dam, Uwe Schindler)
+* SOLR-9034: Atomic updates failed to work when there were copyField targets that had docValues
+ enabled. (Karthik Ramachandran, Ishan Chattopadhyaya, yonik)
+
Optimizations
----------------------
* SOLR-8722: Don't force a full ZkStateReader refresh on every Overseer operation.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c897917c/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
index 8332c79..0908f00 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
@@ -342,7 +342,7 @@ public class RealTimeGetComponent extends SearchComponent
if (docid < 0) return null;
Document luceneDocument = searcher.doc(docid);
sid = toSolrInputDocument(luceneDocument, core.getLatestSchema());
- searcher.decorateDocValueFields(sid, docid, searcher.getNonStoredDVs(false));
+ searcher.decorateDocValueFields(sid, docid, searcher.getNonStoredDVsWithoutCopyTargets());
}
} finally {
if (searcherHolder != null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c897917c/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index 4c9790a..6ff5469 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -183,6 +183,9 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
/** Contains the names/patterns of all docValues=true,stored=false,useDocValuesAsStored=true fields in the schema. */
private final Set<String> nonStoredDVsUsedAsStored;
+ /** Contains the names/patterns of all docValues=true,stored=false fields, excluding those that are copyField targets in the schema. */
+ private final Set<String> nonStoredDVsWithoutCopyTargets;
+
private Collection<String> storedHighlightFieldNames;
private DirectoryFactory directoryFactory;
@@ -359,6 +362,8 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
final Set<String> nonStoredDVsUsedAsStored = new HashSet<>();
final Set<String> allNonStoredDVs = new HashSet<>();
+ final Set<String> nonStoredDVsWithoutCopyTargets = new HashSet<>();
+
this.fieldInfos = leafReader.getFieldInfos();
for (FieldInfo fieldInfo : fieldInfos) {
final SchemaField schemaField = schema.getFieldOrNull(fieldInfo.name);
@@ -367,11 +372,15 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
nonStoredDVsUsedAsStored.add(fieldInfo.name);
}
allNonStoredDVs.add(fieldInfo.name);
+ if (!schema.isCopyFieldTarget(schemaField)) {
+ nonStoredDVsWithoutCopyTargets.add(fieldInfo.name);
+ }
}
}
this.nonStoredDVsUsedAsStored = Collections.unmodifiableSet(nonStoredDVsUsedAsStored);
this.allNonStoredDVs = Collections.unmodifiableSet(allNonStoredDVs);
+ this.nonStoredDVsWithoutCopyTargets = Collections.unmodifiableSet(nonStoredDVsWithoutCopyTargets);
// We already have our own filter cache
setQueryCache(null);
@@ -876,6 +885,13 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
return onlyUseDocValuesAsStored ? nonStoredDVsUsedAsStored : allNonStoredDVs;
}
+ /**
+ * Returns an unmodifiable set of names of non-stored docValues fields, except those that are targets of a copy field.
+ */
+ public Set<String> getNonStoredDVsWithoutCopyTargets() {
+ return nonStoredDVsWithoutCopyTargets;
+ }
+
/* ********************** end document retrieval *************************/
////////////////////////////////////////////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c897917c/solr/core/src/test-files/solr/collection1/conf/schema.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema.xml b/solr/core/src/test-files/solr/collection1/conf/schema.xml
index 64bca9a..44b3d72 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema.xml
@@ -648,20 +648,30 @@
<dynamicField name="*_sev_enum" type="severityType" indexed="true" stored="true" docValues="true" multiValued="true" />
<!-- With DocValues=true -->
- <dynamicField name="*_i_dv" type="int" indexed="true" stored="true" docValues="true"/>
- <dynamicField name="*_l_dv" type="long" indexed="true" stored="true" docValues="true"/>
- <dynamicField name="*_f_dv" type="float" indexed="true" stored="true" docValues="true"/>
- <dynamicField name="*_d_dv" type="double" indexed="true" stored="true" docValues="true"/>
- <dynamicField name="*_dt_dv" type="date" indexed="true" stored="true" docValues="true"/>
- <dynamicField name="*_f1_dv" type="float" indexed="true" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_i_dv" type="int" indexed="true" stored="true" docValues="true" />
+ <dynamicField name="*_l_dv" type="long" indexed="true" stored="true" docValues="true" />
+ <dynamicField name="*_f_dv" type="float" indexed="true" stored="true" docValues="true" />
+ <dynamicField name="*_d_dv" type="double" indexed="true" stored="true" docValues="true" />
+ <dynamicField name="*_dt_dv" type="date" indexed="true" stored="true" docValues="true" />
+ <dynamicField name="*_f1_dv" type="float" indexed="true" stored="true" docValues="true" multiValued="false" />
<!-- Non-stored, DocValues=true -->
- <dynamicField name="*_i_dvo" multiValued="false" type="int" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true"/>
- <dynamicField name="*_d_dvo" multiValued="false" type="double" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true"/>
- <dynamicField name="*_s_dvo" multiValued="false" type="string" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true"/>
- <dynamicField name="*_ii_dvo" multiValued="true" type="int" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true"/>
- <dynamicField name="*_dd_dvo" multiValued="true" type="double" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true"/>
-
+ <dynamicField name="*_i_dvo" multiValued="false" type="int" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true" />
+ <dynamicField name="*_d_dvo" multiValued="false" type="double" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true" />
+ <dynamicField name="*_s_dvo" multiValued="false" type="string" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true" />
+ <dynamicField name="*_ii_dvo" multiValued="true" type="int" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true" />
+ <dynamicField name="*_dd_dvo" multiValued="true" type="double" docValues="true" indexed="true" stored="false" useDocValuesAsStored="true" />
+
+ <!-- Non-stored, DocValues=true, useDocValuesAsStored=false -->
+ <field name="single_i_dvn" multiValued="false" type="int" indexed="true" stored="true" />
+ <field name="single_d_dvn" multiValued="false" type="double" indexed="true" stored="true" />
+ <field name="single_s_dvn" multiValued="false" type="string" indexed="true" stored="true" />
+ <field name="copy_single_i_dvn" multiValued="false" type="int" docValues="true" indexed="true" stored="false" useDocValuesAsStored="false" />
+ <field name="copy_single_d_dvn" multiValued="false" type="double" docValues="true" indexed="true" stored="false" useDocValuesAsStored="false" />
+ <field name="copy_single_s_dvn" multiValued="false" type="string" docValues="true" indexed="true" stored="false" useDocValuesAsStored="false" />
+ <copyField source="single_i_dvn" dest="copy_single_i_dvn" />
+ <copyField source="single_d_dvn" dest="copy_single_d_dvn" />
+ <copyField source="single_s_dvn" dest="copy_single_s_dvn" />
</fields>
<defaultSearchField>text</defaultSearchField>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c897917c/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdatesTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdatesTest.java b/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdatesTest.java
index e1726f8..a91a34b 100644
--- a/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdatesTest.java
+++ b/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdatesTest.java
@@ -43,6 +43,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
h.update("<delete><query>*:*</query></delete>");
assertU(commit());
}
+
@Test
public void testRemove() throws Exception {
SolrInputDocument doc;
@@ -1065,6 +1066,49 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
"/response/docs/[0]/multi_ii_dvo/[0]==100",
"/response/docs/[0]/multi_ii_dvo/[1]==" + Integer.MAX_VALUE);
}
+
+ @Test
+ public void testAtomicUpdatesOnNonStoredDocValuesCopyField() throws Exception {
+ assertU(adoc(sdoc("id", 101, "title", "title2", "single_i_dvn", 100)));
+ assertU(adoc(sdoc("id", 102, "title", "title3", "single_d_dvn", 3.14)));
+ assertU(adoc(sdoc("id", 103, "single_s_dvn", "abc", "single_i_dvn", 1)));
+ assertU(commit());
+
+ // Do each one twice... the first time it will be retrieved from the index, and the second time from the transaction log.
+ for (int i=0; i<2; i++) {
+ assertU(adoc(sdoc("id", 101, "title", ImmutableMap.of("set", "newtitle2"),
+ "single_i_dvn", ImmutableMap.of("inc", 1))));
+ assertU(adoc(sdoc("id", 102, "title", ImmutableMap.of("set", "newtitle3"),
+ "single_d_dvn", ImmutableMap.of("inc", 1))));
+ assertU(adoc(sdoc("id", 103, "single_i_dvn", ImmutableMap.of("inc", 1))));
+ }
+ assertU(commit());
+
+ assertJQ(req("q", "id:101"),
+ "/response/docs/[0]/id==101",
+ "/response/docs/[0]/title/[0]=='newtitle2'",
+ "/response/docs/[0]/single_i_dvn==102");
+
+ assertJQ(req("q", "id:102"),
+ 1e-4,
+ "/response/docs/[0]/id==102",
+ "/response/docs/[0]/title/[0]=='newtitle3'",
+ "/response/docs/[0]/single_d_dvn==5.14");
+
+ assertJQ(req("q", "id:103"),
+ "/response/docs/[0]/id==103",
+ "/response/docs/[0]/single_s_dvn=='abc'",
+ "/response/docs/[0]/single_i_dvn==3");
+
+ // test that non stored docvalues was carried forward for a non-docvalue update
+ assertU(adoc(sdoc("id", 103, "single_s_dvn", ImmutableMap.of("set", "abcupdate"),
+ "single_i_dvn", ImmutableMap.of("set", 5))));
+ assertU(commit());
+ assertJQ(req("q", "id:103"),
+ "/response/docs/[0]/id==103",
+ "/response/docs/[0]/single_s_dvn=='abcupdate'",
+ "/response/docs/[0]/single_i_dvn==5");
+ }
@Test
public void testInvalidOperation() {