You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2016/03/25 15:19:26 UTC
lucene-solr:branch_6x: SOLR-8865: Real-time get sometimes fails to
retrieve stored fields from docValues
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 1d64c47aa -> ca374d947
SOLR-8865: Real-time get sometimes fails to retrieve stored fields from docValues
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ca374d94
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ca374d94
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ca374d94
Branch: refs/heads/branch_6x
Commit: ca374d947678a81df665095387dadc0551590729
Parents: 1d64c47
Author: yonik <yo...@apache.org>
Authored: Fri Mar 25 10:18:31 2016 -0400
Committer: yonik <yo...@apache.org>
Committed: Fri Mar 25 10:18:58 2016 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 3 ++
.../handler/component/RealTimeGetComponent.java | 13 ++++-
.../solr/collection1/conf/schema15.xml | 1 +
.../solr/collection1/conf/schema_latest.xml | 51 +++++++++++++-------
.../org/apache/solr/search/TestRealTimeGet.java | 49 +++++++++++++++++--
5 files changed, 93 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ca374d94/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index a9731e1..4bd24df 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -351,6 +351,9 @@ Bug Fixes
* SOLR-8891: Fix StrField.toObject and toExternal to work with docValue IndexableField
instances. (yonik)
+* SOLR-8865: Real-time get sometimes fails to retrieve stored fields from docValues.
+ (Ishan Chattopadhyaya, yonik)
+
Optimizations
----------------------
* SOLR-7876: Speed up queries and operations that use many terms when timeAllowed has not been
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ca374d94/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
index 2f71b3e..8332c79 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReaderContext;
@@ -258,6 +259,7 @@ public class RealTimeGetComponent extends SearchComponent
if (docid < 0) continue;
Document luceneDocument = searcher.doc(docid, rsp.getReturnFields().getLuceneFieldNames());
SolrDocument doc = toSolrDoc(luceneDocument, core.getLatestSchema());
+ searcher.decorateDocValueFields(doc, docid, searcher.getNonStoredDVs(true));
if( transformer != null ) {
transformer.transform(doc, docid, 0);
}
@@ -408,8 +410,15 @@ public class RealTimeGetComponent extends SearchComponent
// copy the stored fields only
Document out = new Document();
for (IndexableField f : doc.getFields()) {
- if (f.fieldType().stored() ) {
- out.add((IndexableField) f);
+ if (f.fieldType().stored()) {
+ out.add(f);
+ } else if (f.fieldType().docValuesType() != DocValuesType.NONE) {
+ SchemaField schemaField = schema.getFieldOrNull(f.name());
+ if (schemaField != null && !schemaField.stored() && schemaField.useDocValuesAsStored()) {
+ out.add(f);
+ }
+ } else {
+ log.debug("Don't know how to handle field " + f);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ca374d94/solr/core/src/test-files/solr/collection1/conf/schema15.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema15.xml b/solr/core/src/test-files/solr/collection1/conf/schema15.xml
index ea46d85..55ec3f1 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema15.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema15.xml
@@ -530,6 +530,7 @@
<dynamicField name="*_s" type="string" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_ss" type="string" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_l" type="long" indexed="true" stored="true"/>
+ <dynamicField name="*_ll" type="long" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_t" type="text" indexed="true" stored="true"/>
<dynamicField name="*_tt" type="text" indexed="true" stored="true"/>
<dynamicField name="*_ws" type="nametext" indexed="true" stored="true"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ca374d94/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml b/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml
index ef82aa1..cae373c 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml
@@ -110,7 +110,7 @@
<!-- If you remove this field, you must _also_ disable the update log in solrconfig.xml
or Solr won't start. _version_ and update log are required for SolrCloud
-->
- <field name="_version_" type="long" indexed="true" stored="true"/>
+ <field name="_version_" type="long" indexed="false" stored="false" docValues="true" />
<!-- points to the root document of a block of nested documents. Required for nested
document support, may be removed otherwise
@@ -199,27 +199,48 @@
RESTRICTION: the glob-like pattern in the name attribute must have
a "*" only at the start or the end. -->
+ <!-- docvalues and stored are exclusive -->
<dynamicField name="*_i" type="int" indexed="true" stored="true"/>
<dynamicField name="*_is" type="int" indexed="true" stored="true" multiValued="true"/>
- <dynamicField name="*_id" type="int" indexed="true" stored="true" docValues="true" />
- <dynamicField name="*_ids" type="int" indexed="true" stored="true" multiValued="true" docValues="true" />
+ <dynamicField name="*_id" type="int" indexed="true" stored="false" docValues="true" />
+ <dynamicField name="*_ids" type="int" indexed="true" stored="false" multiValued="true" docValues="true" />
<dynamicField name="*_s" type="string" indexed="true" stored="true" />
<dynamicField name="*_s1" type="string" indexed="true" stored="true" />
<dynamicField name="*_ss" type="string" indexed="true" stored="true" multiValued="true"/>
- <dynamicField name="*_sd" type="string" indexed="true" stored="true" docValues="true" />
- <dynamicField name="*_sds" type="string" indexed="true" stored="true" multiValued="true" docValues="true" />
+ <dynamicField name="*_sd" type="string" indexed="true" stored="false" docValues="true" />
+ <dynamicField name="*_sds" type="string" indexed="true" stored="false" multiValued="true" docValues="true" />
<dynamicField name="*_l" type="long" indexed="true" stored="true"/>
<dynamicField name="*_ls" type="long" indexed="true" stored="true" multiValued="true"/>
- <dynamicField name="*_ld" type="long" indexed="true" stored="true" docValues="true" />
- <dynamicField name="*_lds" type="long" indexed="true" stored="true" multiValued="true" docValues="true" />
+ <dynamicField name="*_ld" type="long" indexed="true" stored="false" docValues="true" />
+ <dynamicField name="*_lds" type="long" indexed="true" stored="false" multiValued="true" docValues="true" />
<dynamicField name="*_f" type="float" indexed="true" stored="true"/>
<dynamicField name="*_fs" type="float" indexed="true" stored="true" multiValued="true"/>
- <dynamicField name="*_fd" type="float" indexed="true" stored="true" docValues="true" />
- <dynamicField name="*_fds" type="float" indexed="true" stored="true" multiValued="true" docValues="true" />
+ <dynamicField name="*_fd" type="float" indexed="true" stored="false" docValues="true" />
+ <dynamicField name="*_fds" type="float" indexed="true" stored="false" multiValued="true" docValues="true" />
<dynamicField name="*_d" type="double" indexed="true" stored="true"/>
<dynamicField name="*_ds" type="double" indexed="true" stored="true" multiValued="true"/>
- <dynamicField name="*_dd" type="double" indexed="true" stored="true" docValues="true" />
- <dynamicField name="*_dds" type="double" indexed="true" stored="true" multiValued="true" docValues="true" />
+ <dynamicField name="*_dd" type="double" indexed="true" stored="false" docValues="true" />
+ <dynamicField name="*_dds" type="double" indexed="true" stored="false" multiValued="true" docValues="true" />
+ <dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
+ <dynamicField name="*_dts" type="date" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="*_dtd" type="date" indexed="true" stored="false" docValues="true" />
+ <dynamicField name="*_dtds" type="date" indexed="true" stored="false" multiValued="true" docValues="true" />
+
+ <!-- docvalues and stored (S suffix) -->
+ <dynamicField name="*_idS" type="int" indexed="true" stored="true" docValues="true" />
+ <dynamicField name="*_idsS" type="int" indexed="true" stored="true" multiValued="true" docValues="true" />
+ <dynamicField name="*_sdS" type="string" indexed="true" stored="true" docValues="true" />
+ <dynamicField name="*_sdsS" type="string" indexed="true" stored="true" multiValued="true" docValues="true" />
+ <dynamicField name="*_ldS" type="long" indexed="true" stored="true" docValues="true" />
+ <dynamicField name="*_ldsS" type="long" indexed="true" stored="true" multiValued="true" docValues="true" />
+ <dynamicField name="*_fdS" type="float" indexed="true" stored="true" docValues="true" />
+ <dynamicField name="*_fdsS" type="float" indexed="true" stored="true" multiValued="true" docValues="true" />
+ <dynamicField name="*_ddS" type="double" indexed="true" stored="true" docValues="true" />
+ <dynamicField name="*_ddsS" type="double" indexed="true" stored="true" multiValued="true" docValues="true" />
+ <dynamicField name="*_dtdS" type="date" indexed="true" stored="true" docValues="true" />
+ <dynamicField name="*_dtdsS" type="date" indexed="true" stored="true" multiValued="true" docValues="true" />
+
+
<dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
<dynamicField name="*_bs" type="boolean" indexed="true" stored="true" multiValued="true"/>
@@ -230,10 +251,6 @@
<!-- Type used to index the lat and lon components for the "location" FieldType -->
<dynamicField name="*_coordinate" type="tdouble" indexed="true" stored="false" />
- <dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
- <dynamicField name="*_dts" type="date" indexed="true" stored="true" multiValued="true"/>
- <dynamicField name="*_dtd" type="date" indexed="true" stored="true" docValues="true" />
- <dynamicField name="*_dtds" type="date" indexed="true" stored="true" multiValued="true" docValues="true" />
<dynamicField name="*_p" type="location" indexed="true" stored="true"/>
<!-- some trie-coded dynamic fields for faster range queries -->
@@ -258,8 +275,8 @@
<!-- needed by dedup config -->
- <dynamicField name="*_sS" type="string" indexed="false" stored="true"/>
<field name="signatureField" type="string" indexed="true" stored="false"/>
+ <dynamicField name="*_sS" type="string" indexed="false" stored="true"/>
</fields>
@@ -695,7 +712,7 @@
<!-- since fields of this type are by default not stored or indexed,
any data added to them will be ignored outright. -->
- <fieldtype name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" />
+ <fieldtype name="ignored" stored="false" indexed="false" docValues="false" multiValued="true" class="solr.StrField" />
<!-- This point type indexes the coordinates as separate fields (subFields)
If subFieldType is defined, it references a type, and a dynamic field
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ca374d94/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java b/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java
index 3f30440..fe02108 100644
--- a/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java
+++ b/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java
@@ -43,7 +43,7 @@ public class TestRealTimeGet extends TestRTGBase {
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig-tlog.xml","schema15.xml");
+ initCore("solrconfig-tlog.xml","schema_latest.xml");
}
@@ -52,12 +52,22 @@ public class TestRealTimeGet extends TestRTGBase {
clearIndex();
assertU(commit());
- assertU(adoc("id","1"));
+ assertU(adoc("id","1",
+ "a_f","-1.5", "a_fd","-1.5", "a_fdS","-1.5", "a_fs","1.0","a_fs","2.5", "a_fds","1.0","a_fds","2.5", "a_fdsS","1.0","a_fdsS","2.5",
+ "a_d","-1.2E99", "a_dd","-1.2E99", "a_ddS","-1.2E99", "a_ds","1.0","a_ds","2.5", "a_dds","1.0","a_dds","2.5", "a_ddsS","1.0","a_ddsS","2.5",
+ "a_i","-1", "a_id","-1", "a_idS","-1", "a_is","1","a_is","2", "a_ids","1","a_ids","2", "a_idsS","1","a_idsS","2",
+ "a_l","-9999999999", "a_ld","-9999999999", "a_ldS","-9999999999", "a_ls","1","a_ls","9999999999", "a_lds","1","a_lds","9999999999", "a_ldsS","1","a_ldsS","9999999999"
+ ));
assertJQ(req("q","id:1")
,"/response/numFound==0"
);
- assertJQ(req("qt","/get", "id","1", "fl","id")
- ,"=={'doc':{'id':'1'}}"
+ assertJQ(req("qt","/get", "id","1", "fl","id, a_f,a_fd,a_fdS a_fs,a_fds,a_fdsS, a_d,a_dd,a_ddS, a_ds,a_dds,a_ddsS, a_i,a_id,a_idS a_is,a_ids,a_idsS, a_l,a_ld,a_ldS a_ls,a_lds,a_ldsS")
+ ,"=={'doc':{'id':'1'" +
+ ", a_f:-1.5, a_fd:-1.5, a_fdS:-1.5, a_fs:[1.0,2.5], a_fds:[1.0,2.5],a_fdsS:[1.0,2.5]" +
+ ", a_d:-1.2E99, a_dd:-1.2E99, a_ddS:-1.2E99, a_ds:[1.0,2.5],a_dds:[1.0,2.5],a_ddsS:[1.0,2.5]" +
+ ", a_i:-1, a_id:-1, a_idS:-1, a_is:[1,2],a_ids:[1,2],a_idsS:[1,2]" +
+ ", a_l:-9999999999, a_ld:-9999999999, a_ldS:-9999999999, a_ls:[1,9999999999],a_lds:[1,9999999999],a_ldsS:[1,9999999999]" +
+ " }}"
);
assertJQ(req("qt","/get","ids","1", "fl","id")
,"=={" +
@@ -72,6 +82,17 @@ public class TestRealTimeGet extends TestRTGBase {
assertJQ(req("q","id:1")
,"/response/numFound==1"
);
+
+ // a cut-n-paste of the first big query, but this time it will be retrieved from the index rather than the transaction log
+ assertJQ(req("qt","/get", "id","1", "fl","id, a_f,a_fd,a_fdS a_fs,a_fds,a_fdsS, a_d,a_dd,a_ddS, a_ds,a_dds,a_ddsS, a_i,a_id,a_idS a_is,a_ids,a_idsS, a_l,a_ld,a_ldS a_ls,a_lds,a_ldsS")
+ ,"=={'doc':{'id':'1'" +
+ ", a_f:-1.5, a_fd:-1.5, a_fdS:-1.5, a_fs:[1.0,2.5], a_fds:[1.0,2.5],a_fdsS:[1.0,2.5]" +
+ ", a_d:-1.2E99, a_dd:-1.2E99, a_ddS:-1.2E99, a_ds:[1.0,2.5],a_dds:[1.0,2.5],a_ddsS:[1.0,2.5]" +
+ ", a_i:-1, a_id:-1, a_idS:-1, a_is:[1,2],a_ids:[1,2],a_idsS:[1,2]" +
+ ", a_l:-9999999999, a_ld:-9999999999, a_ldS:-9999999999, a_ls:[1,9999999999],a_lds:[1,9999999999],a_ldsS:[1,9999999999]" +
+ " }}"
+ );
+
assertJQ(req("qt","/get","id","1", "fl","id")
,"=={'doc':{'id':'1'}}"
);
@@ -101,7 +122,7 @@ public class TestRealTimeGet extends TestRTGBase {
assertJQ(req("qt","/get","id","10", "fl","id")
,"=={'doc':{'id':'10'}}"
);
- assertU(delQ("id:10 abcdef"));
+ assertU(delQ("id:10 foo_s:abcdef"));
assertJQ(req("qt","/get","id","10")
,"=={'doc':null}"
);
@@ -109,6 +130,24 @@ public class TestRealTimeGet extends TestRTGBase {
,"=={'doc':{'id':'11'}}"
);
+ // multivalued field
+ assertU(adoc("id","12", "val_ls","1", "val_ls","2"));
+ assertJQ(req("q","id:12")
+ ,"/response/numFound==0"
+ );
+ assertJQ(req("qt","/get", "id","12", "fl","id,val_ls")
+ ,"=={'doc':{'id':'12', 'val_ls':[1,2]}}"
+ );
+
+ assertU(commit());
+
+ assertJQ(req("qt","/get", "id","12", "fl","id,val_ls")
+ ,"=={'doc':{'id':'12', 'val_ls':[1,2]}}"
+ );
+ assertJQ(req("q","id:12")
+ ,"/response/numFound==1"
+ );
+
SolrQueryRequest req = req();
RefCounted<SolrIndexSearcher> realtimeHolder = req.getCore().getRealtimeSearcher();