You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ho...@apache.org on 2014/01/28 20:12:05 UTC
svn commit: r1562160 - in /lucene/dev/branches/branch_4x: ./ dev-tools/
lucene/ lucene/analysis/
lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/
lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/backwards/
luce...
Author: hossman
Date: Tue Jan 28 19:12:03 2014
New Revision: 1562160
URL: http://svn.apache.org/r1562160
Log:
SOLR-5652: test fixes: add 'plain' docvalue field variants to schema, and check codecs for support of missing docValues before trying to sort on them (merge r1562155)
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/dev-tools/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/BUILD.txt (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (props changed)
lucene/dev/branches/branch_4x/lucene/JRE_VERSION_MIGRATION.txt (props changed)
lucene/dev/branches/branch_4x/lucene/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/MIGRATE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/README.txt (props changed)
lucene/dev/branches/branch_4x/lucene/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/ (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/ASCIITLD.jflex-macro (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/SUPPLEMENTARY.jflex-macro (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/StandardTokenizerImpl40.java (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/StandardTokenizerImpl40.jflex (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/UAX29URLEmailTokenizerImpl40.java (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/UAX29URLEmailTokenizerImpl40.jflex (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/package.html (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilterFactory.java (props changed)
lucene/dev/branches/branch_4x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_4x/lucene/benchmark/ (props changed)
lucene/dev/branches/branch_4x/lucene/build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/classification/ (props changed)
lucene/dev/branches/branch_4x/lucene/classification/build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/classification/ivy.xml (props changed)
lucene/dev/branches/branch_4x/lucene/classification/src/ (props changed)
lucene/dev/branches/branch_4x/lucene/codecs/ (props changed)
lucene/dev/branches/branch_4x/lucene/common-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSort.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java (props changed)
lucene/dev/branches/branch_4x/lucene/demo/ (props changed)
lucene/dev/branches/branch_4x/lucene/expressions/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/grouping/ (props changed)
lucene/dev/branches/branch_4x/lucene/highlighter/ (props changed)
lucene/dev/branches/branch_4x/lucene/ivy-settings.xml (props changed)
lucene/dev/branches/branch_4x/lucene/ivy-versions.properties (props changed)
lucene/dev/branches/branch_4x/lucene/join/ (props changed)
lucene/dev/branches/branch_4x/lucene/licenses/ (props changed)
lucene/dev/branches/branch_4x/lucene/memory/ (props changed)
lucene/dev/branches/branch_4x/lucene/misc/ (props changed)
lucene/dev/branches/branch_4x/lucene/module-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/queries/ (props changed)
lucene/dev/branches/branch_4x/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java (props changed)
lucene/dev/branches/branch_4x/lucene/queryparser/ (props changed)
lucene/dev/branches/branch_4x/lucene/replicator/ (props changed)
lucene/dev/branches/branch_4x/lucene/sandbox/ (props changed)
lucene/dev/branches/branch_4x/lucene/site/ (props changed)
lucene/dev/branches/branch_4x/lucene/spatial/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
lucene/dev/branches/branch_4x/lucene/tools/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (props changed)
lucene/dev/branches/branch_4x/solr/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/solr/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/README.txt (props changed)
lucene/dev/branches/branch_4x/solr/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_4x/solr/build.xml (props changed)
lucene/dev/branches/branch_4x/solr/cloud-dev/ (props changed)
lucene/dev/branches/branch_4x/solr/common-build.xml (props changed)
lucene/dev/branches/branch_4x/solr/contrib/ (props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/CursorPagingTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestConfig.java (props changed)
lucene/dev/branches/branch_4x/solr/example/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/scripts/ (props changed)
lucene/dev/branches/branch_4x/solr/site/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/ (props changed)
lucene/dev/branches/branch_4x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_4x/solr/webapp/ (props changed)
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1562160&r1=1562159&r2=1562160&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Tue Jan 28 19:12:03 2014
@@ -1379,6 +1379,18 @@ public abstract class LuceneTestCase ext
}
}
+ /** Returns true if the default codec supports single valued docvalues with missing values */
+ public static boolean defaultCodecSupportsMissingDocValues() {
+ String name = Codec.getDefault().getName();
+ if (name.equals("Lucene3x") ||
+ name.equals("Lucene40") || name.equals("Appending") ||
+ name.equals("Lucene41") ||
+ name.equals("Lucene42")) {
+ return false;
+ }
+ return true;
+ }
+
/** Returns true if the default codec supports SORTED_SET docvalues */
public static boolean defaultCodecSupportsSortedSet() {
if (!defaultCodecSupportsDocValues()) {
Modified: lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml?rev=1562160&r1=1562159&r2=1562160&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml Tue Jan 28 19:12:03 2014
@@ -32,36 +32,42 @@ NOTE: Tests expect every field in this s
<field name="str" type="str" />
<field name="str_last" type="str_last" />
<field name="str_first" type="str_first" />
+ <field name="str_dv" type="str_dv" />
<field name="str_dv_last" type="str_dv_last" />
<field name="str_dv_first" type="str_dv_first" />
<field name="bin" type="bin" />
<field name="bin_last" type="bin_last" />
<field name="bin_first" type="bin_first" />
+ <field name="bin_dv" type="bin_dv" />
<field name="bin_dv_last" type="bin_dv_last" />
<field name="bin_dv_first" type="bin_dv_first" />
<field name="int" type="int" />
<field name="int_last" type="int_last" />
<field name="int_first" type="int_first" />
+ <field name="int_dv" type="int_dv" />
<field name="int_dv_last" type="int_dv_last" />
<field name="int_dv_first" type="int_dv_first" />
<field name="long" type="long" />
<field name="long_last" type="long_last" />
<field name="long_first" type="long_first" />
+ <field name="long_dv" type="long_dv" />
<field name="long_dv_last" type="long_dv_last" />
<field name="long_dv_first" type="long_dv_first" />
<field name="float" type="float" />
<field name="float_last" type="float_last" />
<field name="float_first" type="float_first" />
+ <field name="float_dv" type="float_dv" />
<field name="float_dv_last" type="float_dv_last" />
<field name="float_dv_first" type="float_dv_first" />
<field name="double" type="double" />
<field name="double_last" type="double_last" />
<field name="double_first" type="double_first" />
+ <field name="double_dv" type="double_dv" />
<field name="double_dv_last" type="double_dv_last" />
<field name="double_dv_first" type="double_dv_first" />
@@ -73,26 +79,37 @@ NOTE: Tests expect every field in this s
<copyField source="str" dest="str_last" />
<copyField source="str" dest="str_first" />
+ <copyField source="str" dest="str_dv" />
<copyField source="str" dest="str_dv_last" />
<copyField source="str" dest="str_dv_first" />
+
<copyField source="bin" dest="bin_last" />
<copyField source="bin" dest="bin_first" />
+ <copyField source="bin" dest="bin_dv" />
<copyField source="bin" dest="bin_dv_last" />
<copyField source="bin" dest="bin_dv_first" />
+
<copyField source="int" dest="int_last" />
<copyField source="int" dest="int_first" />
+ <copyField source="int" dest="int_dv" />
<copyField source="int" dest="int_dv_last" />
<copyField source="int" dest="int_dv_first" />
+
<copyField source="long" dest="long_last" />
<copyField source="long" dest="long_first" />
+ <copyField source="long" dest="long_dv" />
<copyField source="long" dest="long_dv_last" />
<copyField source="long" dest="long_dv_first" />
+
<copyField source="float" dest="float_last" />
<copyField source="float" dest="float_first" />
+ <copyField source="float" dest="float_dv" />
<copyField source="float" dest="float_dv_last" />
<copyField source="float" dest="float_dv_first" />
+
<copyField source="double" dest="double_last" />
<copyField source="double" dest="double_first" />
+ <copyField source="double" dest="double_dv" />
<copyField source="double" dest="double_dv_last" />
<copyField source="double" dest="double_dv_first" />
@@ -101,36 +118,42 @@ NOTE: Tests expect every field in this s
<fieldtype name="str" class="solr.StrField" stored="true" indexed="true" />
<fieldtype name="str_last" class="solr.StrField" stored="true" indexed="true" sortMissingLast="true"/>
<fieldtype name="str_first" class="solr.StrField" stored="true" indexed="true" sortMissingFirst="true"/>
+ <fieldtype name="str_dv" class="solr.StrField" stored="true" indexed="false" docValues="true"/>
<fieldtype name="str_dv_last" class="solr.StrField" stored="true" indexed="false" docValues="true" sortMissingLast="true"/>
<fieldtype name="str_dv_first" class="solr.StrField" stored="true" indexed="false" docValues="true" sortMissingFirst="true"/>
<fieldtype name="bin" class="solr.SortableBinaryField" stored="true" indexed="true" />
<fieldtype name="bin_last" class="solr.SortableBinaryField" stored="true" indexed="true" sortMissingLast="true"/>
<fieldtype name="bin_first" class="solr.SortableBinaryField" stored="true" indexed="true" sortMissingFirst="true"/>
+ <fieldtype name="bin_dv" class="solr.SortableBinaryField" stored="true" indexed="false" docValues="true"/>
<fieldtype name="bin_dv_last" class="solr.SortableBinaryField" stored="true" indexed="false" docValues="true" sortMissingLast="true"/>
<fieldtype name="bin_dv_first" class="solr.SortableBinaryField" stored="true" indexed="false" docValues="true" sortMissingFirst="true"/>
<fieldtype name="int" class="solr.TrieIntField" stored="true" indexed="true" />
<fieldtype name="int_last" class="solr.TrieIntField" stored="true" indexed="true" sortMissingLast="true"/>
<fieldtype name="int_first" class="solr.TrieIntField" stored="true" indexed="true" sortMissingFirst="true"/>
+ <fieldtype name="int_dv" class="solr.TrieIntField" stored="true" indexed="false" docValues="true"/>
<fieldtype name="int_dv_last" class="solr.TrieIntField" stored="true" indexed="false" docValues="true" sortMissingLast="true"/>
<fieldtype name="int_dv_first" class="solr.TrieIntField" stored="true" indexed="false" docValues="true" sortMissingFirst="true"/>
<fieldtype name="long" class="solr.TrieLongField" stored="true" indexed="true" />
<fieldtype name="long_last" class="solr.TrieLongField" stored="true" indexed="true" sortMissingLast="true"/>
<fieldtype name="long_first" class="solr.TrieLongField" stored="true" indexed="true" sortMissingFirst="true"/>
+ <fieldtype name="long_dv" class="solr.TrieLongField" stored="true" indexed="false" docValues="true"/>
<fieldtype name="long_dv_last" class="solr.TrieLongField" stored="true" indexed="false" docValues="true" sortMissingLast="true"/>
<fieldtype name="long_dv_first" class="solr.TrieLongField" stored="true" indexed="false" docValues="true" sortMissingFirst="true"/>
<fieldtype name="float" class="solr.TrieFloatField" stored="true" indexed="true" />
<fieldtype name="float_last" class="solr.TrieFloatField" stored="true" indexed="true" sortMissingLast="true"/>
<fieldtype name="float_first" class="solr.TrieFloatField" stored="true" indexed="true" sortMissingFirst="true"/>
+ <fieldtype name="float_dv" class="solr.TrieFloatField" stored="true" indexed="false" docValues="true"/>
<fieldtype name="float_dv_last" class="solr.TrieFloatField" stored="true" indexed="false" docValues="true" sortMissingLast="true"/>
<fieldtype name="float_dv_first" class="solr.TrieFloatField" stored="true" indexed="false" docValues="true" sortMissingFirst="true"/>
<fieldtype name="double" class="solr.TrieDoubleField" stored="true" indexed="true" />
<fieldtype name="double_last" class="solr.TrieDoubleField" stored="true" indexed="true" sortMissingLast="true"/>
<fieldtype name="double_first" class="solr.TrieDoubleField" stored="true" indexed="true" sortMissingFirst="true"/>
+ <fieldtype name="double_dv" class="solr.TrieDoubleField" stored="true" indexed="false" docValues="true"/>
<fieldtype name="double_dv_last" class="solr.TrieDoubleField" stored="true" indexed="false" docValues="true" sortMissingLast="true"/>
<fieldtype name="double_dv_first" class="solr.TrieDoubleField" stored="true" indexed="false" docValues="true" sortMissingFirst="true"/>
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/CursorPagingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/CursorPagingTest.java?rev=1562160&r1=1562159&r2=1562160&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/CursorPagingTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/CursorPagingTest.java Tue Jan 28 19:12:03 2014
@@ -122,6 +122,9 @@ public class CursorPagingTest extends So
String cursorMark;
SolrParams params = null;
+ final String intsort = "int" + (random().nextBoolean() ? "" : "_dv");
+ final String intmissingsort = defaultCodecSupportsMissingDocValues() ? intsort : "int";
+
// trivial base case: ensure cursorMark against an empty index doesn't blow up
cursorMark = CURSOR_MARK_START;
params = params("q", "*:*",
@@ -147,7 +150,7 @@ public class CursorPagingTest extends So
assertU(adoc("id", "6", "str", "a", "float", "64.5", "int", "7"));
assertU(adoc("id", "1", "str", "a", "float", "64.5", "int", "7"));
assertU(adoc("id", "4", "str", "a", "float", "11.1", "int", "6"));
- assertU(adoc("id", "3", "str", "a", "float", "11.1", "int", "3"));
+ assertU(adoc("id", "3", "str", "a", "float", "11.1")); // int is missing
assertU(commit());
// base case: ensure cursorMark that matches no docs doesn't blow up
@@ -243,7 +246,7 @@ public class CursorPagingTest extends So
"facet", "true",
"facet.field", "str",
"json.nl", "map",
- "sort", "int asc, id asc");
+ "sort", intsort + " asc, id asc");
cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark)
,"/response/numFound==8"
,"/response/start==0"
@@ -271,6 +274,66 @@ public class CursorPagingTest extends So
,"/facet_counts/facet_fields/str=={'a':4,'b':1,'c':3}"
));
+ // int missing first sort with dups, id tie breaker
+ cursorMark = CURSOR_MARK_START;
+ params = params("q", "-int:2001 -int:4055",
+ "rows","3",
+ "fl", "id",
+ "json.nl", "map",
+ "sort", intmissingsort + "_first asc, id asc");
+ cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark)
+ ,"/response/numFound==8"
+ ,"/response/start==0"
+ ,"/response/docs==[{'id':3},{'id':7},{'id':0}]"
+ );
+ cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark)
+ ,"/response/numFound==8"
+ ,"/response/start==0"
+ ,"/response/docs==[{'id':4},{'id':1},{'id':6}]"
+ );
+ cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark)
+ ,"/response/numFound==8"
+ ,"/response/start==0"
+ ,"/response/docs==[{'id':9},{'id':2}]"
+ );
+ // no more, so no change to cursorMark, and no new docs
+ assertEquals(cursorMark,
+ assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark)
+ ,"/response/numFound==8"
+ ,"/response/start==0"
+ ,"/response/docs==[]"
+ ));
+
+ // int missing last sort with dups, id tie breaker
+ cursorMark = CURSOR_MARK_START;
+ params = params("q", "-int:2001 -int:4055",
+ "rows","3",
+ "fl", "id",
+ "json.nl", "map",
+ "sort", intmissingsort + "_last asc, id asc");
+ cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark)
+ ,"/response/numFound==8"
+ ,"/response/start==0"
+ ,"/response/docs==[{'id':7},{'id':0},{'id':4}]"
+ );
+ cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark)
+ ,"/response/numFound==8"
+ ,"/response/start==0"
+ ,"/response/docs==[{'id':1},{'id':6},{'id':9}]"
+ );
+ cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark)
+ ,"/response/numFound==8"
+ ,"/response/start==0"
+ ,"/response/docs==[{'id':2},{'id':3}]"
+ );
+ // no more, so no change to cursorMark, and no new docs
+ assertEquals(cursorMark,
+ assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark)
+ ,"/response/numFound==8"
+ ,"/response/start==0"
+ ,"/response/docs==[]"
+ ));
+
// string sort with dups, id tie breaker
cursorMark = CURSOR_MARK_START;
params = params("q", "*:*",
@@ -300,7 +363,7 @@ public class CursorPagingTest extends So
params = params("q", "*:*",
"rows","2",
"fl", "id",
- "sort", "float asc, int desc, id desc");
+ "sort", "float asc, "+intsort+" desc, id desc");
cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark)
,"/response/numFound==10"
,"/response/start==0"
@@ -340,7 +403,7 @@ public class CursorPagingTest extends So
params = params("q", "id:3 id:7",
"rows","111",
"fl", "id",
- "sort", "int asc, id asc");
+ "sort", intsort + " asc, id asc");
cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark)
,"/response/numFound==2"
,"/response/start==0"
@@ -369,7 +432,7 @@ public class CursorPagingTest extends So
ids = assertFullWalkNoDups(9, params("q", "*:*",
"rows", "3",
"fq", "-id:6",
- "sort", "float desc, id asc, int asc"));
+ "sort", "float desc, id asc, "+intsort+" asc"));
assertEquals(9, ids.size());
assertFalse("matched on id:6 unexpectedly", ids.exists(6));
ids = assertFullWalkNoDups(9, params("q", "float:[0 TO *] int:7 id:6",
@@ -453,7 +516,7 @@ public class CursorPagingTest extends So
assertU(adoc("id", "3", "str", "a", "float", "11.1", "int", "3"));
assertU(commit());
- final Collection<String> allFieldNames = getAllFieldNames();
+ final Collection<String> allFieldNames = getAllSortFieldNames();
final SolrInfoMBean filterCacheStats
= h.getCore().getInfoRegistry().get("filterCache");
@@ -490,7 +553,7 @@ public class CursorPagingTest extends So
/** randomized testing of a non-trivial number of docs using assertFullWalkNoDups
*/
public void testRandomSortsOnLargeIndex() throws Exception {
- final Collection<String> allFieldNames = getAllFieldNames();
+ final Collection<String> allFieldNames = getAllSortFieldNames();
final int initialDocs = _TestUtil.nextInt(random(),100,200);
final int totalDocs = atLeast(5000);
@@ -557,16 +620,42 @@ public class CursorPagingTest extends So
}
/**
- * An immutable list of the fields in the schema (excluding _version_) in a
+ * An immutable list of the fields in the schema that can be used for sorting,
* deterministically random order.
*/
- private List<String> getAllFieldNames() {
+ private List<String> getAllSortFieldNames() {
+ return pruneAndDeterministicallySort
+ (h.getCore().getLatestSchema().getFields().keySet());
+ }
+
+
+ /**
+ * <p>
+ * Given a list of field names in the schema, returns an immutable list in
+ * deterministically random order with the following things removed:
+ * </p>
+ * <ul>
+ * <li><code>_version_</code> is removed</li>
+ * <li><code>dv_last</code> and <code>dv_first</code> fields are removed
+ * if the codec doesn't support them</li>
+ * </ul>
+ * @see #defaultCodecSupportsMissingDocValues
+ */
+ public static List<String> pruneAndDeterministicallySort(Collection<String> raw) {
+
+ final boolean prune_dv_missing = ! defaultCodecSupportsMissingDocValues();
+
ArrayList<String> names = new ArrayList<String>(37);
- for (String f : h.getCore().getLatestSchema().getFields().keySet()) {
- if (! f.equals("_version_")) {
- names.add(f);
+ for (String f : raw) {
+ if (f.equals("_version_")) {
+ continue;
+ }
+ if (prune_dv_missing && (f.endsWith("_dv_last") || f.endsWith("_dv_first")) ) {
+ continue;
}
+ names.add(f);
}
+
Collections.sort(names);
Collections.shuffle(names,random());
return Collections.<String>unmodifiableList(names);
@@ -630,9 +719,9 @@ public class CursorPagingTest extends So
}
assertU(commit());
- Collection<String> allFieldNames = getAllFieldNames();
+ Collection<String> allFieldNames = getAllSortFieldNames();
String[] fieldNames = new String[allFieldNames.size()];
- getAllFieldNames().toArray(fieldNames);
+ allFieldNames.toArray(fieldNames);
String f = fieldNames[_TestUtil.nextInt(random(), 0, fieldNames.length - 1)];
String order = 0 == _TestUtil.nextInt(random(), 0, 1) ? " asc" : " desc";
String sort = f + order + (f.equals("id") ? "" : ", id" + order);
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java?rev=1562160&r1=1562159&r2=1562160&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java Tue Jan 28 19:12:03 2014
@@ -151,7 +151,10 @@ public class DistribCursorPagingTest ext
String cursorMark = CURSOR_MARK_START;
SolrParams params = null;
QueryResponse rsp = null;
-
+
+ final String intsort = "int" + (random().nextBoolean() ? "" : "_dv");
+ final String intmissingsort = defaultCodecSupportsMissingDocValues() ? intsort : "int";
+
// trivial base case: ensure cursorMark against an empty index doesn't blow up
cursorMark = CURSOR_MARK_START;
params = params("q", "*:*",
@@ -175,7 +178,7 @@ public class DistribCursorPagingTest ext
indexDoc(sdoc("id", "6", "str", "a", "float", "64.5", "int", "7"));
indexDoc(sdoc("id", "1", "str", "a", "float", "64.5", "int", "7"));
indexDoc(sdoc("id", "4", "str", "a", "float", "11.1", "int", "6"));
- indexDoc(sdoc("id", "3", "str", "a", "float", "11.1", "int", "3"));
+ indexDoc(sdoc("id", "3", "str", "a", "float", "11.1")); // int is missing
commit();
// base case: ensure cursorMark that matches no docs doesn't blow up
@@ -250,7 +253,7 @@ public class DistribCursorPagingTest ext
"facet", "true",
"facet.field", "str",
"json.nl", "map",
- "sort", "int asc, id asc");
+ "sort", intsort + " asc, id asc");
rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark));
assertNumFound(8, rsp);
assertStartsAt(0, rsp);
@@ -284,6 +287,70 @@ public class DistribCursorPagingTest ext
assertEquals("no more docs, but cursorMark has changed",
cursorMark, assertHashNextCursorMark(rsp));
+ // int missing first sort with dups, id tie breaker
+ cursorMark = CURSOR_MARK_START;
+ params = params("q", "-int:2001 -int:4055",
+ "rows","3",
+ "fl", "id",
+ "json.nl", "map",
+ "sort", intmissingsort + "_first asc, id asc");
+ rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark));
+ assertNumFound(8, rsp);
+ assertStartsAt(0, rsp);
+ assertDocList(rsp, 3, 7, 0);
+ cursorMark = assertHashNextCursorMark(rsp);
+ //
+ rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark));
+ assertNumFound(8, rsp);
+ assertStartsAt(0, rsp);
+ assertDocList(rsp, 4, 1, 6);
+ cursorMark = assertHashNextCursorMark(rsp);
+ //
+ rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark));
+ assertNumFound(8, rsp);
+ assertStartsAt(0, rsp);
+ assertDocList(rsp, 9, 2);
+ cursorMark = assertHashNextCursorMark(rsp);
+ //
+ rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark));
+ assertNumFound(8, rsp);
+ assertStartsAt(0, rsp);
+ assertDocList(rsp);
+ assertEquals("no more docs, but cursorMark has changed",
+ cursorMark, assertHashNextCursorMark(rsp));
+
+ // int missing last sort with dups, id tie breaker
+ cursorMark = CURSOR_MARK_START;
+ params = params("q", "-int:2001 -int:4055",
+ "rows","3",
+ "fl", "id",
+ "json.nl", "map",
+ "sort", intmissingsort + "_last asc, id asc");
+ rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark));
+ assertNumFound(8, rsp);
+ assertStartsAt(0, rsp);
+ assertDocList(rsp, 7, 0, 4);
+ cursorMark = assertHashNextCursorMark(rsp);
+ //
+ rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark));
+ assertNumFound(8, rsp);
+ assertStartsAt(0, rsp);
+ assertDocList(rsp, 1, 6, 9);
+ cursorMark = assertHashNextCursorMark(rsp);
+ //
+ rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark));
+ assertNumFound(8, rsp);
+ assertStartsAt(0, rsp);
+ assertDocList(rsp, 2, 3);
+ cursorMark = assertHashNextCursorMark(rsp);
+ //
+ rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark));
+ assertNumFound(8, rsp);
+ assertStartsAt(0, rsp);
+ assertDocList(rsp);
+ assertEquals("no more docs, but cursorMark has changed",
+ cursorMark, assertHashNextCursorMark(rsp));
+
// string sort with dups, id tie breaker
cursorMark = CURSOR_MARK_START;
params = params("q", "*:*",
@@ -314,7 +381,7 @@ public class DistribCursorPagingTest ext
params = params("q", "*:*",
"rows","2",
"fl", "id",
- "sort", "float asc, int desc, id desc");
+ "sort", "float asc, "+intsort+" desc, id desc");
rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark));
assertNumFound(10, rsp);
assertStartsAt(0, rsp);
@@ -358,7 +425,7 @@ public class DistribCursorPagingTest ext
params = params("q", "id:3 id:7",
"rows","111",
"fl", "id",
- "sort", "int asc, id asc");
+ "sort", intsort + " asc, id asc");
rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark));
assertNumFound(2, rsp);
assertStartsAt(0, rsp);
@@ -451,7 +518,7 @@ public class DistribCursorPagingTest ext
/** randomized testing of a non-trivial number of docs using assertFullWalkNoDups
*/
public void doRandomSortsOnLargeIndex() throws Exception {
- final Collection<String> allFieldNames = getAllFieldNames();
+ final Collection<String> allFieldNames = getAllSortFieldNames();
final int numInitialDocs = _TestUtil.nextInt(random(),100,200);
final int totalDocs = atLeast(5000);
@@ -540,24 +607,20 @@ public class DistribCursorPagingTest ext
/**
* Asks the LukeRequestHandler on the control client for a list of the fields in the
- * schema (excluding _version_) and then returns the field names in a deterministically
- * random order.
+ * schema and then prunes that list down to just the fields that can be used for sorting,
+ * and returns them as an immutable list in a deterministically random order.
*/
- private List<String> getAllFieldNames() throws SolrServerException, IOException {
+ private List<String> getAllSortFieldNames() throws SolrServerException, IOException {
LukeRequest req = new LukeRequest("/admin/luke");
req.setShowSchema(true);
NamedList<Object> rsp = controlClient.request(req);
NamedList<Object> fields = (NamedList) ((NamedList)rsp.get("schema")).get("fields");
ArrayList<String> names = new ArrayList<String>(fields.size());
for (Map.Entry<String,Object> item : fields) {
- String f = item.getKey();
- if (! f.equals("_version_")) {
- names.add(item.getKey());
- }
+ names.add(item.getKey());
}
- Collections.sort(names);
- Collections.shuffle(names,random());
- return Collections.<String>unmodifiableList(names);
+
+ return CursorPagingTest.pruneAndDeterministicallySort(names);
}
/**