You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/02/15 19:00:23 UTC
[19/52] [abbrv] lucene-solr:jira/solr-9858: SOLR-10072: The test
TestSelectiveWeightCreation appears to be unreliable.
SOLR-10072: The test TestSelectiveWeightCreation appears to be unreliable.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/387c8e9c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/387c8e9c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/387c8e9c
Branch: refs/heads/jira/solr-9858
Commit: 387c8e9c2daff15bb1cba09aeffc5d9e5723cd4e
Parents: b5c7bd1
Author: markrmiller <ma...@apache.org>
Authored: Sun Feb 12 17:04:23 2017 -0500
Committer: markrmiller <ma...@apache.org>
Committed: Sun Feb 12 17:04:23 2017 -0500
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../featureExamples/external_features.json | 6 +-
.../modelExamples/external_model.json | 4 +-
.../modelExamples/external_model2.json | 13 +++
.../modelExamples/external_model_store.json | 13 ---
.../solr/ltr/TestParallelWeightCreation.java | 24 +++---
.../solr/ltr/TestSelectiveWeightCreation.java | 83 ++++++++++----------
.../solr/ltr/feature/TestExternalFeatures.java | 32 ++++----
8 files changed, 88 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/387c8e9c/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index ba6e574..4a8766e 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -195,6 +195,8 @@ Other Changes
* SOLR-9914: SimpleFacets: refactor "contains" check into "SubstringBytesRefFilter" class.
(Jonny Marks, David Smiley, Christine Poerschke)
+* SOLR-10072: The test TestSelectiveWeightCreation appears to be unreliable. (Michael Nilsson via Mark Miller)
+
================== 6.4.1 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/387c8e9c/solr/contrib/ltr/src/test-files/featureExamples/external_features.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/featureExamples/external_features.json b/solr/contrib/ltr/src/test-files/featureExamples/external_features.json
index 6c0cfa6..d8a9eca 100644
--- a/solr/contrib/ltr/src/test-files/featureExamples/external_features.json
+++ b/solr/contrib/ltr/src/test-files/featureExamples/external_features.json
@@ -5,11 +5,11 @@
"q" : "{!terms f=title}${user_query}"
}
}, {
- "name" : "confidence",
- "class" : "org.apache.solr.ltr.feature.ValueFeature",
+ "name" : "docPopularity",
+ "class" : "org.apache.solr.ltr.feature.FieldValueFeature",
"store": "fstore2",
"params" : {
- "value" : "${myconf}"
+ "field" : "popularity"
}
}, {
"name":"originalScore",
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/387c8e9c/solr/contrib/ltr/src/test-files/modelExamples/external_model.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/modelExamples/external_model.json b/solr/contrib/ltr/src/test-files/modelExamples/external_model.json
index 04ab229..68f659c 100644
--- a/solr/contrib/ltr/src/test-files/modelExamples/external_model.json
+++ b/solr/contrib/ltr/src/test-files/modelExamples/external_model.json
@@ -2,11 +2,11 @@
"class":"org.apache.solr.ltr.model.LinearModel",
"name":"externalmodel",
"features":[
- { "name": "matchedTitle"}
+ { "name": "titlePhraseMatch"}
],
"params":{
"weights": {
- "matchedTitle": 0.999
+ "titlePhraseMatch": 0.555
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/387c8e9c/solr/contrib/ltr/src/test-files/modelExamples/external_model2.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/modelExamples/external_model2.json b/solr/contrib/ltr/src/test-files/modelExamples/external_model2.json
new file mode 100644
index 0000000..f3d33f7
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/modelExamples/external_model2.json
@@ -0,0 +1,13 @@
+{
+ "class":"org.apache.solr.ltr.model.LinearModel",
+ "name":"externalmodel2",
+ "store": "fstore2",
+ "features":[
+ { "name": "docPopularity"}
+ ],
+ "params":{
+ "weights": {
+ "docPopularity": 0.5
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/387c8e9c/solr/contrib/ltr/src/test-files/modelExamples/external_model_store.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/modelExamples/external_model_store.json b/solr/contrib/ltr/src/test-files/modelExamples/external_model_store.json
deleted file mode 100644
index f8e6648..0000000
--- a/solr/contrib/ltr/src/test-files/modelExamples/external_model_store.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "class":"org.apache.solr.ltr.model.LinearModel",
- "name":"externalmodelstore",
- "store": "fstore2",
- "features":[
- { "name": "confidence"}
- ],
- "params":{
- "weights": {
- "confidence": 0.999
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/387c8e9c/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestParallelWeightCreation.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestParallelWeightCreation.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestParallelWeightCreation.java
index f4c21fd..630a68c 100644
--- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestParallelWeightCreation.java
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestParallelWeightCreation.java
@@ -25,21 +25,15 @@ public class TestParallelWeightCreation extends TestRerankBase{
public void testLTRScoringQueryParallelWeightCreationResultOrder() throws Exception {
setuptest("solrconfig-ltr_Th10_10.xml", "schema.xml");
- assertU(adoc("id", "1", "title", "w1 w3", "description", "w1", "popularity",
- "1"));
- assertU(adoc("id", "2", "title", "w2", "description", "w2", "popularity",
- "2"));
- assertU(adoc("id", "3", "title", "w3", "description", "w3", "popularity",
- "3"));
- assertU(adoc("id", "4", "title", "w4 w3", "description", "w4", "popularity",
- "4"));
- assertU(adoc("id", "5", "title", "w5", "description", "w5", "popularity",
- "5"));
+ assertU(adoc("id", "1", "title", "w1 w3", "description", "w1", "popularity", "1"));
+ assertU(adoc("id", "2", "title", "w2", "description", "w2", "popularity", "2"));
+ assertU(adoc("id", "3", "title", "w3", "description", "w3", "popularity", "3"));
+ assertU(adoc("id", "4", "title", "w3 w3", "description", "w4", "popularity", "4"));
+ assertU(adoc("id", "5", "title", "w5", "description", "w5", "popularity", "5"));
assertU(commit());
loadFeatures("external_features.json");
loadModels("external_model.json");
- loadModels("external_model_store.json");
// check to make sure that the order of results will be the same when using parallel weight creation
final SolrQuery query = new SolrQuery();
@@ -47,10 +41,10 @@ public class TestParallelWeightCreation extends TestRerankBase{
query.add("fl", "*,score");
query.add("rows", "4");
- query.add("rq", "{!ltr reRankDocs=4 model=externalmodel efi.user_query=w3}");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='3'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='4'");
+ query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='1'");
aftertest();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/387c8e9c/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java
index e44d4ac..5cfd999 100644
--- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java
@@ -106,21 +106,16 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
public static void before() throws Exception {
setuptest(false);
- assertU(adoc("id", "1", "title", "w1 w3", "description", "w1", "popularity",
- "1"));
- assertU(adoc("id", "2", "title", "w2", "description", "w2", "popularity",
- "2"));
- assertU(adoc("id", "3", "title", "w3", "description", "w3", "popularity",
- "3"));
- assertU(adoc("id", "4", "title", "w4 w3", "description", "w4", "popularity",
- "4"));
- assertU(adoc("id", "5", "title", "w5", "description", "w5", "popularity",
- "5"));
+ assertU(adoc("id", "1", "title", "w3 w1", "description", "w1", "popularity", "1"));
+ assertU(adoc("id", "2", "title", "w2", "description", "w2", "popularity", "2"));
+ assertU(adoc("id", "3", "title", "w3", "description", "w3", "popularity", "3"));
+ assertU(adoc("id", "4", "title", "w3 w3", "description", "w4", "popularity", "4"));
+ assertU(adoc("id", "5", "title", "w5", "description", "w5", "popularity", "5"));
assertU(commit());
loadFeatures("external_features.json");
loadModels("external_model.json");
- loadModels("external_model_store.json");
+ loadModels("external_model2.json");
}
@AfterClass
@@ -134,14 +129,14 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
Document doc = new Document();
- doc.add(newStringField("id", "0", Field.Store.YES));
+ doc.add(newStringField("id", "10", Field.Store.YES));
doc.add(newTextField("field", "wizard the the the the the oz",
Field.Store.NO));
doc.add(new FloatDocValuesField("final-score", 1.0f));
w.addDocument(doc);
doc = new Document();
- doc.add(newStringField("id", "1", Field.Store.YES));
+ doc.add(newStringField("id", "11", Field.Store.YES));
// 1 extra token, but wizard and oz are close;
doc.add(newTextField("field", "wizard oz the the the the the the",
Field.Store.NO));
@@ -159,8 +154,8 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
// first run the standard query
final TopDocs hits = searcher.search(bqBuilder.build(), 10);
assertEquals(2, hits.totalHits);
- assertEquals("0", searcher.doc(hits.scoreDocs[0].doc).get("id"));
- assertEquals("1", searcher.doc(hits.scoreDocs[1].doc).get("id"));
+ assertEquals("10", searcher.doc(hits.scoreDocs[0].doc).get("id"));
+ assertEquals("11", searcher.doc(hits.scoreDocs[1].doc).get("id"));
List<Feature> features = makeFeatures(new int[] {0, 1, 2});
final List<Feature> allFeatures = makeFeatures(new int[] {0, 1, 2, 3, 4, 5,
@@ -206,7 +201,7 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
}
assertEquals(validFeatures, allFeatures.size());
- assertU(delI("0"));assertU(delI("1"));
+ assertU(delI("10"));assertU(delI("11"));
r.close();
dir.close();
}
@@ -216,41 +211,49 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
public void testSelectiveWeightsRequestFeaturesFromDifferentStore() throws Exception {
final String docs0fv = FeatureLoggerTestUtils.toFeatureVector(
- "matchedTitle","1.0", "titlePhraseMatch","0.40254828");
+ "matchedTitle","1.0", "titlePhraseMatch","0.6103343");
final String docs0fv_fstore4= FeatureLoggerTestUtils.toFeatureVector(
"popularity","3.0", "originalScore","1.0");
+ // extract all features in externalmodel's store (default store)
+ // rerank using externalmodel (default store)
final SolrQuery query = new SolrQuery();
query.setQuery("*:*");
- query.add("fl", "*,score");
- query.add("rows", "4");
- query.add("rq", "{!ltr reRankDocs=4 model=externalmodel efi.user_query=w3}");
- query.add("fl", "fv:[fv]");
-
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='3'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='4'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv+"'"); // extract all features in default store
-
+ query.add("fl", "*,score,fv:[fv]");
+ query.add("rows", "5");
+ query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}");
+
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='1'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv+"'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.33873552");
+
+ // extract all features from fstore4
+ // rerank using externalmodel (default store)
query.remove("fl");
query.remove("rq");
- query.add("fl", "*,score");
- query.add("rq", "{!ltr reRankDocs=4 model=externalmodel efi.user_query=w3}");
- query.add("fl", "fv:[fv store=fstore4 efi.myPop=3]");
-
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.999");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'"); // extract all features from fstore4
+ query.add("fl", "*,score,fv:[fv store=fstore4 efi.myPop=3]");
+ query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='1'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.33873552");
+ // extract all features from fstore4
+ // rerank using externalmodel2 (fstore2)
query.remove("fl");
query.remove("rq");
- query.add("fl", "*,score");
- query.add("rq", "{!ltr reRankDocs=4 model=externalmodelstore efi.user_query=w3 efi.myconf=0.8}");
- query.add("fl", "fv:[fv store=fstore4 efi.myPop=3]");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'"); // score using fstore2 used by externalmodelstore
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.7992");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'"); // extract all features from fstore4
+ query.add("fl", "*,score,fv:[fv store=fstore4 efi.myPop=3]");
+ query.add("rq", "{!ltr reRankDocs=10 model=externalmodel2 efi.user_query=w3}");
+
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='5'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='3'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==2.5");
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/387c8e9c/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java
index 10ababb..4010ee1 100644
--- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java
@@ -65,24 +65,24 @@ public class TestExternalFeatures extends TestRerankBase {
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='3'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/score==1.0");
- query.add("fl", "[fv]");
- query.add("rq", "{!ltr reRankDocs=3 model=externalmodel efi.user_query=w3}");
+ query.remove("fl");
+ query.add("fl", "*,score,[fv]");
+ query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.999");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='1'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.7693934");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/score==0.0");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='2'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/score==0.0");
// Adding an efi in the transformer should not affect the rq ranking with a
// different value for efi of the same parameter
query.remove("fl");
- query.add("fl", "id,[fv efi.user_query=w2]");
+ query.add("fl", "*,score,[fv efi.user_query=w2]");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='1'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='2'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.7693934");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/score==0.0");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/score==0.0");
}
@Test
@@ -92,7 +92,7 @@ public class TestExternalFeatures extends TestRerankBase {
query.add("fl", "*,score,fv:[fv]");
query.add("rows", "1");
// Stopword only query passed in
- query.add("rq", "{!ltr reRankDocs=3 model=externalmodel efi.user_query='a'}");
+ query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query='a'}");
final String docs0fv_dense_csv = FeatureLoggerTestUtils.toFeatureVector(
"matchedTitle","0.0",
@@ -112,7 +112,7 @@ public class TestExternalFeatures extends TestRerankBase {
query.add("rows", "1");
final String docs0fv_csv = FeatureLoggerTestUtils.toFeatureVector(
- "confidence","2.3", "originalScore","1.0");
+ "occurrences","2.3", "originalScore","1.0");
// Features we're extracting depend on external feature info not passed in
query.add("fl", "[fv]");
@@ -120,13 +120,13 @@ public class TestExternalFeatures extends TestRerankBase {
// Adding efi in features section should make it work
query.remove("fl");
- query.add("fl", "score,fvalias:[fv store=fstore2 efi.myconf=2.3]");
+ query.add("fl", "score,fvalias:[fv store=fstore3 efi.myOcc=2.3]");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='"+docs0fv_csv+"'");
- // Adding efi in transformer + rq should still use the transformer's params for feature extraction
+ // Adding efi in transformer + rq should still returns features
query.remove("fl");
- query.add("fl", "score,fvalias:[fv store=fstore2 efi.myconf=2.3]");
- query.add("rq", "{!ltr reRankDocs=3 model=externalmodel efi.user_query=w3}");
+ query.add("fl", "score,fvalias:[fv store=fstore3 efi.myOcc=2.3]");
+ query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='"+docs0fv_csv+"'");
}
@@ -137,7 +137,7 @@ public class TestExternalFeatures extends TestRerankBase {
query.add("rows", "1");
final String docs0fvalias_dense_csv = FeatureLoggerTestUtils.toFeatureVector(
- "confidence","0.0",
+ "occurrences","0.0",
"originalScore","0.0");
final String docs0fvalias_sparse_csv = FeatureLoggerTestUtils.toFeatureVector(
"originalScore","0.0");
@@ -146,7 +146,7 @@ public class TestExternalFeatures extends TestRerankBase {
// Efi is explicitly not required, so we do not score the feature
query.remove("fl");
- query.add("fl", "fvalias:[fv store=fstore2]");
+ query.add("fl", "fvalias:[fv store=fstore3]");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='"+docs0fvalias_default_csv+"'");
}