You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ho...@apache.org on 2014/10/16 21:05:21 UTC
svn commit: r1632414 - in /lucene/dev/trunk: lucene/
lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/
lucene/queries/src/test/org/apache/lucene/queries/function/ solr/
solr/core/src/test/org/apache/solr/handler/component/ solr/co...
Author: hossman
Date: Thu Oct 16 19:05:20 2014
New Revision: 1632414
URL: http://svn.apache.org/r1632414
Log:
LUCENE-5961: Fix the exists() method for FunctionValues returned by many ValueSoures to behave properly when wrapping other ValueSources which do not exist for the specified document
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/MIGRATE.txt
lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DualFloatFunction.java
lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java
lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LinearFloatFunction.java
lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java
lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java
lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFunction.java
lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java
lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReciprocalFloatFunction.java
lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ScaleFloatFunction.java
lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1632414&r1=1632413&r2=1632414&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Thu Oct 16 19:05:20 2014
@@ -184,6 +184,10 @@ Bug Fixes
* LUCENE-5980: Don't let document length overflow. (Robert Muir)
+* LUCENE-5961: Fix the exists() method for FunctionValues returned by many ValueSoures to
+ behave properly when wrapping other ValueSources which do not exist for the specified document
+ (hossman)
+
Documentation
* LUCENE-5392: Add/improve analysis package documentation to reflect
Modified: lucene/dev/trunk/lucene/MIGRATE.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/MIGRATE.txt?rev=1632414&r1=1632413&r2=1632414&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/MIGRATE.txt (original)
+++ lucene/dev/trunk/lucene/MIGRATE.txt Thu Oct 16 19:05:20 2014
@@ -6,3 +6,11 @@ The API of oal.document was restructured
documents and indexed documents. IndexReader.document(int) now returns
StoredDocument instead of Document. In most cases a simple replacement
of the return type is enough to upgrade.
+
+## FunctionValues.exist() Behavior Changes due to ValueSource bug fixes (LUCENE-5961)
+
+Bugs fixed in several ValueSource functions may result in different behavior in
+situations where some documents do not have values for fields wrapped in other
+ValueSources. Users who want to preserve the previous behavior may need to wrap
+their ValueSources in a "DefFunction" along with a ConstValueSource of "0.0".
+
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DualFloatFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DualFloatFunction.java?rev=1632414&r1=1632413&r2=1632414&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DualFloatFunction.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DualFloatFunction.java Thu Oct 16 19:05:20 2014
@@ -60,7 +60,14 @@ public abstract class DualFloatFunction
public float floatVal(int doc) {
return func(doc, aVals, bVals);
}
-
+ /**
+ * True if and only if <em>all</em> of the wrapped {@link FunctionValues}
+ * <code>exists</code> for the specified doc
+ */
+ @Override
+ public boolean exists(int doc) {
+ return MultiFunction.allExists(doc, aVals, bVals);
+ }
@Override
public String toString(int doc) {
return name() + '(' + aVals.toString(doc) + ',' + bVals.toString(doc) + ')';
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java?rev=1632414&r1=1632413&r2=1632414&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java Thu Oct 16 19:05:20 2014
@@ -102,7 +102,7 @@ public class IfFunction extends BoolFunc
@Override
public boolean exists(int doc) {
- return true; // TODO: flow through to any sub-sources?
+ return ifVals.boolVal(doc) ? trueVals.exists(doc) : falseVals.exists(doc);
}
@Override
@@ -148,4 +148,4 @@ public class IfFunction extends BoolFunc
trueSource.createWeight(context, searcher);
falseSource.createWeight(context, searcher);
}
-}
\ No newline at end of file
+}
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LinearFloatFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LinearFloatFunction.java?rev=1632414&r1=1632413&r2=1632414&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LinearFloatFunction.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LinearFloatFunction.java Thu Oct 16 19:05:20 2014
@@ -59,6 +59,10 @@ public class LinearFloatFunction extends
return vals.floatVal(doc) * slope + intercept;
}
@Override
+ public boolean exists(int doc) {
+ return vals.exists(doc);
+ }
+ @Override
public String toString(int doc) {
return slope + "*float(" + vals.toString(doc) + ")+" + intercept;
}
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java?rev=1632414&r1=1632413&r2=1632414&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java Thu Oct 16 19:05:20 2014
@@ -35,11 +35,25 @@ public class MaxFloatFunction extends Mu
@Override
protected float func(int doc, FunctionValues[] valsArr) {
- if (valsArr.length == 0) return 0.0f;
+ if ( ! exists(doc, valsArr) ) return 0.0f;
+
float val = Float.NEGATIVE_INFINITY;
for (FunctionValues vals : valsArr) {
- val = Math.max(vals.floatVal(doc), val);
+ if (vals.exists(doc)) {
+ val = Math.max(vals.floatVal(doc), val);
+ }
}
return val;
}
+
+ /**
+ * True if <em>any</em> of the specified <code>values</code>
+ * {@link FunctionValues#exists} for the specified doc, else false.
+ *
+ * @see MultiFunction#anyExists
+ */
+ @Override
+ protected boolean exists(int doc, FunctionValues[] valsArr) {
+ return MultiFunction.anyExists(doc, valsArr);
+ }
}
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java?rev=1632414&r1=1632413&r2=1632414&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java Thu Oct 16 19:05:20 2014
@@ -35,11 +35,26 @@ public class MinFloatFunction extends Mu
@Override
protected float func(int doc, FunctionValues[] valsArr) {
- if (valsArr.length == 0) return 0.0f;
+ if ( ! exists(doc, valsArr) ) return 0.0f;
+
float val = Float.POSITIVE_INFINITY;
for (FunctionValues vals : valsArr) {
- val = Math.min(vals.floatVal(doc), val);
+ if (vals.exists(doc)) {
+ val = Math.min(vals.floatVal(doc), val);
+ }
}
return val;
}
+
+ /**
+ * True if <em>any</em> of the specified <code>values</code>
+ * {@link FunctionValues#exists} for the specified doc, else false.
+ *
+ * @see MultiFunction#anyExists
+ */
+ @Override
+ protected boolean exists(int doc, FunctionValues[] valsArr) {
+ return MultiFunction.anyExists(doc, valsArr);
+ }
+
}
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFunction.java?rev=1632414&r1=1632413&r2=1632414&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFunction.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFunction.java Thu Oct 16 19:05:20 2014
@@ -40,6 +40,18 @@ public abstract class MultiFloatFunction
abstract protected String name();
abstract protected float func(int doc, FunctionValues[] valsArr);
+ /**
+ * Called by {@link FunctionValues#exists} for each document.
+ *
+ * Default impl returns true if <em>all</em> of the specified <code>values</code>
+ * {@link FunctionValues#exists} for the specified doc, else false.
+ *
+ * @see FunctionValues#exists
+ * @see MultiFunction#allExists
+ */
+ protected boolean exists(int doc, FunctionValues[] valsArr) {
+ return MultiFunction.allExists(doc, valsArr);
+ }
@Override
public String description() {
@@ -70,21 +82,12 @@ public abstract class MultiFloatFunction
public float floatVal(int doc) {
return func(doc, valsArr);
}
- @Override
+ public boolean exists(int doc) {
+ return MultiFloatFunction.this.exists(doc, valsArr);
+ }
+ @Override
public String toString(int doc) {
- StringBuilder sb = new StringBuilder();
- sb.append(name()).append('(');
- boolean firstTime=true;
- for (FunctionValues vals : valsArr) {
- if (firstTime) {
- firstTime=false;
- } else {
- sb.append(',');
- }
- sb.append(vals.toString(doc));
- }
- sb.append(')');
- return sb.toString();
+ return MultiFunction.toString(name(), valsArr, doc);
}
};
}
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java?rev=1632414&r1=1632413&r2=1632414&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java Thu Oct 16 19:05:20 2014
@@ -43,6 +43,35 @@ public abstract class MultiFunction exte
return description(name(), sources);
}
+ /**
+ * Helper utility for {@link FunctionValues} wrapping multiple {@link FunctionValues}
+ *
+ * @return true if <em>all</em> of the specified <code>values</code>
+ * {@link FunctionValues#exists} for the specified doc, else false.
+ */
+ public static boolean allExists(int doc, FunctionValues... values) {
+ for (FunctionValues v : values) {
+ if ( ! v.exists(doc) ) {
+ return false;
+ }
+ }
+ return true;
+ }
+ /**
+ * Helper utility for {@link FunctionValues} wrapping multiple {@link FunctionValues}
+ *
+ * @return true if <em>any</em> of the specified <code>values</code>
+ * {@link FunctionValues#exists} for the specified doc, else false.
+ */
+ public static boolean anyExists(int doc, FunctionValues... values) {
+ for (FunctionValues v : values) {
+ if ( v.exists(doc) ) {
+ return true;
+ }
+ }
+ return false;
+ }
+
public static String description(String name, List<ValueSource> sources) {
StringBuilder sb = new StringBuilder();
sb.append(name).append('(');
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReciprocalFloatFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReciprocalFloatFunction.java?rev=1632414&r1=1632413&r2=1632414&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReciprocalFloatFunction.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReciprocalFloatFunction.java Thu Oct 16 19:05:20 2014
@@ -69,6 +69,10 @@ public class ReciprocalFloatFunction ext
return a/(m*vals.floatVal(doc) + b);
}
@Override
+ public boolean exists(int doc) {
+ return vals.exists(doc);
+ }
+ @Override
public String toString(int doc) {
return Float.toString(a) + "/("
+ m + "*float(" + vals.toString(doc) + ')'
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ScaleFloatFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ScaleFloatFunction.java?rev=1632414&r1=1632413&r2=1632414&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ScaleFloatFunction.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ScaleFloatFunction.java Thu Oct 16 19:05:20 2014
@@ -70,21 +70,23 @@ public class ScaleFloatFunction extends
int maxDoc = leaf.reader().maxDoc();
FunctionValues vals = source.getValues(context, leaf);
for (int i=0; i<maxDoc; i++) {
-
- float val = vals.floatVal(i);
- if ((Float.floatToRawIntBits(val) & (0xff<<23)) == 0xff<<23) {
- // if the exponent in the float is all ones, then this is +Inf, -Inf or NaN
- // which don't make sense to factor into the scale function
- continue;
- }
- if (val < minVal) {
- minVal = val;
- }
- if (val > maxVal) {
- maxVal = val;
+ if ( ! vals.exists(i) ) {
+ continue;
+ }
+ float val = vals.floatVal(i);
+ if ((Float.floatToRawIntBits(val) & (0xff<<23)) == 0xff<<23) {
+ // if the exponent in the float is all ones, then this is +Inf, -Inf or NaN
+ // which don't make sense to factor into the scale function
+ continue;
+ }
+ if (val < minVal) {
+ minVal = val;
+ }
+ if (val > maxVal) {
+ maxVal = val;
+ }
}
}
- }
if (minVal == Float.POSITIVE_INFINITY) {
// must have been an empty index
@@ -114,6 +116,10 @@ public class ScaleFloatFunction extends
return new FloatDocValues(this) {
@Override
+ public boolean exists(int doc) {
+ return vals.exists(doc);
+ }
+ @Override
public float floatVal(int doc) {
return (vals.floatVal(doc) - minSource) * scale + min;
}
Modified: lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java?rev=1632414&r1=1632413&r2=1632414&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java (original)
+++ lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java Thu Oct 16 19:05:20 2014
@@ -19,6 +19,8 @@ package org.apache.lucene.queries.functi
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
+import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -31,9 +33,12 @@ import org.apache.lucene.document.Numeri
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.queries.function.docvalues.FloatDocValues;
import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource;
import org.apache.lucene.queries.function.valuesource.ConstValueSource;
import org.apache.lucene.queries.function.valuesource.DivFloatFunction;
@@ -51,6 +56,7 @@ import org.apache.lucene.queries.functio
import org.apache.lucene.queries.function.valuesource.MaxDocValueSource;
import org.apache.lucene.queries.function.valuesource.MaxFloatFunction;
import org.apache.lucene.queries.function.valuesource.MinFloatFunction;
+import org.apache.lucene.queries.function.valuesource.MultiFloatFunction;
import org.apache.lucene.queries.function.valuesource.NormValueSource;
import org.apache.lucene.queries.function.valuesource.NumDocsValueSource;
import org.apache.lucene.queries.function.valuesource.PowFloatFunction;
@@ -71,6 +77,7 @@ import org.apache.lucene.search.ScoreDoc
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
@@ -88,6 +95,11 @@ public class TestValueSources extends Lu
static IndexReader reader;
static IndexSearcher searcher;
+ static final ValueSource BOGUS_FLOAT_VS = new FloatFieldSource("bogus_field");
+ static final ValueSource BOGUS_DOUBLE_VS = new DoubleFieldSource("bogus_field");
+ static final ValueSource BOGUS_INT_VS = new IntFieldSource("bogus_field");
+ static final ValueSource BOGUS_LONG_VS = new LongFieldSource("bogus_field");
+
static final List<String[]> documents = Arrays.asList(new String[][] {
/* id, double, float, int, long, string, text */
new String[] { "0", "3.63", "5.2", "35", "4343", "test", "this is a test test test" },
@@ -160,100 +172,165 @@ public class TestValueSources extends Lu
}
public void testConst() throws Exception {
- assertHits(new FunctionQuery(new ConstValueSource(0.3f)),
- new float[] { 0.3f, 0.3f });
+ ValueSource vs = new ConstValueSource(0.3f);
+ assertHits(new FunctionQuery(vs),
+ new float[] { 0.3f, 0.3f });
+ assertAllExist(vs);
}
public void testDiv() throws Exception {
- assertHits(new FunctionQuery(new DivFloatFunction(
- new ConstValueSource(10f), new ConstValueSource(5f))),
- new float[] { 2f, 2f });
+ ValueSource vs = new DivFloatFunction(new ConstValueSource(10f), new ConstValueSource(5f));
+ assertHits(new FunctionQuery(vs),
+ new float[] { 2f, 2f });
+ assertAllExist(vs);
+ vs = new DivFloatFunction(new ConstValueSource(10f), BOGUS_FLOAT_VS);
+ assertNoneExist(vs);
+ vs = new DivFloatFunction(BOGUS_FLOAT_VS, new ConstValueSource(10f));
+ assertNoneExist(vs);
}
public void testDocFreq() throws Exception {
- assertHits(new FunctionQuery(
- new DocFreqValueSource("bogus", "bogus", "text", new BytesRef("test"))),
- new float[] { 2f, 2f });
+ ValueSource vs = new DocFreqValueSource("bogus", "bogus", "text", new BytesRef("test"));
+ assertHits(new FunctionQuery(vs), new float[] { 2f, 2f });
+ assertAllExist(vs);
}
public void testDoubleConst() throws Exception {
- assertHits(new FunctionQuery(new DoubleConstValueSource(0.3d)),
- new float[] { 0.3f, 0.3f });
+ ValueSource vs = new DoubleConstValueSource(0.3d);
+ assertHits(new FunctionQuery(vs), new float[] { 0.3f, 0.3f });
+ assertAllExist(vs);
}
public void testDouble() throws Exception {
- assertHits(new FunctionQuery(new DoubleFieldSource("double")),
- new float[] { 3.63f, 5.65f });
+ ValueSource vs = new DoubleFieldSource("double");
+ assertHits(new FunctionQuery(vs), new float[] { 3.63f, 5.65f });
+ assertAllExist(vs);
+ assertNoneExist(BOGUS_DOUBLE_VS);
}
public void testFloat() throws Exception {
- assertHits(new FunctionQuery(new FloatFieldSource("float")),
- new float[] { 5.2f, 9.3f });
+ ValueSource vs = new FloatFieldSource("float");
+ assertHits(new FunctionQuery(vs), new float[] { 5.2f, 9.3f });
+ assertAllExist(vs);
+ assertNoneExist(BOGUS_FLOAT_VS);
}
public void testIDF() throws Exception {
Similarity saved = searcher.getSimilarity();
try {
searcher.setSimilarity(new DefaultSimilarity());
- assertHits(new FunctionQuery(
- new IDFValueSource("bogus", "bogus", "text", new BytesRef("test"))),
- new float[] { 0.5945349f, 0.5945349f });
+ ValueSource vs = new IDFValueSource("bogus", "bogus", "text", new BytesRef("test"));
+ assertHits(new FunctionQuery(vs), new float[] { 0.5945349f, 0.5945349f });
+ assertAllExist(vs);
} finally {
searcher.setSimilarity(saved);
}
}
public void testIf() throws Exception {
- assertHits(new FunctionQuery(new IfFunction(
- new BytesRefFieldSource("id"),
- new ConstValueSource(1.0f),
- new ConstValueSource(2.0f)
- )),
- new float[] { 1f, 1f });
- // true just if a value exists...
- assertHits(new FunctionQuery(new IfFunction(
- new LiteralValueSource("false"),
- new ConstValueSource(1.0f),
- new ConstValueSource(2.0f)
- )),
- new float[] { 1f, 1f });
+ ValueSource vs = new IfFunction(new BytesRefFieldSource("id"),
+ new ConstValueSource(1.0f), // match
+ new ConstValueSource(2.0f));
+ assertHits(new FunctionQuery(vs), new float[] { 1f, 1f });
+ assertAllExist(vs);
+
+ // true just if a test value exists...
+ vs = new IfFunction(new LiteralValueSource("false"),
+ new ConstValueSource(1.0f), // match
+ new ConstValueSource(2.0f));
+ assertHits(new FunctionQuery(vs), new float[] { 1f, 1f });
+ assertAllExist(vs);
+
+ // false value if tests value does not exist
+ vs = new IfFunction(BOGUS_FLOAT_VS,
+ new ConstValueSource(1.0f),
+ new ConstValueSource(2.0f)); // match
+ assertHits(new FunctionQuery(vs), new float[] { 2F, 2F });
+ assertAllExist(vs);
+
+ // final value may still not exist
+ vs = new IfFunction(new BytesRefFieldSource("id"),
+ BOGUS_FLOAT_VS, // match
+ new ConstValueSource(1.0f));
+ assertNoneExist(vs);
}
public void testInt() throws Exception {
- assertHits(new FunctionQuery(new IntFieldSource("int")),
- new float[] { 35f, 54f });
+ ValueSource vs = new IntFieldSource("int");
+ assertHits(new FunctionQuery(vs), new float[] { 35f, 54f });
+ assertAllExist(vs);
+ assertNoneExist(BOGUS_INT_VS);
+
}
public void testJoinDocFreq() throws Exception {
- assertHits(new FunctionQuery(new JoinDocFreqValueSource("string", "text")),
- new float[] { 2f, 0f });
+ assertHits(new FunctionQuery(new JoinDocFreqValueSource("string", "text")),
+ new float[] { 2f, 0f });
+
+ // TODO: what *should* the rules be for exist() ?
}
public void testLinearFloat() throws Exception {
- assertHits(new FunctionQuery(new LinearFloatFunction(new ConstValueSource(2.0f), 3, 1)),
- new float[] { 7f, 7f });
+ ValueSource vs = new LinearFloatFunction(new ConstValueSource(2.0f), 3, 1);
+ assertHits(new FunctionQuery(vs), new float[] { 7f, 7f });
+ assertAllExist(vs);
+ vs = new LinearFloatFunction(BOGUS_FLOAT_VS, 3, 1);
+ assertNoneExist(vs);
}
public void testLong() throws Exception {
- assertHits(new FunctionQuery(new LongFieldSource("long")),
- new float[] { 4343f, 1954f });
+ ValueSource vs = new LongFieldSource("long");
+ assertHits(new FunctionQuery(vs), new float[] { 4343f, 1954f });
+ assertAllExist(vs);
+ assertNoneExist(BOGUS_LONG_VS);
}
public void testMaxDoc() throws Exception {
- assertHits(new FunctionQuery(new MaxDocValueSource()),
- new float[] { 2f, 2f });
+ ValueSource vs = new MaxDocValueSource();
+ assertHits(new FunctionQuery(vs), new float[] { 2f, 2f });
+ assertAllExist(vs);
}
public void testMaxFloat() throws Exception {
- assertHits(new FunctionQuery(new MaxFloatFunction(new ValueSource[] {
- new ConstValueSource(1f), new ConstValueSource(2f)})),
- new float[] { 2f, 2f });
+ ValueSource vs = new MaxFloatFunction(new ValueSource[] {
+ new ConstValueSource(1f), new ConstValueSource(2f)});
+
+ assertHits(new FunctionQuery(vs), new float[] { 2f, 2f });
+ assertAllExist(vs);
+
+ // as long as one value exists, then max exists
+ vs = new MaxFloatFunction(new ValueSource[] {
+ BOGUS_FLOAT_VS, new ConstValueSource(2F)});
+ assertAllExist(vs);
+ vs = new MaxFloatFunction(new ValueSource[] {
+ BOGUS_FLOAT_VS, new ConstValueSource(2F), BOGUS_DOUBLE_VS});
+ assertAllExist(vs);
+ // if none exist, then max doesn't exist
+ vs = new MaxFloatFunction(new ValueSource[] {
+ BOGUS_FLOAT_VS, BOGUS_INT_VS, BOGUS_DOUBLE_VS});
+ assertNoneExist(vs);
}
public void testMinFloat() throws Exception {
- assertHits(new FunctionQuery(new MinFloatFunction(new ValueSource[] {
- new ConstValueSource(1f), new ConstValueSource(2f)})),
- new float[] { 1f, 1f });
+ ValueSource vs = new MinFloatFunction(new ValueSource[] {
+ new ConstValueSource(1f), new ConstValueSource(2f)});
+
+ assertHits(new FunctionQuery(vs), new float[] { 1f, 1f });
+ assertAllExist(vs);
+
+ // as long as one value exists, then min exists
+ vs = new MinFloatFunction(new ValueSource[] {
+ BOGUS_FLOAT_VS, new ConstValueSource(2F)});
+ assertHits(new FunctionQuery(vs), new float[] { 2F, 2F });
+ assertAllExist(vs);
+ vs = new MinFloatFunction(new ValueSource[] {
+ BOGUS_FLOAT_VS, new ConstValueSource(2F), BOGUS_DOUBLE_VS});
+ assertAllExist(vs);
+
+ // if none exist, then min doesn't exist
+ vs = new MinFloatFunction(new ValueSource[] {
+ BOGUS_FLOAT_VS, BOGUS_INT_VS, BOGUS_DOUBLE_VS});
+ assertNoneExist(vs);
}
public void testNorm() throws Exception {
@@ -261,35 +338,95 @@ public class TestValueSources extends Lu
try {
// no norm field (so agnostic to indexed similarity)
searcher.setSimilarity(new DefaultSimilarity());
- assertHits(new FunctionQuery(
- new NormValueSource("byte")),
- new float[] { 0f, 0f });
+ ValueSource vs = new NormValueSource("byte");
+ assertHits(new FunctionQuery(vs), new float[] { 0f, 0f });
+
+ // regardless of wether norms exist, value source exists == 0
+ assertAllExist(vs);
+
+ vs = new NormValueSource("text");
+ assertAllExist(vs);
+
} finally {
searcher.setSimilarity(saved);
}
}
public void testNumDocs() throws Exception {
- assertHits(new FunctionQuery(new NumDocsValueSource()),
- new float[] { 2f, 2f });
+ ValueSource vs = new NumDocsValueSource();
+ assertHits(new FunctionQuery(vs), new float[] { 2f, 2f });
+ assertAllExist(vs);
}
public void testPow() throws Exception {
- assertHits(new FunctionQuery(new PowFloatFunction(
- new ConstValueSource(2f), new ConstValueSource(3f))),
- new float[] { 8f, 8f });
+ ValueSource vs = new PowFloatFunction(new ConstValueSource(2f), new ConstValueSource(3f));
+ assertHits(new FunctionQuery(vs), new float[] { 8f, 8f });
+ assertAllExist(vs);
+ vs = new PowFloatFunction(BOGUS_FLOAT_VS, new ConstValueSource(3f));
+ assertNoneExist(vs);
+ vs = new PowFloatFunction(new ConstValueSource(3f), BOGUS_FLOAT_VS);
+ assertNoneExist(vs);
}
public void testProduct() throws Exception {
- assertHits(new FunctionQuery(new ProductFloatFunction(new ValueSource[] {
- new ConstValueSource(2f), new ConstValueSource(3f)})),
- new float[] { 6f, 6f });
+ ValueSource vs = new ProductFloatFunction(new ValueSource[] {
+ new ConstValueSource(2f), new ConstValueSource(3f)});
+ assertHits(new FunctionQuery(vs), new float[] { 6f, 6f });
+ assertAllExist(vs);
+
+ vs = new ProductFloatFunction(new ValueSource[] {
+ BOGUS_FLOAT_VS, new ConstValueSource(3f)});
+ assertNoneExist(vs);
}
+ public void testQueryWrapedFuncWrapedQuery() throws Exception {
+ ValueSource vs = new QueryValueSource(new FunctionQuery(new ConstValueSource(2f)), 0f);
+ assertHits(new FunctionQuery(vs), new float[] { 2f, 2f });
+ assertAllExist(vs);
+ }
+
public void testQuery() throws Exception {
- assertHits(new FunctionQuery(new QueryValueSource(
- new FunctionQuery(new ConstValueSource(2f)), 0f)),
- new float[] { 2f, 2f });
+ Similarity saved = searcher.getSimilarity();
+
+ try {
+ searcher.setSimilarity(new DefaultSimilarity());
+
+ ValueSource vs = new QueryValueSource(new TermQuery(new Term("string","bar")), 42F);
+ assertHits(new FunctionQuery(vs), new float[] { 42F, 1F });
+
+ // valuesource should exist only for things matching the term query
+ // sanity check via quick & dirty wrapper arround tf
+ ValueSource expected = new MultiFloatFunction(new ValueSource[] {
+ new TFValueSource("bogus", "bogus", "string", new BytesRef("bar"))}) {
+
+ @Override
+ protected String name() { return "tf_based_exists"; }
+
+ @Override
+ protected float func(int doc, FunctionValues[] valsArr) {
+ return valsArr[0].floatVal(doc);
+ }
+ @Override
+ protected boolean exists(int doc, FunctionValues[] valsArr) {
+ // if tf > 0, then it should exist
+ return 0 < func(doc, valsArr);
+ }
+ };
+
+ assertExists(expected, vs);
+
+
+ // Query matches all docs, func exists for all docs
+ vs = new QueryValueSource(new TermQuery(new Term("text","test")), 0F);
+ assertAllExist(vs);
+
+ // Query matches no docs, func exists for no docs
+ vs = new QueryValueSource(new TermQuery(new Term("bogus","does not exist")), 0F);
+ assertNoneExist(vs);
+
+ } finally {
+ searcher.setSimilarity(saved);
+ }
}
public void testRangeMap() throws Exception {
@@ -300,37 +437,59 @@ public class TestValueSources extends Lu
5, 6, new SumFloatFunction(new ValueSource[] {new ConstValueSource(1f), new ConstValueSource(2f)}),
new ConstValueSource(11f))),
new float[] { 3f, 11f });
+
+ // TODO: what *should* the rules be for exist() ?
+ // ((source exists && source in range && target exists) OR (source not in range && default exists)) ?
}
public void testReciprocal() throws Exception {
- assertHits(new FunctionQuery(new ReciprocalFloatFunction(new ConstValueSource(2f),
- 3, 1, 4)),
- new float[] { 0.1f, 0.1f });
+ ValueSource vs = new ReciprocalFloatFunction(new ConstValueSource(2f), 3, 1, 4);
+ assertHits(new FunctionQuery(vs), new float[] { 0.1f, 0.1f });
+ assertAllExist(vs);
+
+ vs = new ReciprocalFloatFunction(BOGUS_FLOAT_VS, 3, 1, 4);
+ assertNoneExist(vs);
}
public void testScale() throws Exception {
- assertHits(new FunctionQuery(new ScaleFloatFunction(new IntFieldSource("int"), 0, 1)),
- new float[] { 0.0f, 1.0f });
+ ValueSource vs = new ScaleFloatFunction(new IntFieldSource("int"), 0, 1);
+ assertHits(new FunctionQuery(vs), new float[] { 0.0f, 1.0f });
+ assertAllExist(vs);
+
+ vs = new ScaleFloatFunction(BOGUS_INT_VS, 0, 1);
+ assertNoneExist(vs);
}
public void testSumFloat() throws Exception {
- assertHits(new FunctionQuery(new SumFloatFunction(new ValueSource[] {
- new ConstValueSource(1f), new ConstValueSource(2f)})),
- new float[] { 3f, 3f });
+ ValueSource vs = new SumFloatFunction(new ValueSource[] {
+ new ConstValueSource(1f), new ConstValueSource(2f)});
+ assertHits(new FunctionQuery(vs), new float[] { 3f, 3f });
+ assertAllExist(vs);
+
+ vs = new SumFloatFunction(new ValueSource[] {
+ BOGUS_FLOAT_VS, new ConstValueSource(2f)});
+ assertNoneExist(vs);
}
public void testSumTotalTermFreq() throws Exception {
- assertHits(new FunctionQuery(new SumTotalTermFreqValueSource("text")),
- new float[] { 8f, 8f });
+ ValueSource vs = new SumTotalTermFreqValueSource("text");
+ assertHits(new FunctionQuery(vs), new float[] { 8f, 8f });
+ assertAllExist(vs);
}
public void testTermFreq() throws Exception {
- assertHits(new FunctionQuery(
- new TermFreqValueSource("bogus", "bogus", "text", new BytesRef("test"))),
- new float[] { 3f, 1f });
- assertHits(new FunctionQuery(
- new TermFreqValueSource("bogus", "bogus", "string", new BytesRef("bar"))),
- new float[] { 0f, 1f });
+ ValueSource vs = new TermFreqValueSource("bogus", "bogus", "text", new BytesRef("test"));
+ assertHits(new FunctionQuery(vs), new float[] { 3f, 1f });
+ assertAllExist(vs);
+
+ vs = new TermFreqValueSource("bogus", "bogus", "string", new BytesRef("bar"));
+ assertHits(new FunctionQuery(vs), new float[] { 0f, 1f });
+ assertAllExist(vs);
+
+ // regardless of wether norms exist, value source exists == 0
+ vs = new TermFreqValueSource("bogus", "bogus", "bogus", new BytesRef("bogus"));
+ assertHits(new FunctionQuery(vs), new float[] { 0F, 0F });
+ assertAllExist(vs);
}
public void testTF() throws Exception {
@@ -338,23 +497,71 @@ public class TestValueSources extends Lu
try {
// no norm field (so agnostic to indexed similarity)
searcher.setSimilarity(new DefaultSimilarity());
- assertHits(new FunctionQuery(
- new TFValueSource("bogus", "bogus", "text", new BytesRef("test"))),
- new float[] { (float)Math.sqrt(3d), (float)Math.sqrt(1d) });
- assertHits(new FunctionQuery(
- new TFValueSource("bogus", "bogus", "string", new BytesRef("bar"))),
- new float[] { 0f, 1f });
+
+ ValueSource vs = new TFValueSource("bogus", "bogus", "text", new BytesRef("test"));
+ assertHits(new FunctionQuery(vs),
+ new float[] { (float)Math.sqrt(3d), (float)Math.sqrt(1d) });
+ assertAllExist(vs);
+
+ vs = new TFValueSource("bogus", "bogus", "string", new BytesRef("bar"));
+ assertHits(new FunctionQuery(vs), new float[] { 0f, 1f });
+ assertAllExist(vs);
+
+ // regardless of wether norms exist, value source exists == 0
+ vs = new TFValueSource("bogus", "bogus", "bogus", new BytesRef("bogus"));
+ assertHits(new FunctionQuery(vs), new float[] { 0F, 0F });
+ assertAllExist(vs);
+
} finally {
searcher.setSimilarity(saved);
}
}
public void testTotalTermFreq() throws Exception {
- assertHits(new FunctionQuery(
- new TotalTermFreqValueSource("bogus", "bogus", "text", new BytesRef("test"))),
- new float[] { 4f, 4f });
+ ValueSource vs = new TotalTermFreqValueSource("bogus", "bogus", "text", new BytesRef("test"));
+ assertHits(new FunctionQuery(vs), new float[] { 4f, 4f });
+ assertAllExist(vs);
+ }
+
+ /**
+ * Asserts that for every doc, the {@link FunctionValues#exists} value
+ * from the {@link ValueSource} is <b>true</b>.
+ */
+ void assertAllExist(ValueSource vs) {
+ assertExists(ALL_EXIST_VS, vs);
+ }
+ /**
+ * Asserts that for every doc, the {@link FunctionValues#exists} value
+ * from the {@link ValueSource} is <b>false</b>.
+ */
+ void assertNoneExist(ValueSource vs) {
+ assertExists(NONE_EXIST_VS, vs);
+ }
+ /**
+ * Asserts that for every doc, the {@link FunctionValues#exists} value from the
+ * <code>actual</code> {@link ValueSource} matches the {@link FunctionValues#exists}
+ * value from the <code>expected</code> {@link ValueSource}
+ */
+ void assertExists(ValueSource expected, ValueSource actual) {
+ Map context = ValueSource.newContext(searcher);
+ try {
+ expected.createWeight(context, searcher);
+ actual.createWeight(context, searcher);
+
+ for (org.apache.lucene.index.LeafReaderContext leaf : reader.leaves()) {
+ final FunctionValues expectedVals = expected.getValues(context, leaf);
+ final FunctionValues actualVals = actual.getValues(context, leaf);
+
+ String msg = expected.toString() + " ?= " + actual.toString() + " -> ";
+ for (int i = 0; i < leaf.reader().maxDoc(); ++i) {
+ assertEquals(msg + i, expectedVals.exists(i), actualVals.exists(i));
+ }
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(actual.toString(), e);
+ }
}
-
+
void assertHits(Query q, float scores[]) throws Exception {
ScoreDoc expected[] = new ScoreDoc[scores.length];
int expectedDocs[] = new int[scores.length];
@@ -368,4 +575,54 @@ public class TestValueSources extends Lu
CheckHits.checkHitsQuery(q, expected, docs.scoreDocs, expectedDocs);
CheckHits.checkExplanations(q, "", searcher);
}
+
+ /**
+ * Simple test value source that implements {@link FunctionValues#exists} as a constant
+ * @see #ALL_EXIST_VS
+ * @see #NONE_EXIST_VS
+ */
+ private static final class ExistsValueSource extends ValueSource {
+
+ final boolean expected;
+ final int value;
+
+ public ExistsValueSource(boolean expected) {
+ this.expected = expected;
+ this.value = expected ? 1 : 0;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return o == this;
+ }
+
+ @Override
+ public int hashCode() {
+ return value;
+ }
+
+ @Override
+ public String description() {
+ return expected ? "ALL_EXIST" : "NONE_EXIST";
+ }
+
+ @Override
+ public FunctionValues getValues(Map context, LeafReaderContext readerContext) {
+ return new FloatDocValues(this) {
+ @Override
+ public float floatVal(int doc) {
+ return (float)value;
+ }
+ @Override
+ public boolean exists(int doc) {
+ return expected;
+ }
+ };
+ }
+ };
+
+ /** @see ExistsValueSource */
+ private static final ValueSource ALL_EXIST_VS = new ExistsValueSource(true);
+ /** @see ExistsValueSource */
+ private static final ValueSource NONE_EXIST_VS = new ExistsValueSource(false);
}
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1632414&r1=1632413&r2=1632414&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Thu Oct 16 19:05:20 2014
@@ -94,6 +94,12 @@ Upgrading from Solr 4.x
If you need the old functionality back , please add an extra parameter f=/**
example: /update/json/docs?f=/**
+* Bugs fixed in several ValueSource functions may result in different behavior in
+ situations where some documents do not have values for fields wrapped in other value
+ sources. Users who want to preserve the previous behavior may need to wrap fields
+ in the "def()" function. Example: changing "fl=sum(fieldA,fieldB)" to
+ "fl=sum(def(fieldA,0.0),def(fieldB,0.0))". See LUCENE-5961 for more details.
+
Detailed Change List
----------------------
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java?rev=1632414&r1=1632413&r2=1632414&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java Thu Oct 16 19:05:20 2014
@@ -469,10 +469,9 @@ public class StatsComponentTest extends
"{!type=func key="+key+" ex=key_ex_tag}field("+f+")",
"{!type=func key="+key+" ex=key_ex_tag v=field("+f+")}",
"{!type=func key="+key+" ex=key_ex_tag v='field("+f+")'}",
-
- // identity math functions don't work as expected due to LUCENE-5961
- // "{!type=func key="+key+" ex=key_ex_tag v='sum(0,"+f+")'}",
- // "{!type=func key="+key+" ex=key_ex_tag v='product(1,"+f+")'}",
+ // identity math functions
+ "{!type=func key="+key+" ex=key_ex_tag v='sum(0,"+f+")'}",
+ "{!type=func key="+key+" ex=key_ex_tag v='product(1,"+f+")'}",
}) {
assertQ("test statistics over field specified as a function: " + param,
@@ -555,10 +554,9 @@ public class StatsComponentTest extends
"{!type=func key="+f+" ex=key_ex_tag}field("+f+")",
"{!type=func key="+f+" ex=key_ex_tag v=field("+f+")}",
"{!type=func key="+f+" ex=key_ex_tag v='field("+f+")'}",
-
- // identity math functions don't work as expected due to LUCENE-5961
- // "{!type=func key="+f+" ex=key_ex_tag v='sum(0,"+f+")'}",
- // "{!type=func key="+f+" ex=key_ex_tag v='product(1,"+f+")'}",
+ // identity math functions
+ "{!type=func key="+f+" ex=key_ex_tag v='sum(0,"+f+")'}",
+ "{!type=func key="+f+" ex=key_ex_tag v='product(1,"+f+")'}",
}) {
assertQ("test statis & stats.facet over field specified as a function: " + param,
req("q", "*:*", "stats", "true", "stats.calcdistinct", "true",
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java?rev=1632414&r1=1632413&r2=1632414&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java Thu Oct 16 19:05:20 2014
@@ -731,12 +731,17 @@ public class TestFunctionQuery extends S
@Test
public void testPseudoFieldFunctions() throws Exception {
- assertU(adoc("id", "1", "text", "hello", "foo_s","A"));
+ assertU(adoc("id", "1", "text", "hello", "foo_s","A", "yak_i", "32"));
assertU(adoc("id", "2"));
assertU(commit());
- assertJQ(req("q", "id:1", "fl", "a:1,b:2.0,c:'X',d:{!func}foo_s,e:{!func}bar_s") // if exists() is false, no pseudo-field should be added
- , "/response/docs/[0]=={'a':1, 'b':2.0,'c':'X','d':'A'}");
+ // if exists() is false, no pseudo-field should be added
+ assertJQ(req("q", "id:1", "fl", "a:1,b:2.0,c:'X',d:{!func}foo_s,e:{!func}bar_s")
+ , "/response/docs/[0]=={'a':1, 'b':2.0,'c':'X','d':'A'}");
+ assertJQ(req("q", "id:1", "fl", "a:sum(yak_i,bog_i),b:mul(yak_i,bog_i),c:min(yak_i,bog_i)")
+ , "/response/docs/[0]=={ 'c':32.0 }");
+ assertJQ(req("q", "id:1", "fl", "a:sum(yak_i,def(bog_i,42)), b:max(yak_i,bog_i)")
+ , "/response/docs/[0]=={ 'a': 74.0, 'b':32.0 }");
}
public void testMissingFieldFunctionBehavior() throws Exception {