You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by is...@apache.org on 2017/01/16 21:27:26 UTC
[01/14] lucene-solr:jira/solr-5944: add test that EdgeNGram filter
keeps payloads
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-5944 0402c1a06 -> 80cdc0427
add test that EdgeNGram filter keeps payloads
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/61e45283
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/61e45283
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/61e45283
Branch: refs/heads/jira/solr-5944
Commit: 61e45283061ae486acc5882c5a770025c8291222
Parents: 987e265
Author: Nathan Gass <ga...@search.ch>
Authored: Mon Jan 9 14:59:31 2017 +0100
Committer: Nathan Gass <ga...@search.ch>
Committed: Fri Jan 13 12:14:26 2017 +0100
----------------------------------------------------------------------
.../lucene/analysis/ngram/TestNGramFilters.java | 22 ++++++++++++++++++++
1 file changed, 22 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/61e45283/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/TestNGramFilters.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/TestNGramFilters.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/TestNGramFilters.java
index 1243352..b6f4405 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/TestNGramFilters.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/TestNGramFilters.java
@@ -22,7 +22,10 @@ import java.io.StringReader;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.payloads.PayloadHelper;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
+import org.apache.lucene.util.BytesRef;
/**
* Simple tests to ensure the NGram filter factories are working.
@@ -123,6 +126,25 @@ public class TestNGramFilters extends BaseTokenStreamFactoryTestCase {
assertTokenStreamContents(stream,
new String[] { "t", "te" });
}
+
+ public void testEdgeNGramFilterPayload() throws Exception {
+ Reader reader = new StringReader("test|0.1");
+ TokenStream stream = whitespaceMockTokenizer(reader);
+ stream = tokenFilterFactory("DelimitedPayload", "encoder", "float").create(stream);
+ stream = tokenFilterFactory("EdgeNGram", "minGramSize", "1", "maxGramSize", "2").create(stream);
+
+ stream.reset();
+ while (stream.incrementToken()) {
+ PayloadAttribute payAttr = stream.getAttribute(PayloadAttribute.class);
+ assertNotNull(payAttr);
+ BytesRef payData = payAttr.getPayload();
+ assertNotNull(payData);
+ float payFloat = PayloadHelper.decodeFloat(payData.bytes);
+ assertEquals(0.1f, payFloat, 0.0f);
+ }
+ stream.end();
+ stream.close();
+ }
/** Test that bogus arguments result in exception */
public void testBogusArguments() throws Exception {
[13/14] lucene-solr:jira/solr-5944: Merge the two problem sections in
org.eclipse.jdt.core.prefs settings.
Posted by is...@apache.org.
Merge the two problem sections in org.eclipse.jdt.core.prefs settings.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/205f9cc5
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/205f9cc5
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/205f9cc5
Branch: refs/heads/jira/solr-5944
Commit: 205f9cc59ed001e6d262930482b88e723e2ea3f8
Parents: 2301900
Author: Christine Poerschke <cp...@apache.org>
Authored: Mon Jan 16 18:23:12 2017 +0000
Committer: Christine Poerschke <cp...@apache.org>
Committed: Mon Jan 16 18:42:07 2017 +0000
----------------------------------------------------------------------
dev-tools/eclipse/dot.settings/org.eclipse.jdt.core.prefs | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/205f9cc5/dev-tools/eclipse/dot.settings/org.eclipse.jdt.core.prefs
----------------------------------------------------------------------
diff --git a/dev-tools/eclipse/dot.settings/org.eclipse.jdt.core.prefs b/dev-tools/eclipse/dot.settings/org.eclipse.jdt.core.prefs
index 6f6533a..0f0b112 100644
--- a/dev-tools/eclipse/dot.settings/org.eclipse.jdt.core.prefs
+++ b/dev-tools/eclipse/dot.settings/org.eclipse.jdt.core.prefs
@@ -4,6 +4,7 @@ org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.doc.comment.support=enabled
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
+org.eclipse.jdt.core.compiler.problem.comparingIdentical=error
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.invalidJavadoc=error
org.eclipse.jdt.core.compiler.problem.invalidJavadocTags=enabled
@@ -18,6 +19,9 @@ org.eclipse.jdt.core.compiler.problem.missingJavadocTags=ignore
org.eclipse.jdt.core.compiler.problem.missingJavadocTagsMethodTypeParameters=disabled
org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=disabled
org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=public
+org.eclipse.jdt.core.compiler.problem.noEffectAssignment=error
+org.eclipse.jdt.core.compiler.problem.unusedImport=error
+org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=error
org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
org.eclipse.jdt.core.compiler.source=1.8
org.eclipse.jdt.core.compiler.taskCaseSensitive=enabled
@@ -304,7 +308,3 @@ org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=true
org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true
org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true
org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=error
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=error
-org.eclipse.jdt.core.compiler.problem.unusedImport=error
-org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=error
[10/14] lucene-solr:jira/solr-5944: SOLR-9906: Fix dodgy test check
Posted by is...@apache.org.
SOLR-9906: Fix dodgy test check
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/efc7ee0f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/efc7ee0f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/efc7ee0f
Branch: refs/heads/jira/solr-5944
Commit: efc7ee0f0c9154fe58671601fdc053540c97ff62
Parents: 478de2a
Author: Alan Woodward <ro...@apache.org>
Authored: Mon Jan 16 15:24:02 2017 +0000
Committer: Alan Woodward <ro...@apache.org>
Committed: Mon Jan 16 15:24:02 2017 +0000
----------------------------------------------------------------------
.../java/org/apache/solr/cloud/AbstractDistribZkTestBase.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/efc7ee0f/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java
index 0669cbe..7141eed 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java
@@ -29,7 +29,6 @@ import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.Slice.State;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.core.Diagnostics;
@@ -240,8 +239,8 @@ public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTes
ClusterState clusterState = zkStateReader.getClusterState();
DocCollection coll = clusterState.getCollection("collection1");
Slice slice = coll.getSlice(shardName);
- if (slice.getLeader() != null && !slice.getLeader().equals(oldLeader) && slice.getState() == State.ACTIVE) {
- log.info("Old leader {}, new leader. New leader got elected in {} ms", oldLeader, slice.getLeader(),timeOut.timeElapsed(MILLISECONDS) );
+ if (slice.getLeader() != null && !slice.getLeader().equals(oldLeader) && slice.getLeader().getState() == Replica.State.ACTIVE) {
+ log.info("Old leader {}, new leader {}. New leader got elected in {} ms", oldLeader, slice.getLeader(),timeOut.timeElapsed(MILLISECONDS) );
break;
}
[11/14] lucene-solr:jira/solr-5944: Remove unnecessary @Override
annotation in CoreParser.java class.
Posted by is...@apache.org.
Remove unnecessary @Override annotation in CoreParser.java class.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/649c58de
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/649c58de
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/649c58de
Branch: refs/heads/jira/solr-5944
Commit: 649c58de0252ba608963e0fe699f8332c870b294
Parents: efc7ee0
Author: Christine Poerschke <cp...@apache.org>
Authored: Mon Jan 16 15:10:51 2017 +0000
Committer: Christine Poerschke <cp...@apache.org>
Committed: Mon Jan 16 15:29:30 2017 +0000
----------------------------------------------------------------------
.../src/java/org/apache/lucene/queryparser/xml/CoreParser.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/649c58de/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/CoreParser.java
----------------------------------------------------------------------
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/CoreParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/CoreParser.java
index 1bf82ac..d8aa8ef 100644
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/CoreParser.java
+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/CoreParser.java
@@ -141,8 +141,6 @@ public class CoreParser implements QueryBuilder {
return doc;
}
-
- @Override
public Query getQuery(Element e) throws ParserException {
return queryFactory.getQuery(e);
}
[03/14] lucene-solr:jira/solr-5944: use captureState and restoreState
instead of cloneAttributes
Posted by is...@apache.org.
use captureState and restoreState instead of cloneAttributes
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/01f2a87c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/01f2a87c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/01f2a87c
Branch: refs/heads/jira/solr-5944
Commit: 01f2a87c67392a86b533d0c76ba7666845d1945f
Parents: 6570e6e
Author: Nathan Gass <ga...@search.ch>
Authored: Fri Jan 13 15:54:07 2017 +0100
Committer: Nathan Gass <ga...@search.ch>
Committed: Fri Jan 13 15:54:07 2017 +0100
----------------------------------------------------------------------
.../apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/01f2a87c/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
index 303b7e320..df12fda 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
@@ -23,7 +23,7 @@ import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.AttributeSource.State;
/**
* Tokenizes the given token into n-grams of given size(s).
@@ -43,7 +43,7 @@ public final class EdgeNGramTokenFilter extends TokenFilter {
private int curCodePointCount;
private int curGramSize;
private int savePosIncr;
- private AttributeSource attributes;
+ private State state;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
@@ -81,15 +81,14 @@ public final class EdgeNGramTokenFilter extends TokenFilter {
curTermLength = termAtt.length();
curCodePointCount = Character.codePointCount(termAtt, 0, termAtt.length());
curGramSize = minGram;
- attributes = input.cloneAttributes();
+ state = captureState();
savePosIncr += posIncrAtt.getPositionIncrement();
}
}
if (curGramSize <= maxGram) { // if we have hit the end of our n-gram size range, quit
if (curGramSize <= curCodePointCount) { // if the remaining input is too short, we can't generate any n-grams
// grab gramSize chars from front or back
- clearAttributes();
- attributes.copyTo(this);
+ restoreState(state);
// first ngram gets increment, others don't
if (curGramSize == minGram) {
posIncrAtt.setPositionIncrement(savePosIncr);
[02/14] lucene-solr:jira/solr-5944: copy all attributes including
payload to new tokens
Posted by is...@apache.org.
copy all attributes including payload to new tokens
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6570e6ec
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6570e6ec
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6570e6ec
Branch: refs/heads/jira/solr-5944
Commit: 6570e6ecc2b14a28da9873948083791ba47145d0
Parents: 61e4528
Author: Nathan Gass <ga...@search.ch>
Authored: Mon Jan 9 15:00:21 2017 +0100
Committer: Nathan Gass <ga...@search.ch>
Committed: Fri Jan 13 12:14:27 2017 +0100
----------------------------------------------------------------------
.../lucene/analysis/ngram/EdgeNGramTokenFilter.java | 16 ++++------------
1 file changed, 4 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6570e6ec/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
index 827e26f..303b7e320 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
@@ -22,9 +22,8 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
+import org.apache.lucene.util.AttributeSource;
/**
* Tokenizes the given token into n-grams of given size(s).
@@ -43,15 +42,11 @@ public final class EdgeNGramTokenFilter extends TokenFilter {
private int curTermLength;
private int curCodePointCount;
private int curGramSize;
- private int tokStart;
- private int tokEnd; // only used if the length changed before this filter
private int savePosIncr;
- private int savePosLen;
+ private AttributeSource attributes;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
- private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
- private final PositionLengthAttribute posLenAtt = addAttribute(PositionLengthAttribute.class);
/**
* Creates EdgeNGramTokenFilter that can generate n-grams in the sizes of the given range
@@ -86,17 +81,15 @@ public final class EdgeNGramTokenFilter extends TokenFilter {
curTermLength = termAtt.length();
curCodePointCount = Character.codePointCount(termAtt, 0, termAtt.length());
curGramSize = minGram;
- tokStart = offsetAtt.startOffset();
- tokEnd = offsetAtt.endOffset();
+ attributes = input.cloneAttributes();
savePosIncr += posIncrAtt.getPositionIncrement();
- savePosLen = posLenAtt.getPositionLength();
}
}
if (curGramSize <= maxGram) { // if we have hit the end of our n-gram size range, quit
if (curGramSize <= curCodePointCount) { // if the remaining input is too short, we can't generate any n-grams
// grab gramSize chars from front or back
clearAttributes();
- offsetAtt.setOffset(tokStart, tokEnd);
+ attributes.copyTo(this);
// first ngram gets increment, others don't
if (curGramSize == minGram) {
posIncrAtt.setPositionIncrement(savePosIncr);
@@ -104,7 +97,6 @@ public final class EdgeNGramTokenFilter extends TokenFilter {
} else {
posIncrAtt.setPositionIncrement(0);
}
- posLenAtt.setPositionLength(savePosLen);
final int charLength = Character.offsetByCodePoints(curTermBuffer, 0, curTermLength, 0, curGramSize);
termAtt.copyBuffer(curTermBuffer, 0, charLength);
curGramSize++;
[08/14] lucene-solr:jira/solr-5944: LUCENE-7623: Add
FunctionMatchQuery and FunctionScoreQuery
Posted by is...@apache.org.
LUCENE-7623: Add FunctionMatchQuery and FunctionScoreQuery
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/fc2e0fd1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/fc2e0fd1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/fc2e0fd1
Branch: refs/heads/jira/solr-5944
Commit: fc2e0fd13324699fe1ddb15bb09960a8501f52f5
Parents: ceaeb42
Author: Alan Woodward <ro...@apache.org>
Authored: Sun Jan 15 18:37:41 2017 +0000
Committer: Alan Woodward <ro...@apache.org>
Committed: Mon Jan 16 11:15:33 2017 +0000
----------------------------------------------------------------------
lucene/CHANGES.txt | 5 +
.../lucene/search/DoubleValuesSource.java | 92 +++++++++--
.../queries/function/FunctionMatchQuery.java | 99 ++++++++++++
.../queries/function/FunctionScoreQuery.java | 151 +++++++++++++++++++
.../function/TestFunctionMatchQuery.java | 61 ++++++++
.../function/TestFunctionScoreExplanations.java | 105 +++++++++++++
.../function/TestFunctionScoreQuery.java | 114 ++++++++++++++
7 files changed, 611 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fc2e0fd1/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 4963454..851ed72 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -63,6 +63,11 @@ Other
======================= Lucene 6.5.0 =======================
+New Features
+
+* LUCENE-7623: Add FunctionScoreQuery and FunctionMatchQuery (Alan Woodward,
+ Adrien Grand, David Smiley)
+
Bug Fixes
* LUCENE-7630: Fix (Edge)NGramTokenFilter to no longer drop payloads
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fc2e0fd1/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java b/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java
index d4be4e9..af24e1a 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java
@@ -20,7 +20,9 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.Objects;
import java.util.function.DoubleToLongFunction;
+import java.util.function.DoubleUnaryOperator;
import java.util.function.LongToDoubleFunction;
+import java.util.function.ToDoubleBiFunction;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReaderContext;
@@ -173,6 +175,69 @@ public abstract class DoubleValuesSource {
public boolean needsScores() {
return false;
}
+
+ @Override
+ public String toString() {
+ return "constant(" + value + ")";
+ }
+ };
+ }
+
+ /**
+ * Creates a DoubleValuesSource that is a function of another DoubleValuesSource
+ */
+ public static DoubleValuesSource function(DoubleValuesSource in, DoubleUnaryOperator function) {
+ return new DoubleValuesSource() {
+ @Override
+ public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
+ DoubleValues inputs = in.getValues(ctx, scores);
+ return new DoubleValues() {
+ @Override
+ public double doubleValue() throws IOException {
+ return function.applyAsDouble(inputs.doubleValue());
+ }
+
+ @Override
+ public boolean advanceExact(int doc) throws IOException {
+ return inputs.advanceExact(doc);
+ }
+ };
+ }
+
+ @Override
+ public boolean needsScores() {
+ return in.needsScores();
+ }
+ };
+ }
+
+ /**
+ * Creates a DoubleValuesSource that is a function of another DoubleValuesSource and a score
+ * @param in the DoubleValuesSource to use as an input
+ * @param function a function of the form (source, score) == result
+ */
+ public static DoubleValuesSource scoringFunction(DoubleValuesSource in, ToDoubleBiFunction<Double, Double> function) {
+ return new DoubleValuesSource() {
+ @Override
+ public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
+ DoubleValues inputs = in.getValues(ctx, scores);
+ return new DoubleValues() {
+ @Override
+ public double doubleValue() throws IOException {
+ return function.applyAsDouble(inputs.doubleValue(), scores.doubleValue());
+ }
+
+ @Override
+ public boolean advanceExact(int doc) throws IOException {
+ return inputs.advanceExact(doc);
+ }
+ };
+ }
+
+ @Override
+ public boolean needsScores() {
+ return true;
+ }
};
}
@@ -221,7 +286,17 @@ public abstract class DoubleValuesSource {
@Override
public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
final NumericDocValues values = DocValues.getNumeric(ctx.reader(), field);
- return toDoubleValues(values, decoder::applyAsDouble);
+ return new DoubleValues() {
+ @Override
+ public double doubleValue() throws IOException {
+ return decoder.applyAsDouble(values.longValue());
+ }
+
+ @Override
+ public boolean advanceExact(int target) throws IOException {
+ return values.advanceExact(target);
+ }
+ };
}
@Override
@@ -288,21 +363,6 @@ public abstract class DoubleValuesSource {
}
}
- private static DoubleValues toDoubleValues(NumericDocValues in, LongToDoubleFunction map) {
- return new DoubleValues() {
- @Override
- public double doubleValue() throws IOException {
- return map.applyAsDouble(in.longValue());
- }
-
- @Override
- public boolean advanceExact(int target) throws IOException {
- return in.advanceExact(target);
- }
-
- };
- }
-
private static NumericDocValues asNumericDocValues(DoubleValuesHolder in, DoubleToLongFunction converter) {
return new NumericDocValues() {
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fc2e0fd1/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java
new file mode 100644
index 0000000..4a9c709
--- /dev/null
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.queries.function;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.function.DoublePredicate;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.ConstantScoreScorer;
+import org.apache.lucene.search.ConstantScoreWeight;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TwoPhaseIterator;
+import org.apache.lucene.search.Weight;
+
+/**
+ * A query that retrieves all documents with a {@link DoubleValues} value matching a predicate
+ *
+ * This query works by a linear scan of the index, and is best used in
+ * conjunction with other queries that can restrict the number of
+ * documents visited
+ */
+public final class FunctionMatchQuery extends Query {
+
+ private final DoubleValuesSource source;
+ private final DoublePredicate filter;
+
+ /**
+ * Create a FunctionMatchQuery
+ * @param source a {@link DoubleValuesSource} to use for values
+ * @param filter the predicate to match against
+ */
+ public FunctionMatchQuery(DoubleValuesSource source, DoublePredicate filter) {
+ this.source = source;
+ this.filter = filter;
+ }
+
+ @Override
+ public String toString(String field) {
+ return "FunctionMatchQuery(" + source.toString() + ")";
+ }
+
+ @Override
+ public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException {
+ return new ConstantScoreWeight(this, boost) {
+ @Override
+ public Scorer scorer(LeafReaderContext context) throws IOException {
+ DoubleValues values = source.getValues(context, null);
+ DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc());
+ TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) {
+ @Override
+ public boolean matches() throws IOException {
+ return values.advanceExact(approximation.docID()) && filter.test(values.doubleValue());
+ }
+
+ @Override
+ public float matchCost() {
+ return 100; // TODO maybe DoubleValuesSource should have a matchCost?
+ }
+ };
+ return new ConstantScoreScorer(this, score(), twoPhase);
+ }
+ };
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ FunctionMatchQuery that = (FunctionMatchQuery) o;
+ return Objects.equals(source, that.source) && Objects.equals(filter, that.filter);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(source, filter);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fc2e0fd1/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java
new file mode 100644
index 0000000..29ef41f
--- /dev/null
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.queries.function;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Set;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.FilterScorer;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Weight;
+
+/**
+ * A query that wraps another query, and uses a DoubleValuesSource to
+ * replace or modify the wrapped query's score
+ *
+ * If the DoubleValuesSource doesn't return a value for a particular document,
+ * then that document will be given a score of 0.
+ */
+public final class FunctionScoreQuery extends Query {
+
+ private final Query in;
+ private final DoubleValuesSource source;
+
+ /**
+ * Create a new FunctionScoreQuery
+ * @param in the query to wrap
+ * @param source a source of scores
+ */
+ public FunctionScoreQuery(Query in, DoubleValuesSource source) {
+ this.in = in;
+ this.source = source;
+ }
+
+ @Override
+ public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException {
+ Weight inner = in.createWeight(searcher, needsScores && source.needsScores(), 1f);
+ if (needsScores == false)
+ return inner;
+ return new FunctionScoreWeight(this, inner, source, boost);
+ }
+
+ @Override
+ public Query rewrite(IndexReader reader) throws IOException {
+ Query rewritten = in.rewrite(reader);
+ if (rewritten == in)
+ return this;
+ return new FunctionScoreQuery(rewritten, source);
+ }
+
+ @Override
+ public String toString(String field) {
+ return "FunctionScoreQuery(" + in.toString(field) + ", scored by " + source.toString() + ")";
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ FunctionScoreQuery that = (FunctionScoreQuery) o;
+ return Objects.equals(in, that.in) &&
+ Objects.equals(source, that.source);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(in, source);
+ }
+
+ private static class FunctionScoreWeight extends Weight {
+
+ final Weight inner;
+ final DoubleValuesSource valueSource;
+ final float boost;
+
+ FunctionScoreWeight(Query query, Weight inner, DoubleValuesSource valueSource, float boost) {
+ super(query);
+ this.inner = inner;
+ this.valueSource = valueSource;
+ this.boost = boost;
+ }
+
+ @Override
+ public void extractTerms(Set<Term> terms) {
+ this.inner.extractTerms(terms);
+ }
+
+ @Override
+ public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+ Scorer scorer = inner.scorer(context);
+ if (scorer.iterator().advance(doc) != doc)
+ return Explanation.noMatch("No match");
+ DoubleValues scores = valueSource.getValues(context, DoubleValuesSource.fromScorer(scorer));
+ scores.advanceExact(doc);
+ Explanation scoreExpl = scoreExplanation(context, doc, scores);
+ if (boost == 1f)
+ return scoreExpl;
+ return Explanation.match(scoreExpl.getValue() * boost, "product of:",
+ Explanation.match(boost, "boost"), scoreExpl);
+ }
+
+ private Explanation scoreExplanation(LeafReaderContext context, int doc, DoubleValues scores) throws IOException {
+ if (valueSource.needsScores() == false)
+ return Explanation.match((float) scores.doubleValue(), valueSource.toString());
+ float score = (float) scores.doubleValue();
+ return Explanation.match(score, "computed from:",
+ Explanation.match(score, valueSource.toString()),
+ inner.explain(context, doc));
+ }
+
+ @Override
+ public Scorer scorer(LeafReaderContext context) throws IOException {
+ Scorer in = inner.scorer(context);
+ if (in == null)
+ return null;
+ DoubleValues scores = valueSource.getValues(context, DoubleValuesSource.fromScorer(in));
+ return new FilterScorer(in) {
+ @Override
+ public float score() throws IOException {
+ if (scores.advanceExact(docID()))
+ return (float) (scores.doubleValue() * boost);
+ else
+ return 0;
+ }
+ };
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fc2e0fd1/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionMatchQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionMatchQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionMatchQuery.java
new file mode 100644
index 0000000..61faa15
--- /dev/null
+++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionMatchQuery.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.queries.function;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.QueryUtils;
+import org.apache.lucene.search.TopDocs;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+public class TestFunctionMatchQuery extends FunctionTestSetup {
+
+ static IndexReader reader;
+ static IndexSearcher searcher;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ createIndex(true);
+ reader = DirectoryReader.open(dir);
+ searcher = new IndexSearcher(reader);
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ reader.close();
+ }
+
+ public void testRangeMatching() throws IOException {
+ DoubleValuesSource in = DoubleValuesSource.fromFloatField(FLOAT_FIELD);
+ FunctionMatchQuery fmq = new FunctionMatchQuery(in, d -> d >= 2 && d < 4);
+ TopDocs docs = searcher.search(fmq, 10);
+
+ assertEquals(2, docs.totalHits);
+ assertEquals(9, docs.scoreDocs[0].doc);
+ assertEquals(13, docs.scoreDocs[1].doc);
+
+ QueryUtils.check(random(), fmq, searcher, rarely());
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fc2e0fd1/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreExplanations.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreExplanations.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreExplanations.java
new file mode 100644
index 0000000..5c64396
--- /dev/null
+++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreExplanations.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.queries.function;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BaseExplanationTestCase;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.BoostQuery;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
+
+public class TestFunctionScoreExplanations extends BaseExplanationTestCase {
+
+ public void testOneTerm() throws Exception {
+ Query q = new TermQuery(new Term(FIELD, "w1"));
+ FunctionScoreQuery fsq = new FunctionScoreQuery(q, DoubleValuesSource.constant(5));
+ qtest(fsq, new int[] { 0,1,2,3 });
+ }
+
+ public void testBoost() throws Exception {
+ Query q = new TermQuery(new Term(FIELD, "w1"));
+ FunctionScoreQuery csq = new FunctionScoreQuery(q, DoubleValuesSource.constant(5));
+ qtest(new BoostQuery(csq, 4), new int[] { 0,1,2,3 });
+ }
+
+ public void testTopLevelBoost() throws Exception {
+ Query q = new TermQuery(new Term(FIELD, "w1"));
+ FunctionScoreQuery csq = new FunctionScoreQuery(q, DoubleValuesSource.constant(5));
+ BooleanQuery.Builder bqB = new BooleanQuery.Builder();
+ bqB.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
+ bqB.add(csq, BooleanClause.Occur.MUST);
+ BooleanQuery bq = bqB.build();
+ qtest(new BoostQuery(bq, 6), new int[] { 0,1,2,3 });
+ }
+
+ public void testExplanationsIncludingScore() throws Exception {
+
+ DoubleValuesSource scores = DoubleValuesSource.function(DoubleValuesSource.SCORES, v -> v * 2);
+
+ Query q = new TermQuery(new Term(FIELD, "w1"));
+ FunctionScoreQuery csq = new FunctionScoreQuery(q, scores);
+
+ qtest(csq, new int[] { 0, 1, 2, 3 });
+
+ Explanation e1 = searcher.explain(q, 0);
+ Explanation e = searcher.explain(csq, 0);
+
+ assertEquals(e.getDetails().length, 2);
+
+ assertEquals(e1.getValue() * 2, e.getValue(), 0.00001);
+ }
+
+ public void testSubExplanations() throws IOException {
+ Query query = new FunctionScoreQuery(new MatchAllDocsQuery(), DoubleValuesSource.constant(5));
+ IndexSearcher searcher = newSearcher(BaseExplanationTestCase.searcher.getIndexReader());
+ searcher.setSimilarity(new BM25Similarity());
+
+ Explanation expl = searcher.explain(query, 0);
+ assertEquals("constant(5.0)", expl.getDescription());
+ assertEquals(0, expl.getDetails().length);
+
+ query = new BoostQuery(query, 2);
+ expl = searcher.explain(query, 0);
+ assertEquals(2, expl.getDetails().length);
+ // function
+ assertEquals(5f, expl.getDetails()[1].getValue(), 0f);
+ // boost
+ assertEquals("boost", expl.getDetails()[0].getDescription());
+ assertEquals(2f, expl.getDetails()[0].getValue(), 0f);
+
+ searcher.setSimilarity(new ClassicSimilarity()); // in order to have a queryNorm != 1
+ expl = searcher.explain(query, 0);
+ assertEquals(2, expl.getDetails().length);
+ // function
+ assertEquals(5f, expl.getDetails()[1].getValue(), 0f);
+ // boost
+ assertEquals("boost", expl.getDetails()[0].getDescription());
+ assertEquals(2f, expl.getDetails()[0].getValue(), 0f);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fc2e0fd1/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreQuery.java
new file mode 100644
index 0000000..8f6ef8e
--- /dev/null
+++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreQuery.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.queries.function;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.BoostQuery;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryUtils;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+public class TestFunctionScoreQuery extends FunctionTestSetup {
+
+ static IndexReader reader;
+ static IndexSearcher searcher;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ createIndex(true);
+ reader = DirectoryReader.open(dir);
+ searcher = new IndexSearcher(reader);
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ reader.close();
+ }
+
+ // FunctionQuery equivalent
+ public void testSimpleSourceScore() throws Exception {
+
+ FunctionScoreQuery q = new FunctionScoreQuery(new TermQuery(new Term(TEXT_FIELD, "first")),
+ DoubleValuesSource.fromIntField(INT_FIELD));
+
+ QueryUtils.check(random(), q, searcher, rarely());
+
+ int expectedDocs[] = new int[]{ 4, 7, 9 };
+ TopDocs docs = searcher.search(q, 4);
+ assertEquals(expectedDocs.length, docs.totalHits);
+ for (int i = 0; i < expectedDocs.length; i++) {
+ assertEquals(docs.scoreDocs[i].doc, expectedDocs[i]);
+ }
+
+ }
+
+ // CustomScoreQuery and BoostedQuery equivalent
+ public void testScoreModifyingSource() throws Exception {
+
+ DoubleValuesSource iii = DoubleValuesSource.fromIntField("iii");
+ DoubleValuesSource score = DoubleValuesSource.scoringFunction(iii, (v, s) -> v * s);
+
+ BooleanQuery bq = new BooleanQuery.Builder()
+ .add(new TermQuery(new Term(TEXT_FIELD, "first")), BooleanClause.Occur.SHOULD)
+ .add(new TermQuery(new Term(TEXT_FIELD, "text")), BooleanClause.Occur.SHOULD)
+ .build();
+ TopDocs plain = searcher.search(bq, 1);
+
+ FunctionScoreQuery fq = new FunctionScoreQuery(bq, score);
+
+ QueryUtils.check(random(), fq, searcher, rarely());
+
+ int[] expectedDocs = new int[]{ 4, 7, 9, 8, 12 };
+ TopDocs docs = searcher.search(fq, 5);
+ assertEquals(plain.totalHits, docs.totalHits);
+ for (int i = 0; i < expectedDocs.length; i++) {
+ assertEquals(expectedDocs[i], docs.scoreDocs[i].doc);
+
+ }
+
+ }
+
+ // check boosts with non-distributive score source
+ public void testBoostsAreAppliedLast() throws Exception {
+
+ DoubleValuesSource scores
+ = DoubleValuesSource.function(DoubleValuesSource.SCORES, v -> Math.log(v + 4));
+
+ Query q1 = new FunctionScoreQuery(new TermQuery(new Term(TEXT_FIELD, "text")), scores);
+ TopDocs plain = searcher.search(q1, 5);
+
+ Query boosted = new BoostQuery(q1, 2);
+ TopDocs afterboost = searcher.search(boosted, 5);
+ assertEquals(plain.totalHits, afterboost.totalHits);
+ for (int i = 0; i < 5; i++) {
+ assertEquals(plain.scoreDocs[i].doc, afterboost.scoreDocs[i].doc);
+ assertEquals(plain.scoreDocs[i].score, afterboost.scoreDocs[i].score / 2, 0.0001);
+ }
+
+ }
+
+}
[06/14] lucene-solr:jira/solr-5944: LUCENE-7630: Fix
(Edge)NGramTokenFilter to no longer drop payloads and preserve all attributes
[merge branch 'edgepayloads' from Nathan Gass
https://github.com/xabbu42/lucene-solr]
Posted by is...@apache.org.
LUCENE-7630: Fix (Edge)NGramTokenFilter to no longer drop payloads and preserve all attributes
[merge branch 'edgepayloads' from Nathan Gass https://github.com/xabbu42/lucene-solr]
Signed-off-by: Uwe Schindler <us...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c64a0115
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c64a0115
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c64a0115
Branch: refs/heads/jira/solr-5944
Commit: c64a01158e972176256e257d6c1d4629b05783a2
Parents: 9aa78dc ea049b9
Author: Uwe Schindler <us...@apache.org>
Authored: Mon Jan 16 11:16:43 2017 +0100
Committer: Uwe Schindler <us...@apache.org>
Committed: Mon Jan 16 11:16:43 2017 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 7 +++
.../analysis/ngram/EdgeNGramTokenFilter.java | 17 ++-----
.../lucene/analysis/ngram/NGramTokenFilter.java | 19 +++-----
.../lucene/analysis/ngram/TestNGramFilters.java | 47 ++++++++++++++++++++
4 files changed, 63 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c64a0115/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --cc lucene/CHANGES.txt
index 58201d6,4912920..530b0d4
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@@ -61,6 -58,6 +61,13 @@@ Othe
* LUCENE-7360: Remove Explanation.toHtml() (Alan Woodward)
++======================= Lucene 6.5.0 =======================
++
++Bug Fixes
++
++* LUCENE-7630: Fix (Edge)NGramTokenFilter to no longer drop payloads
++ and preserve all attributes. (Nathan Gass via Uwe Schindler)
++
======================= Lucene 6.4.0 =======================
API Changes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c64a0115/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
----------------------------------------------------------------------
diff --cc lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
index 827e26f,df12fda..47b80ff
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
@@@ -95,8 -88,7 +88,7 @@@ public final class EdgeNGramTokenFilte
if (curGramSize <= maxGram) { // if we have hit the end of our n-gram size range, quit
if (curGramSize <= curCodePointCount) { // if the remaining input is too short, we can't generate any n-grams
// grab gramSize chars from front or back
- clearAttributes();
- offsetAtt.setOffset(tokStart, tokEnd);
- restoreState(state);
++ restoreState(state);
// first ngram gets increment, others don't
if (curGramSize == minGram) {
posIncrAtt.setPositionIncrement(savePosIncr);
[05/14] lucene-solr:jira/solr-5944: also copy all attributes for
ngram token filters
Posted by is...@apache.org.
also copy all attributes for ngram token filters
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ea049b96
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ea049b96
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ea049b96
Branch: refs/heads/jira/solr-5944
Commit: ea049b96a24d6afc582ecdf406e8bf256b9911d9
Parents: 80e2854
Author: Nathan Gass <ga...@search.ch>
Authored: Fri Jan 13 17:01:34 2017 +0100
Committer: Nathan Gass <ga...@search.ch>
Committed: Fri Jan 13 17:07:23 2017 +0100
----------------------------------------------------------------------
.../lucene/analysis/ngram/NGramTokenFilter.java | 19 +++++--------------
1 file changed, 5 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ea049b96/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
index e275cfa..cb5d447 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
@@ -23,9 +23,8 @@ import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.miscellaneous.CodepointCountFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
+import org.apache.lucene.util.AttributeSource.State;
/**
* Tokenizes the input into n-grams of the given size(s).
@@ -52,14 +51,11 @@ public final class NGramTokenFilter extends TokenFilter {
private int curCodePointCount;
private int curGramSize;
private int curPos;
- private int curPosInc, curPosLen;
- private int tokStart;
- private int tokEnd;
+ private int curPosInc;
+ private State state;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final PositionIncrementAttribute posIncAtt;
- private final PositionLengthAttribute posLenAtt;
- private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
/**
* Creates NGramTokenFilter with given min and max n-grams.
@@ -79,7 +75,6 @@ public final class NGramTokenFilter extends TokenFilter {
this.maxGram = maxGram;
posIncAtt = addAttribute(PositionIncrementAttribute.class);
- posLenAtt = addAttribute(PositionLengthAttribute.class);
}
/**
@@ -104,9 +99,7 @@ public final class NGramTokenFilter extends TokenFilter {
curGramSize = minGram;
curPos = 0;
curPosInc = posIncAtt.getPositionIncrement();
- curPosLen = posLenAtt.getPositionLength();
- tokStart = offsetAtt.startOffset();
- tokEnd = offsetAtt.endOffset();
+ state = captureState();
}
}
@@ -115,14 +108,12 @@ public final class NGramTokenFilter extends TokenFilter {
curGramSize = minGram;
}
if ((curPos + curGramSize) <= curCodePointCount) {
- clearAttributes();
+ restoreState(state);
final int start = Character.offsetByCodePoints(curTermBuffer, 0, curTermLength, 0, curPos);
final int end = Character.offsetByCodePoints(curTermBuffer, 0, curTermLength, start, curGramSize);
termAtt.copyBuffer(curTermBuffer, start, end - start);
posIncAtt.setPositionIncrement(curPosInc);
curPosInc = 0;
- posLenAtt.setPositionLength(curPosLen);
- offsetAtt.setOffset(tokStart, tokEnd);
curGramSize++;
return true;
}
[07/14] lucene-solr:jira/solr-5944: Add 6.5 version also in trunk;
sync up CHANGES.txt
Posted by is...@apache.org.
Add 6.5 version also in trunk; sync up CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ceaeb42a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ceaeb42a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ceaeb42a
Branch: refs/heads/jira/solr-5944
Commit: ceaeb42a1f31eff6e7f96c5d55a1c21449fbf07f
Parents: c64a011
Author: Uwe Schindler <us...@apache.org>
Authored: Mon Jan 16 11:50:28 2017 +0100
Committer: Uwe Schindler <us...@apache.org>
Committed: Mon Jan 16 11:50:28 2017 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 17 +++++++----------
.../src/java/org/apache/lucene/util/Version.java | 7 +++++++
solr/CHANGES.txt | 17 +++++++++++++++++
3 files changed, 31 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ceaeb42a/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 530b0d4..4963454 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -121,8 +121,7 @@ New features
* LUCENE-5325: Added LongValuesSource and DoubleValuesSource, intended as
type-safe replacements for ValueSource in the queries module. These
- expose per-segment LongValues or DoubleValues iterators, similar to the
- existing DocValues iterator API. (Alan Woodward, Adrien Grand)
+ expose per-segment LongValues or DoubleValues iterators. (Alan Woodward, Adrien Grand)
* LUCENE-7603: Graph token streams are now handled accurately by query
parsers, by enumerating all paths and creating the corresponding
@@ -180,6 +179,9 @@ Bug Fixes
Improvements
+* LUCENE-7532: Add back lost codec file format documentation
+ (Shinichiro Abe via Mike McCandless)
+
* LUCENE-6824: TermAutomatonQuery now rewrites to TermQuery,
PhraseQuery or MultiPhraseQuery when the word automaton is simple
(Mike McCandless)
@@ -206,14 +208,14 @@ Improvements
which can be overridden to return false to eek out more speed in some cases.
(Timothy M. Rodriguez, David Smiley)
-* LUCENE-7537: Index time sorting now supports multi-valued sorts
- using selectors (MIN, MAX, etc.) (Jim Ferenczi via Mike McCandless)
-
* LUCENE-7560: QueryBuilder.createFieldQuery is no longer final,
giving custom query parsers subclassing QueryBuilder more freedom to
control how text is analyzed and converted into a query (Matt Weber
via Mike McCandless)
+* LUCENE-7537: Index time sorting now supports multi-valued sorts
+ using selectors (MIN, MAX, etc.) (Jim Ferenczi via Mike McCandless)
+
* LUCENE-7575: UnifiedHighlighter can now highlight fields with queries that don't
necessarily refer to that field (AKA requireFieldMatch==false). Disabled by default.
See UH get/setFieldMatcher. (Jim Ferenczi via David Smiley)
@@ -254,11 +256,6 @@ Optimizations
* LUCENE-7572: Doc values queries now cache their hash code. (Adrien Grand)
-* LUCENE-7579: Segments are now also sorted during flush, and merging
- on a sorted index is substantially faster by using some of the same
- bulk merge optimizations that non-sorted merging uses (Jim Ferenczi
- via Mike McCandless)
-
Other
* LUCENE-7546: Fixed references to benchmark wikipedia data and the Jenkins line-docs file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ceaeb42a/lucene/core/src/java/org/apache/lucene/util/Version.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/Version.java b/lucene/core/src/java/org/apache/lucene/util/Version.java
index 2355a9a..00fb329 100644
--- a/lucene/core/src/java/org/apache/lucene/util/Version.java
+++ b/lucene/core/src/java/org/apache/lucene/util/Version.java
@@ -81,6 +81,13 @@ public final class Version {
public static final Version LUCENE_6_4_0 = new Version(6, 4, 0);
/**
+ * Match settings and bugs in Lucene's 6.5.0 release.
+ * @deprecated Use latest
+ */
+ @Deprecated
+ public static final Version LUCENE_6_5_0 = new Version(6, 5, 0);
+
+ /**
* Match settings and bugs in Lucene's 7.0.0 release.
* <p>
* Use this to get the latest & greatest settings, bug
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ceaeb42a/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index f8ecf23..4874067 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -77,6 +77,23 @@ Optimizations
* SOLR-9941: Clear the deletes lists at UpdateLog before replaying from log. This prevents redundantly pre-applying
DBQs, during the log replay, to every update in the log as if the DBQs were out of order. (hossman, Ishan Chattopadhyaya)
+================== 6.5.0 ==================
+
+Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
+
+Versions of Major Components
+---------------------
+Apache Tika 1.13
+Carrot2 3.15.0
+Velocity 1.7 and Velocity Tools 2.0
+Apache UIMA 2.3.1
+Apache ZooKeeper 3.4.6
+Jetty 9.3.14.v20161028
+
+
+(No Changes)
+
+
================== 6.4.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
[12/14] lucene-solr:jira/solr-5944: Remove four unnecessary @Override
annotations in SolrQueryBuilder (test) classes.
Posted by is...@apache.org.
Remove four unnecessary @Override annotations in SolrQueryBuilder (test) classes.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/23019006
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/23019006
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/23019006
Branch: refs/heads/jira/solr-5944
Commit: 230190065ca96c6ecc45e581a56f856888c2e321
Parents: 649c58d
Author: Christine Poerschke <cp...@apache.org>
Authored: Mon Jan 16 18:14:36 2017 +0000
Committer: Christine Poerschke <cp...@apache.org>
Committed: Mon Jan 16 18:14:36 2017 +0000
----------------------------------------------------------------------
.../org/apache/solr/search/ApacheLuceneSolrNearQueryBuilder.java | 1 -
solr/core/src/test/org/apache/solr/search/GoodbyeQueryBuilder.java | 1 -
solr/core/src/test/org/apache/solr/search/HandyQueryBuilder.java | 1 -
solr/core/src/test/org/apache/solr/search/HelloQueryBuilder.java | 1 -
4 files changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23019006/solr/core/src/test/org/apache/solr/search/ApacheLuceneSolrNearQueryBuilder.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/ApacheLuceneSolrNearQueryBuilder.java b/solr/core/src/test/org/apache/solr/search/ApacheLuceneSolrNearQueryBuilder.java
index 135ec45..bbc081a 100644
--- a/solr/core/src/test/org/apache/solr/search/ApacheLuceneSolrNearQueryBuilder.java
+++ b/solr/core/src/test/org/apache/solr/search/ApacheLuceneSolrNearQueryBuilder.java
@@ -35,7 +35,6 @@ public class ApacheLuceneSolrNearQueryBuilder extends SolrQueryBuilder {
super(defaultField, analyzer, req, queryFactory);
}
- @Override
public Query getQuery(Element e) throws ParserException {
final String fieldName = DOMUtils.getAttributeWithInheritanceOrFail(e, "fieldName");
final SpanQuery[] spanQueries = new SpanQuery[]{
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23019006/solr/core/src/test/org/apache/solr/search/GoodbyeQueryBuilder.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/GoodbyeQueryBuilder.java b/solr/core/src/test/org/apache/solr/search/GoodbyeQueryBuilder.java
index af258d4..93f4a1b 100644
--- a/solr/core/src/test/org/apache/solr/search/GoodbyeQueryBuilder.java
+++ b/solr/core/src/test/org/apache/solr/search/GoodbyeQueryBuilder.java
@@ -31,7 +31,6 @@ public class GoodbyeQueryBuilder extends SolrQueryBuilder {
super(defaultField, analyzer, req, queryFactory);
}
- @Override
public Query getQuery(Element e) throws ParserException {
return new MatchNoDocsQuery();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23019006/solr/core/src/test/org/apache/solr/search/HandyQueryBuilder.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/HandyQueryBuilder.java b/solr/core/src/test/org/apache/solr/search/HandyQueryBuilder.java
index 14a8aac..c38fb6b 100644
--- a/solr/core/src/test/org/apache/solr/search/HandyQueryBuilder.java
+++ b/solr/core/src/test/org/apache/solr/search/HandyQueryBuilder.java
@@ -35,7 +35,6 @@ public class HandyQueryBuilder extends SolrQueryBuilder {
super(defaultField, analyzer, req, queryFactory);
}
- @Override
public Query getQuery(Element e) throws ParserException {
final BooleanQuery.Builder bq = new BooleanQuery.Builder();
final Query lhsQ = getSubQuery(e, "Left");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23019006/solr/core/src/test/org/apache/solr/search/HelloQueryBuilder.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/HelloQueryBuilder.java b/solr/core/src/test/org/apache/solr/search/HelloQueryBuilder.java
index 642047f..8ea98f1 100644
--- a/solr/core/src/test/org/apache/solr/search/HelloQueryBuilder.java
+++ b/solr/core/src/test/org/apache/solr/search/HelloQueryBuilder.java
@@ -31,7 +31,6 @@ public class HelloQueryBuilder extends SolrQueryBuilder {
super(defaultField, analyzer, req, queryFactory);
}
- @Override
public Query getQuery(Element e) throws ParserException {
return new MatchAllDocsQuery();
}
[14/14] lucene-solr:jira/solr-5944: Merge branch 'master' into
jira/solr-5944
Posted by is...@apache.org.
Merge branch 'master' into jira/solr-5944
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/80cdc042
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/80cdc042
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/80cdc042
Branch: refs/heads/jira/solr-5944
Commit: 80cdc042749360b43fd541baf9774be12a4f27bd
Parents: 0402c1a 205f9cc
Author: Ishan Chattopadhyaya <is...@apache.org>
Authored: Tue Jan 17 02:42:16 2017 +0530
Committer: Ishan Chattopadhyaya <is...@apache.org>
Committed: Tue Jan 17 02:42:16 2017 +0530
----------------------------------------------------------------------
.../dot.settings/org.eclipse.jdt.core.prefs | 8 +-
lucene/CHANGES.txt | 29 ++--
.../analysis/ngram/EdgeNGramTokenFilter.java | 16 +-
.../lucene/analysis/ngram/NGramTokenFilter.java | 18 +--
.../lucene/analysis/ngram/TestNGramFilters.java | 47 ++++++
.../lucene/search/DoubleValuesSource.java | 92 +++++++++--
.../java/org/apache/lucene/util/Version.java | 7 +
.../queries/function/FunctionMatchQuery.java | 99 ++++++++++++
.../queries/function/FunctionScoreQuery.java | 151 +++++++++++++++++++
.../function/TestFunctionMatchQuery.java | 61 ++++++++
.../function/TestFunctionScoreExplanations.java | 105 +++++++++++++
.../function/TestFunctionScoreQuery.java | 114 ++++++++++++++
.../lucene/queryparser/xml/CoreParser.java | 2 -
solr/CHANGES.txt | 17 +++
.../ApacheLuceneSolrNearQueryBuilder.java | 1 -
.../apache/solr/search/GoodbyeQueryBuilder.java | 1 -
.../apache/solr/search/HandyQueryBuilder.java | 1 -
.../apache/solr/search/HelloQueryBuilder.java | 1 -
.../solr/cloud/AbstractDistribZkTestBase.java | 5 +-
19 files changed, 709 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
[04/14] lucene-solr:jira/solr-5944: add comment and test for ngram
token filter
Posted by is...@apache.org.
add comment and test for ngram token filter
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/80e28542
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/80e28542
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/80e28542
Branch: refs/heads/jira/solr-5944
Commit: 80e2854247cce485920b45acdeffa3e68bcea385
Parents: 01f2a87
Author: Nathan Gass <ga...@search.ch>
Authored: Fri Jan 13 16:42:41 2017 +0100
Committer: Nathan Gass <ga...@search.ch>
Committed: Fri Jan 13 16:42:41 2017 +0100
----------------------------------------------------------------------
.../lucene/analysis/ngram/TestNGramFilters.java | 25 ++++++++++++++++++++
1 file changed, 25 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/80e28542/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/TestNGramFilters.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/TestNGramFilters.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/TestNGramFilters.java
index b6f4405..5de532f 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/TestNGramFilters.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/TestNGramFilters.java
@@ -80,6 +80,28 @@ public class TestNGramFilters extends BaseTokenStreamFactoryTestCase {
}
/**
+ * Test NGramFilterFactory on tokens with payloads
+ */
+ public void testNGramFilterPayload() throws Exception {
+ Reader reader = new StringReader("test|0.1");
+ TokenStream stream = whitespaceMockTokenizer(reader);
+ stream = tokenFilterFactory("DelimitedPayload", "encoder", "float").create(stream);
+ stream = tokenFilterFactory("NGram", "minGramSize", "1", "maxGramSize", "2").create(stream);
+
+ stream.reset();
+ while (stream.incrementToken()) {
+ PayloadAttribute payAttr = stream.getAttribute(PayloadAttribute.class);
+ assertNotNull(payAttr);
+ BytesRef payData = payAttr.getPayload();
+ assertNotNull(payData);
+ float payFloat = PayloadHelper.decodeFloat(payData.bytes);
+ assertEquals(0.1f, payFloat, 0.0f);
+ }
+ stream.end();
+ stream.close();
+ }
+
+ /**
* Test EdgeNGramTokenizerFactory
*/
public void testEdgeNGramTokenizer() throws Exception {
@@ -127,6 +149,9 @@ public class TestNGramFilters extends BaseTokenStreamFactoryTestCase {
new String[] { "t", "te" });
}
+ /**
+ * Test EdgeNGramFilterFactory on tokens with payloads
+ */
public void testEdgeNGramFilterPayload() throws Exception {
Reader reader = new StringReader("test|0.1");
TokenStream stream = whitespaceMockTokenizer(reader);
[09/14] lucene-solr:jira/solr-5944: Fix precommit
Posted by is...@apache.org.
Fix precommit
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/478de2a5
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/478de2a5
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/478de2a5
Branch: refs/heads/jira/solr-5944
Commit: 478de2a59a12335bfecf16f9f39f99403d853848
Parents: fc2e0fd
Author: Alan Woodward <ro...@apache.org>
Authored: Mon Jan 16 11:41:59 2017 +0000
Committer: Alan Woodward <ro...@apache.org>
Committed: Mon Jan 16 11:42:36 2017 +0000
----------------------------------------------------------------------
.../java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java | 1 -
.../src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java | 1 -
2 files changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/478de2a5/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
index 47b80ff..56efd89 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
@@ -23,7 +23,6 @@ import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.util.AttributeSource.State;
/**
* Tokenizes the given token into n-grams of given size(s).
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/478de2a5/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
index cb5d447..a2e0aa7 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.miscellaneous.CodepointCountFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.util.AttributeSource.State;
/**
* Tokenizes the input into n-grams of the given size(s).