You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/06/16 22:34:04 UTC
svn commit: r1136656 [4/4] - in /lucene/dev/branches/flexscoring: ./
dev-tools/idea/.idea/ dev-tools/idea/lucene/ dev-tools/idea/lucene/contrib/
dev-tools/maven/ lucene/ lucene/backwards/ lucene/contrib/
lucene/contrib/misc/src/java/org/apache/lucene/i...
Modified: lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/BoostedQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/BoostedQuery.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/BoostedQuery.java (original)
+++ lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/BoostedQuery.java Thu Jun 16 20:33:59 2011
@@ -67,7 +67,7 @@ public class BoostedQuery extends Query
public BoostedWeight(IndexSearcher searcher) throws IOException {
this.searcher = searcher;
- this.qWeight = q.weight(searcher);
+ this.qWeight = q.createWeight(searcher);
this.fcontext = boostVal.newContext(searcher);
boostVal.createWeight(fcontext,searcher);
}
Modified: lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/QueryValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/QueryValueSource.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/QueryValueSource.java (original)
+++ lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/QueryValueSource.java Thu Jun 16 20:33:59 2011
@@ -68,7 +68,7 @@ public class QueryValueSource extends Va
@Override
public void createWeight(Map context, IndexSearcher searcher) throws IOException {
- Weight w = q.weight(searcher);
+ Weight w = searcher.createNormalizedWeight(q);
context.put(this, w);
}
}
@@ -98,7 +98,7 @@ class QueryDocValues extends FloatDocVal
this.q = vs.q;
this.fcontext = fcontext;
- Weight w = fcontext==null ? null : (Weight)fcontext.get(q);
+ Weight w = fcontext==null ? null : (Weight)fcontext.get(vs);
if (w == null) {
IndexSearcher weightSearcher;
if(fcontext == null) {
@@ -109,7 +109,8 @@ class QueryDocValues extends FloatDocVal
weightSearcher = new IndexSearcher(ReaderUtil.getTopLevelContext(readerContext));
}
}
- w = q.weight(weightSearcher);
+ vs.createWeight(fcontext, weightSearcher);
+ w = (Weight)fcontext.get(vs);
}
weight = w;
}
Modified: lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/SimpleBoolFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/SimpleBoolFunction.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/SimpleBoolFunction.java (original)
+++ lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/SimpleBoolFunction.java Thu Jun 16 20:33:59 2011
@@ -63,7 +63,7 @@ public abstract class SimpleBoolFunction
@Override
public boolean equals(Object o) {
if (this.getClass() != o.getClass()) return false;
- SingleFunction other = (SingleFunction)o;
+ SimpleBoolFunction other = (SimpleBoolFunction)o;
return this.source.equals(other.source);
}
Modified: lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/StrDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/StrDocValues.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/StrDocValues.java (original)
+++ lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/StrDocValues.java Thu Jun 16 20:33:59 2011
@@ -1,7 +1,6 @@
package org.apache.solr.search.function;
import org.apache.lucene.util.BytesRef;
-import org.apache.noggit.CharArr;
import org.apache.solr.search.MutableValue;
import org.apache.solr.search.MutableValueFloat;
import org.apache.solr.search.MutableValueStr;
Modified: lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/ValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/ValueSource.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/ValueSource.java (original)
+++ lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/ValueSource.java Thu Jun 16 20:33:59 2011
@@ -141,7 +141,7 @@ public abstract class ValueSource implem
* off of the {@link org.apache.solr.search.function.DocValues} for a ValueSource
* instead of the normal Lucene FieldComparator that works off of a FieldCache.
*/
- class ValueSourceComparator extends FieldComparator {
+ class ValueSourceComparator extends FieldComparator<Double> {
private final double[] values;
private DocValues docVals;
private double bottom;
@@ -195,7 +195,7 @@ public abstract class ValueSource implem
}
@Override
- public Comparable value(int slot) {
+ public Double value(int slot) {
return values[slot];
}
}
Modified: lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/handler/AnalysisRequestHandlerTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/handler/AnalysisRequestHandlerTestBase.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/handler/AnalysisRequestHandlerTestBase.java (original)
+++ lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/handler/AnalysisRequestHandlerTestBase.java Thu Jun 16 20:33:59 2011
@@ -19,6 +19,7 @@ package org.apache.solr.handler;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.util.NamedList;
+import org.apache.commons.lang.ArrayUtils;
/**
* A base class for all analysis request handler tests.
@@ -37,6 +38,7 @@ public abstract class AnalysisRequestHan
assertEquals(new Integer(info.getStart()), token.get("start"));
assertEquals(new Integer(info.getEnd()), token.get("end"));
assertEquals(new Integer(info.getPosition()), token.get("position"));
+ assertArrayEquals(info.getPositionHistory(), ArrayUtils.toPrimitive((Integer[]) token.get("positionHistory")));
if (info.isMatch()) {
assertEquals(Boolean.TRUE, token.get("match"));
}
@@ -57,6 +59,7 @@ public abstract class AnalysisRequestHan
private int end;
private String payload;
private int position;
+ private int[] positionHistory;
private boolean match;
public TokenInfo(
@@ -66,6 +69,7 @@ public abstract class AnalysisRequestHan
int start,
int end,
int position,
+ int[] positionHistory,
String payload,
boolean match) {
@@ -75,6 +79,7 @@ public abstract class AnalysisRequestHan
this.start = start;
this.end = end;
this.position = position;
+ this.positionHistory = positionHistory;
this.payload = payload;
this.match = match;
}
@@ -107,6 +112,10 @@ public abstract class AnalysisRequestHan
return position;
}
+ public int[] getPositionHistory() {
+ return positionHistory;
+ }
+
public boolean isMatch() {
return match;
}
Modified: lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java (original)
+++ lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java Thu Jun 16 20:33:59 2011
@@ -235,7 +235,7 @@ public class DocumentAnalysisRequestHand
assertTrue("Only the default analyzer should be applied", name.matches("org.apache.solr.schema.FieldType\\$DefaultAnalyzer.*"));
List<NamedList> tokenList = (List<NamedList>) queryResult.getVal(0);
assertEquals("Query has only one token", 1, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("JUMPING", null, "word", 0, 7, 1, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("JUMPING", null, "word", 0, 7, 1, new int[]{1}, null, false));
NamedList<Object> indexResult = idResult.get("index");
assertEquals("The id field has only a single value", 1, indexResult.size());
@@ -245,7 +245,7 @@ public class DocumentAnalysisRequestHand
assertTrue("Only the default analyzer should be applied", name.matches("org.apache.solr.schema.FieldType\\$DefaultAnalyzer.*"));
tokenList = valueResult.getVal(0);
assertEquals("The 'id' field value has only one token", 1, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("1", null, "word", 0, 1, 1, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("1", null, "word", 0, 1, 1, new int[]{1}, null, false));
***/
// the name field
@@ -255,14 +255,14 @@ public class DocumentAnalysisRequestHand
tokenList = (List<NamedList>) queryResult.get("org.apache.lucene.analysis.core.WhitespaceTokenizer");
assertNotNull("Expecting the 'WhitespaceTokenizer' to be applied on the query for the 'whitetok' field", tokenList);
assertEquals("Query has only one token", 1, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("JUMPING", null, "word", 0, 7, 1, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("JUMPING", null, "word", 0, 7, 1, new int[]{1}, null, false));
indexResult = whitetokResult.get("index");
assertEquals("The 'whitetok' field has only a single value", 1, indexResult.size());
valueResult = (NamedList<List<NamedList>>) indexResult.get("Jumping Jack");
tokenList = valueResult.getVal(0);
assertEquals("Expecting 2 tokens to be present", 2, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("Jumping", null, "word", 0, 7, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("Jack", null, "word", 8, 12, 2, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("Jumping", null, "word", 0, 7, 1, new int[]{1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("Jack", null, "word", 8, 12, 2, new int[]{2}, null, false));
// the text field
NamedList<NamedList<Object>> textResult = documentResult.get("text");
@@ -271,66 +271,66 @@ public class DocumentAnalysisRequestHand
tokenList = (List<NamedList>) queryResult.get("org.apache.lucene.analysis.standard.StandardTokenizer");
assertNotNull("Expecting the 'StandardTokenizer' to be applied on the query for the 'text' field", tokenList);
assertEquals("Query has only one token", 1, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("JUMPING", null, "<ALPHANUM>", 0, 7, 1, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("JUMPING", null, "<ALPHANUM>", 0, 7, 1, new int[]{1}, null, false));
tokenList = (List<NamedList>) queryResult.get("org.apache.lucene.analysis.standard.StandardFilter");
assertNotNull("Expecting the 'StandardFilter' to be applied on the query for the 'text' field", tokenList);
assertEquals("Query has only one token", 1, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("JUMPING", null, "<ALPHANUM>", 0, 7, 1, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("JUMPING", null, "<ALPHANUM>", 0, 7, 1, new int[]{1,1}, null, false));
tokenList = (List<NamedList>) queryResult.get("org.apache.lucene.analysis.core.LowerCaseFilter");
assertNotNull("Expecting the 'LowerCaseFilter' to be applied on the query for the 'text' field", tokenList);
assertEquals("Query has only one token", 1, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("jumping", null, "<ALPHANUM>", 0, 7, 1, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("jumping", null, "<ALPHANUM>", 0, 7, 1, new int[]{1,1,1}, null, false));
tokenList = (List<NamedList>) queryResult.get("org.apache.lucene.analysis.core.StopFilter");
assertNotNull("Expecting the 'StopFilter' to be applied on the query for the 'text' field", tokenList);
assertEquals("Query has only one token", 1, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("jumping", null, "<ALPHANUM>", 0, 7, 1, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("jumping", null, "<ALPHANUM>", 0, 7, 1, new int[]{1,1,1,1}, null, false));
tokenList = (List<NamedList>) queryResult.get("org.apache.lucene.analysis.en.PorterStemFilter");
assertNotNull("Expecting the 'PorterStemFilter' to be applied on the query for the 'text' field", tokenList);
assertEquals("Query has only one token", 1, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("jump", null, "<ALPHANUM>", 0, 7, 1, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("jump", null, "<ALPHANUM>", 0, 7, 1, new int[]{1,1,1,1,1}, null, false));
indexResult = textResult.get("index");
assertEquals("The 'text' field has only a single value", 1, indexResult.size());
valueResult = (NamedList<List<NamedList>>) indexResult.get("The Fox Jumped Over The Dogs");
tokenList = valueResult.get("org.apache.lucene.analysis.standard.StandardTokenizer");
assertNotNull("Expecting the 'StandardTokenizer' to be applied on the index for the 'text' field", tokenList);
assertEquals("Expecting 6 tokens", 6, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("The", null, "<ALPHANUM>", 0, 3, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("Fox", null, "<ALPHANUM>", 4, 7, 2, null, false));
- assertToken(tokenList.get(2), new TokenInfo("Jumped", null, "<ALPHANUM>", 8, 14, 3, null, false));
- assertToken(tokenList.get(3), new TokenInfo("Over", null, "<ALPHANUM>", 15, 19, 4, null, false));
- assertToken(tokenList.get(4), new TokenInfo("The", null, "<ALPHANUM>", 20, 23, 5, null, false));
- assertToken(tokenList.get(5), new TokenInfo("Dogs", null, "<ALPHANUM>", 24, 28, 6, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("The", null, "<ALPHANUM>", 0, 3, 1, new int[]{1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("Fox", null, "<ALPHANUM>", 4, 7, 2, new int[]{2}, null, false));
+ assertToken(tokenList.get(2), new TokenInfo("Jumped", null, "<ALPHANUM>", 8, 14, 3, new int[]{3}, null, false));
+ assertToken(tokenList.get(3), new TokenInfo("Over", null, "<ALPHANUM>", 15, 19, 4, new int[]{4}, null, false));
+ assertToken(tokenList.get(4), new TokenInfo("The", null, "<ALPHANUM>", 20, 23, 5, new int[]{5}, null, false));
+ assertToken(tokenList.get(5), new TokenInfo("Dogs", null, "<ALPHANUM>", 24, 28, 6, new int[]{6}, null, false));
tokenList = valueResult.get("org.apache.lucene.analysis.standard.StandardFilter");
assertNotNull("Expecting the 'StandardFilter' to be applied on the index for the 'text' field", tokenList);
assertEquals("Expecting 6 tokens", 6, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("The", null, "<ALPHANUM>", 0, 3, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("Fox", null, "<ALPHANUM>", 4, 7, 2, null, false));
- assertToken(tokenList.get(2), new TokenInfo("Jumped", null, "<ALPHANUM>", 8, 14, 3, null, false));
- assertToken(tokenList.get(3), new TokenInfo("Over", null, "<ALPHANUM>", 15, 19, 4, null, false));
- assertToken(tokenList.get(4), new TokenInfo("The", null, "<ALPHANUM>", 20, 23, 5, null, false));
- assertToken(tokenList.get(5), new TokenInfo("Dogs", null, "<ALPHANUM>", 24, 28, 6, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("The", null, "<ALPHANUM>", 0, 3, 1, new int[]{1,1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("Fox", null, "<ALPHANUM>", 4, 7, 2, new int[]{2,2}, null, false));
+ assertToken(tokenList.get(2), new TokenInfo("Jumped", null, "<ALPHANUM>", 8, 14, 3, new int[]{3,3}, null, false));
+ assertToken(tokenList.get(3), new TokenInfo("Over", null, "<ALPHANUM>", 15, 19, 4, new int[]{4,4}, null, false));
+ assertToken(tokenList.get(4), new TokenInfo("The", null, "<ALPHANUM>", 20, 23, 5, new int[]{5,5}, null, false));
+ assertToken(tokenList.get(5), new TokenInfo("Dogs", null, "<ALPHANUM>", 24, 28, 6, new int[]{6,6}, null, false));
tokenList = valueResult.get("org.apache.lucene.analysis.core.LowerCaseFilter");
assertNotNull("Expecting the 'LowerCaseFilter' to be applied on the index for the 'text' field", tokenList);
assertEquals("Expecting 6 tokens", 6, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("the", null, "<ALPHANUM>", 0, 3, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("fox", null, "<ALPHANUM>", 4, 7, 2, null, false));
- assertToken(tokenList.get(2), new TokenInfo("jumped", null, "<ALPHANUM>", 8, 14, 3, null, false));
- assertToken(tokenList.get(3), new TokenInfo("over", null, "<ALPHANUM>", 15, 19, 4, null, false));
- assertToken(tokenList.get(4), new TokenInfo("the", null, "<ALPHANUM>", 20, 23, 5, null, false));
- assertToken(tokenList.get(5), new TokenInfo("dogs", null, "<ALPHANUM>", 24, 28, 6, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("the", null, "<ALPHANUM>", 0, 3, 1, new int[]{1,1,1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("fox", null, "<ALPHANUM>", 4, 7, 2, new int[]{2,2,2}, null, false));
+ assertToken(tokenList.get(2), new TokenInfo("jumped", null, "<ALPHANUM>", 8, 14, 3, new int[]{3,3,3}, null, false));
+ assertToken(tokenList.get(3), new TokenInfo("over", null, "<ALPHANUM>", 15, 19, 4, new int[]{4,4,4}, null, false));
+ assertToken(tokenList.get(4), new TokenInfo("the", null, "<ALPHANUM>", 20, 23, 5, new int[]{5,5,5}, null, false));
+ assertToken(tokenList.get(5), new TokenInfo("dogs", null, "<ALPHANUM>", 24, 28, 6, new int[]{6,6,6}, null, false));
tokenList = valueResult.get("org.apache.lucene.analysis.core.StopFilter");
assertNotNull("Expecting the 'StopFilter' to be applied on the index for the 'text' field", tokenList);
assertEquals("Expecting 4 tokens after stop word removal", 4, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("fox", null, "<ALPHANUM>", 4, 7, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("jumped", null, "<ALPHANUM>", 8, 14, 2, null, false));
- assertToken(tokenList.get(2), new TokenInfo("over", null, "<ALPHANUM>", 15, 19, 3, null, false));
- assertToken(tokenList.get(3), new TokenInfo("dogs", null, "<ALPHANUM>", 24, 28, 4, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("fox", null, "<ALPHANUM>", 4, 7, 1, new int[]{2,2,2,1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("jumped", null, "<ALPHANUM>", 8, 14, 2, new int[]{3,3,3,2}, null, false));
+ assertToken(tokenList.get(2), new TokenInfo("over", null, "<ALPHANUM>", 15, 19, 3, new int[]{4,4,4,3}, null, false));
+ assertToken(tokenList.get(3), new TokenInfo("dogs", null, "<ALPHANUM>", 24, 28, 4, new int[]{6,6,6,4}, null, false));
tokenList = valueResult.get("org.apache.lucene.analysis.en.PorterStemFilter");
assertNotNull("Expecting the 'PorterStemFilter' to be applied on the index for the 'text' field", tokenList);
assertEquals("Expecting 4 tokens", 4, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("fox", null, "<ALPHANUM>", 4, 7, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("jump", null, "<ALPHANUM>", 8, 14, 2, null, true));
- assertToken(tokenList.get(2), new TokenInfo("over", null, "<ALPHANUM>", 15, 19, 3, null, false));
- assertToken(tokenList.get(3), new TokenInfo("dog", null, "<ALPHANUM>", 24, 28, 4, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("fox", null, "<ALPHANUM>", 4, 7, 1, new int[]{2,2,2,1,1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("jump", null, "<ALPHANUM>", 8, 14, 2, new int[]{3,3,3,2,2}, null, true));
+ assertToken(tokenList.get(2), new TokenInfo("over", null, "<ALPHANUM>", 15, 19, 3, new int[]{4,4,4,3,3}, null, false));
+ assertToken(tokenList.get(3), new TokenInfo("dog", null, "<ALPHANUM>", 24, 28, 4, new int[]{6,6,6,4,4}, null, false));
}
}
Modified: lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java (original)
+++ lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java Thu Jun 16 20:33:59 2011
@@ -139,64 +139,64 @@ public class FieldAnalysisRequestHandler
List<NamedList> tokenList = indexPart.get("org.apache.lucene.analysis.standard.StandardTokenizer");
assertNotNull("Expcting StandardTokenizer analysis breakdown", tokenList);
assertEquals(tokenList.size(), 10);
- assertToken(tokenList.get(0), new TokenInfo("the", null, "<ALPHANUM>", 0, 3, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("quick", null, "<ALPHANUM>", 4, 9, 2, null, false));
- assertToken(tokenList.get(2), new TokenInfo("red", null, "<ALPHANUM>", 10, 13, 3, null, false));
- assertToken(tokenList.get(3), new TokenInfo("fox", null, "<ALPHANUM>", 14, 17, 4, null, true));
- assertToken(tokenList.get(4), new TokenInfo("jumped", null, "<ALPHANUM>", 18, 24, 5, null, false));
- assertToken(tokenList.get(5), new TokenInfo("over", null, "<ALPHANUM>", 25, 29, 6, null, false));
- assertToken(tokenList.get(6), new TokenInfo("the", null, "<ALPHANUM>", 30, 33, 7, null, false));
- assertToken(tokenList.get(7), new TokenInfo("lazy", null, "<ALPHANUM>", 34, 38, 8, null, false));
- assertToken(tokenList.get(8), new TokenInfo("brown", null, "<ALPHANUM>", 39, 44, 9, null, true));
- assertToken(tokenList.get(9), new TokenInfo("dogs", null, "<ALPHANUM>", 45, 49, 10, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("the", null, "<ALPHANUM>", 0, 3, 1, new int[]{1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("quick", null, "<ALPHANUM>", 4, 9, 2, new int[]{2}, null, false));
+ assertToken(tokenList.get(2), new TokenInfo("red", null, "<ALPHANUM>", 10, 13, 3, new int[]{3}, null, false));
+ assertToken(tokenList.get(3), new TokenInfo("fox", null, "<ALPHANUM>", 14, 17, 4, new int[]{4}, null, true));
+ assertToken(tokenList.get(4), new TokenInfo("jumped", null, "<ALPHANUM>", 18, 24, 5, new int[]{5}, null, false));
+ assertToken(tokenList.get(5), new TokenInfo("over", null, "<ALPHANUM>", 25, 29, 6, new int[]{6}, null, false));
+ assertToken(tokenList.get(6), new TokenInfo("the", null, "<ALPHANUM>", 30, 33, 7, new int[]{7}, null, false));
+ assertToken(tokenList.get(7), new TokenInfo("lazy", null, "<ALPHANUM>", 34, 38, 8, new int[]{8}, null, false));
+ assertToken(tokenList.get(8), new TokenInfo("brown", null, "<ALPHANUM>", 39, 44, 9, new int[]{9}, null, true));
+ assertToken(tokenList.get(9), new TokenInfo("dogs", null, "<ALPHANUM>", 45, 49, 10, new int[]{10}, null, false));
tokenList = indexPart.get("org.apache.lucene.analysis.standard.StandardFilter");
assertNotNull("Expcting StandardFilter analysis breakdown", tokenList);
assertEquals(tokenList.size(), 10);
- assertToken(tokenList.get(0), new TokenInfo("the", null, "<ALPHANUM>", 0, 3, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("quick", null, "<ALPHANUM>", 4, 9, 2, null, false));
- assertToken(tokenList.get(2), new TokenInfo("red", null, "<ALPHANUM>", 10, 13, 3, null, false));
- assertToken(tokenList.get(3), new TokenInfo("fox", null, "<ALPHANUM>", 14, 17, 4, null, true));
- assertToken(tokenList.get(4), new TokenInfo("jumped", null, "<ALPHANUM>", 18, 24, 5, null, false));
- assertToken(tokenList.get(5), new TokenInfo("over", null, "<ALPHANUM>", 25, 29, 6, null, false));
- assertToken(tokenList.get(6), new TokenInfo("the", null, "<ALPHANUM>", 30, 33, 7, null, false));
- assertToken(tokenList.get(7), new TokenInfo("lazy", null, "<ALPHANUM>", 34, 38, 8, null, false));
- assertToken(tokenList.get(8), new TokenInfo("brown", null, "<ALPHANUM>", 39, 44, 9, null, true));
- assertToken(tokenList.get(9), new TokenInfo("dogs", null, "<ALPHANUM>", 45, 49, 10, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("the", null, "<ALPHANUM>", 0, 3, 1, new int[]{1,1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("quick", null, "<ALPHANUM>", 4, 9, 2, new int[]{2,2}, null, false));
+ assertToken(tokenList.get(2), new TokenInfo("red", null, "<ALPHANUM>", 10, 13, 3, new int[]{3,3}, null, false));
+ assertToken(tokenList.get(3), new TokenInfo("fox", null, "<ALPHANUM>", 14, 17, 4, new int[]{4,4}, null, true));
+ assertToken(tokenList.get(4), new TokenInfo("jumped", null, "<ALPHANUM>", 18, 24, 5, new int[]{5,5}, null, false));
+ assertToken(tokenList.get(5), new TokenInfo("over", null, "<ALPHANUM>", 25, 29, 6, new int[]{6,6}, null, false));
+ assertToken(tokenList.get(6), new TokenInfo("the", null, "<ALPHANUM>", 30, 33, 7, new int[]{7,7}, null, false));
+ assertToken(tokenList.get(7), new TokenInfo("lazy", null, "<ALPHANUM>", 34, 38, 8, new int[]{8,8}, null, false));
+ assertToken(tokenList.get(8), new TokenInfo("brown", null, "<ALPHANUM>", 39, 44, 9, new int[]{9,9}, null, true));
+ assertToken(tokenList.get(9), new TokenInfo("dogs", null, "<ALPHANUM>", 45, 49, 10, new int[]{10,10}, null, false));
tokenList = indexPart.get("org.apache.lucene.analysis.core.LowerCaseFilter");
assertNotNull("Expcting LowerCaseFilter analysis breakdown", tokenList);
assertEquals(tokenList.size(), 10);
- assertToken(tokenList.get(0), new TokenInfo("the", null, "<ALPHANUM>", 0, 3, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("quick", null, "<ALPHANUM>", 4, 9, 2, null, false));
- assertToken(tokenList.get(2), new TokenInfo("red", null, "<ALPHANUM>", 10, 13, 3, null, false));
- assertToken(tokenList.get(3), new TokenInfo("fox", null, "<ALPHANUM>", 14, 17, 4, null, true));
- assertToken(tokenList.get(4), new TokenInfo("jumped", null, "<ALPHANUM>", 18, 24, 5, null, false));
- assertToken(tokenList.get(5), new TokenInfo("over", null, "<ALPHANUM>", 25, 29, 6, null, false));
- assertToken(tokenList.get(6), new TokenInfo("the", null, "<ALPHANUM>", 30, 33, 7, null, false));
- assertToken(tokenList.get(7), new TokenInfo("lazy", null, "<ALPHANUM>", 34, 38, 8, null, false));
- assertToken(tokenList.get(8), new TokenInfo("brown", null, "<ALPHANUM>", 39, 44, 9, null, true));
- assertToken(tokenList.get(9), new TokenInfo("dogs", null, "<ALPHANUM>", 45, 49, 10, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("the", null, "<ALPHANUM>", 0, 3, 1, new int[]{1,1,1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("quick", null, "<ALPHANUM>", 4, 9, 2, new int[]{2,2,2}, null, false));
+ assertToken(tokenList.get(2), new TokenInfo("red", null, "<ALPHANUM>", 10, 13, 3, new int[]{3,3,3}, null, false));
+ assertToken(tokenList.get(3), new TokenInfo("fox", null, "<ALPHANUM>", 14, 17, 4, new int[]{4,4,4}, null, true));
+ assertToken(tokenList.get(4), new TokenInfo("jumped", null, "<ALPHANUM>", 18, 24, 5, new int[]{5,5,5}, null, false));
+ assertToken(tokenList.get(5), new TokenInfo("over", null, "<ALPHANUM>", 25, 29, 6, new int[]{6,6,6}, null, false));
+ assertToken(tokenList.get(6), new TokenInfo("the", null, "<ALPHANUM>", 30, 33, 7, new int[]{7,7,7}, null, false));
+ assertToken(tokenList.get(7), new TokenInfo("lazy", null, "<ALPHANUM>", 34, 38, 8, new int[]{8,8,8}, null, false));
+ assertToken(tokenList.get(8), new TokenInfo("brown", null, "<ALPHANUM>", 39, 44, 9, new int[]{9,9,9}, null, true));
+ assertToken(tokenList.get(9), new TokenInfo("dogs", null, "<ALPHANUM>", 45, 49, 10, new int[]{10,10,10}, null, false));
tokenList = indexPart.get("org.apache.lucene.analysis.core.StopFilter");
assertNotNull("Expcting StopFilter analysis breakdown", tokenList);
assertEquals(tokenList.size(), 8);
- assertToken(tokenList.get(0), new TokenInfo("quick", null, "<ALPHANUM>", 4, 9, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("red", null, "<ALPHANUM>", 10, 13, 2, null, false));
- assertToken(tokenList.get(2), new TokenInfo("fox", null, "<ALPHANUM>", 14, 17, 3, null, true));
- assertToken(tokenList.get(3), new TokenInfo("jumped", null, "<ALPHANUM>", 18, 24, 4, null, false));
- assertToken(tokenList.get(4), new TokenInfo("over", null, "<ALPHANUM>", 25, 29, 5, null, false));
- assertToken(tokenList.get(5), new TokenInfo("lazy", null, "<ALPHANUM>", 34, 38, 6, null, false));
- assertToken(tokenList.get(6), new TokenInfo("brown", null, "<ALPHANUM>", 39, 44, 7, null, true));
- assertToken(tokenList.get(7), new TokenInfo("dogs", null, "<ALPHANUM>", 45, 49, 8, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("quick", null, "<ALPHANUM>", 4, 9, 1, new int[]{2,2,2,1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("red", null, "<ALPHANUM>", 10, 13, 2, new int[]{3,3,3,2}, null, false));
+ assertToken(tokenList.get(2), new TokenInfo("fox", null, "<ALPHANUM>", 14, 17, 3, new int[]{4,4,4,3}, null, true));
+ assertToken(tokenList.get(3), new TokenInfo("jumped", null, "<ALPHANUM>", 18, 24, 4, new int[]{5,5,5,4}, null, false));
+ assertToken(tokenList.get(4), new TokenInfo("over", null, "<ALPHANUM>", 25, 29, 5, new int[]{6,6,6,5}, null, false));
+ assertToken(tokenList.get(5), new TokenInfo("lazy", null, "<ALPHANUM>", 34, 38, 6, new int[]{8,8,8,6}, null, false));
+ assertToken(tokenList.get(6), new TokenInfo("brown", null, "<ALPHANUM>", 39, 44, 7, new int[]{9,9,9,7}, null, true));
+ assertToken(tokenList.get(7), new TokenInfo("dogs", null, "<ALPHANUM>", 45, 49, 8, new int[]{10,10,10,8}, null, false));
tokenList = indexPart.get("org.apache.lucene.analysis.en.PorterStemFilter");
assertNotNull("Expcting PorterStemFilter analysis breakdown", tokenList);
assertEquals(tokenList.size(), 8);
- assertToken(tokenList.get(0), new TokenInfo("quick", null, "<ALPHANUM>", 4, 9, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("red", null, "<ALPHANUM>", 10, 13, 2, null, false));
- assertToken(tokenList.get(2), new TokenInfo("fox", null, "<ALPHANUM>", 14, 17, 3, null, true));
- assertToken(tokenList.get(3), new TokenInfo("jump", null, "<ALPHANUM>", 18, 24, 4, null, false));
- assertToken(tokenList.get(4), new TokenInfo("over", null, "<ALPHANUM>", 25, 29, 5, null, false));
- assertToken(tokenList.get(5), new TokenInfo("lazi", null, "<ALPHANUM>", 34, 38, 6, null, false));
- assertToken(tokenList.get(6), new TokenInfo("brown", null, "<ALPHANUM>", 39, 44, 7, null, true));
- assertToken(tokenList.get(7), new TokenInfo("dog", null, "<ALPHANUM>", 45, 49, 8, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("quick", null, "<ALPHANUM>", 4, 9, 1, new int[]{2,2,2,1,1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("red", null, "<ALPHANUM>", 10, 13, 2, new int[]{3,3,3,2,2}, null, false));
+ assertToken(tokenList.get(2), new TokenInfo("fox", null, "<ALPHANUM>", 14, 17, 3, new int[]{4,4,4,3,3}, null, true));
+ assertToken(tokenList.get(3), new TokenInfo("jump", null, "<ALPHANUM>", 18, 24, 4, new int[]{5,5,5,4,4}, null, false));
+ assertToken(tokenList.get(4), new TokenInfo("over", null, "<ALPHANUM>", 25, 29, 5, new int[]{6,6,6,5,5}, null, false));
+ assertToken(tokenList.get(5), new TokenInfo("lazi", null, "<ALPHANUM>", 34, 38, 6, new int[]{8,8,8,6,6}, null, false));
+ assertToken(tokenList.get(6), new TokenInfo("brown", null, "<ALPHANUM>", 39, 44, 7, new int[]{9,9,9,7,7}, null, true));
+ assertToken(tokenList.get(7), new TokenInfo("dog", null, "<ALPHANUM>", 45, 49, 8, new int[]{10,10,10,8,8}, null, false));
NamedList<List<NamedList>> queryPart = textType.get("query");
assertNotNull("expecting a query token analysis for field type 'text'", queryPart);
@@ -204,28 +204,28 @@ public class FieldAnalysisRequestHandler
tokenList = queryPart.get("org.apache.lucene.analysis.standard.StandardTokenizer");
assertNotNull("Expecting StandardTokenizer analysis breakdown", tokenList);
assertEquals("Expecting StandardTokenizer to produce 2 tokens from '" + request.getQuery() + "'", 2, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("fox", null, "<ALPHANUM>", 0, 3, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("brown", null, "<ALPHANUM>", 4, 9, 2, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("fox", null, "<ALPHANUM>", 0, 3, 1, new int[]{1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("brown", null, "<ALPHANUM>", 4, 9, 2, new int[]{2}, null, false));
tokenList = queryPart.get("org.apache.lucene.analysis.standard.StandardFilter");
assertNotNull("Expcting StandardFilter analysis breakdown", tokenList);
assertEquals(2, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("fox", null, "<ALPHANUM>", 0, 3, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("brown", null, "<ALPHANUM>", 4, 9, 2, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("fox", null, "<ALPHANUM>", 0, 3, 1, new int[]{1,1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("brown", null, "<ALPHANUM>", 4, 9, 2, new int[]{2,2}, null, false));
tokenList = queryPart.get("org.apache.lucene.analysis.core.LowerCaseFilter");
assertNotNull("Expcting LowerCaseFilter analysis breakdown", tokenList);
assertEquals(2, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("fox", null, "<ALPHANUM>", 0, 3, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("brown", null, "<ALPHANUM>", 4, 9, 2, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("fox", null, "<ALPHANUM>", 0, 3, 1, new int[]{1,1,1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("brown", null, "<ALPHANUM>", 4, 9, 2, new int[]{2,2,2}, null, false));
tokenList = queryPart.get("org.apache.lucene.analysis.core.StopFilter");
assertNotNull("Expcting StopFilter analysis breakdown", tokenList);
assertEquals(2, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("fox", null, "<ALPHANUM>", 0, 3, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("brown", null, "<ALPHANUM>", 4, 9, 2, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("fox", null, "<ALPHANUM>", 0, 3, 1, new int[]{1,1,1,1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("brown", null, "<ALPHANUM>", 4, 9, 2, new int[]{2,2,2,2}, null, false));
tokenList = queryPart.get("org.apache.lucene.analysis.en.PorterStemFilter");
assertNotNull("Expcting PorterStemFilter analysis breakdown", tokenList);
assertEquals(2, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("fox", null, "<ALPHANUM>", 0, 3, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("brown", null, "<ALPHANUM>", 4, 9, 2, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("fox", null, "<ALPHANUM>", 0, 3, 1, new int[]{1,1,1,1,1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("brown", null, "<ALPHANUM>", 4, 9, 2, new int[]{2,2,2,2,2}, null, false));
NamedList<NamedList> nameTextType = fieldTypes.get("nametext");
assertNotNull("expecting result for field type 'nametext'", nameTextType);
@@ -236,22 +236,22 @@ public class FieldAnalysisRequestHandler
tokenList = indexPart.get("org.apache.lucene.analysis.core.WhitespaceTokenizer");
assertNotNull("Expcting WhitespaceTokenizer analysis breakdown", tokenList);
assertEquals(10, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("the", null, "word", 0, 3, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("quick", null, "word", 4, 9, 2, null, false));
- assertToken(tokenList.get(2), new TokenInfo("red", null, "word", 10, 13, 3, null, false));
- assertToken(tokenList.get(3), new TokenInfo("fox", null, "word", 14, 17, 4, null, true));
- assertToken(tokenList.get(4), new TokenInfo("jumped", null, "word", 18, 24, 5, null, false));
- assertToken(tokenList.get(5), new TokenInfo("over", null, "word", 25, 29, 6, null, false));
- assertToken(tokenList.get(6), new TokenInfo("the", null, "word", 30, 33, 7, null, false));
- assertToken(tokenList.get(7), new TokenInfo("lazy", null, "word", 34, 38, 8, null, false));
- assertToken(tokenList.get(8), new TokenInfo("brown", null, "word", 39, 44, 9, null, true));
- assertToken(tokenList.get(9), new TokenInfo("dogs", null, "word", 45, 49, 10, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("the", null, "word", 0, 3, 1, new int[]{1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("quick", null, "word", 4, 9, 2, new int[]{2}, null, false));
+ assertToken(tokenList.get(2), new TokenInfo("red", null, "word", 10, 13, 3, new int[]{3}, null, false));
+ assertToken(tokenList.get(3), new TokenInfo("fox", null, "word", 14, 17, 4, new int[]{4}, null, true));
+ assertToken(tokenList.get(4), new TokenInfo("jumped", null, "word", 18, 24, 5, new int[]{5}, null, false));
+ assertToken(tokenList.get(5), new TokenInfo("over", null, "word", 25, 29, 6, new int[]{6}, null, false));
+ assertToken(tokenList.get(6), new TokenInfo("the", null, "word", 30, 33, 7, new int[]{7}, null, false));
+ assertToken(tokenList.get(7), new TokenInfo("lazy", null, "word", 34, 38, 8, new int[]{8}, null, false));
+ assertToken(tokenList.get(8), new TokenInfo("brown", null, "word", 39, 44, 9, new int[]{9}, null, true));
+ assertToken(tokenList.get(9), new TokenInfo("dogs", null, "word", 45, 49, 10, new int[]{10}, null, false));
queryPart = nameTextType.get("query");
assertNotNull("expecting a query token analysis for field type 'nametext'", queryPart);
tokenList = queryPart.get(WhitespaceTokenizer.class.getName());
- assertToken(tokenList.get(0), new TokenInfo("fox", null, "word", 0, 3, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("brown", null, "word", 4, 9, 2, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("fox", null, "word", 0, 3, 1, new int[]{1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("brown", null, "word", 4, 9, 2, new int[]{2}, null, false));
NamedList<NamedList> fieldNames = result.get("field_names");
assertNotNull("field_nameds should never be null", fieldNames);
@@ -265,16 +265,16 @@ public class FieldAnalysisRequestHandler
tokenList = indexPart.get(WhitespaceTokenizer.class.getName());
assertNotNull("expecting only WhitespaceTokenizer to be applied", tokenList);
assertEquals("expecting WhitespaceTokenizer to produce 10 tokens", 10, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("the", null, "word", 0, 3, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("quick", null, "word", 4, 9, 2, null, false));
- assertToken(tokenList.get(2), new TokenInfo("red", null, "word", 10, 13, 3, null, false));
- assertToken(tokenList.get(3), new TokenInfo("fox", null, "word", 14, 17, 4, null, true));
- assertToken(tokenList.get(4), new TokenInfo("jumped", null, "word", 18, 24, 5, null, false));
- assertToken(tokenList.get(5), new TokenInfo("over", null, "word", 25, 29, 6, null, false));
- assertToken(tokenList.get(6), new TokenInfo("the", null, "word", 30, 33, 7, null, false));
- assertToken(tokenList.get(7), new TokenInfo("lazy", null, "word", 34, 38, 8, null, false));
- assertToken(tokenList.get(8), new TokenInfo("brown", null, "word", 39, 44, 9, null, true));
- assertToken(tokenList.get(9), new TokenInfo("dogs", null, "word", 45, 49, 10, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("the", null, "word", 0, 3, 1, new int[]{1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("quick", null, "word", 4, 9, 2, new int[]{2}, null, false));
+ assertToken(tokenList.get(2), new TokenInfo("red", null, "word", 10, 13, 3, new int[]{3}, null, false));
+ assertToken(tokenList.get(3), new TokenInfo("fox", null, "word", 14, 17, 4, new int[]{4}, null, true));
+ assertToken(tokenList.get(4), new TokenInfo("jumped", null, "word", 18, 24, 5, new int[]{5}, null, false));
+ assertToken(tokenList.get(5), new TokenInfo("over", null, "word", 25, 29, 6, new int[]{6}, null, false));
+ assertToken(tokenList.get(6), new TokenInfo("the", null, "word", 30, 33, 7, new int[]{7}, null, false));
+ assertToken(tokenList.get(7), new TokenInfo("lazy", null, "word", 34, 38, 8, new int[]{8}, null, false));
+ assertToken(tokenList.get(8), new TokenInfo("brown", null, "word", 39, 44, 9, new int[]{9}, null, true));
+ assertToken(tokenList.get(9), new TokenInfo("dogs", null, "word", 45, 49, 10, new int[]{10}, null, false));
queryPart = whitetok.get("query");
assertNotNull("expecting a query token analysis for field 'whitetok'", queryPart);
@@ -282,8 +282,8 @@ public class FieldAnalysisRequestHandler
tokenList = queryPart.get(WhitespaceTokenizer.class.getName());
assertNotNull("expecting only WhitespaceTokenizer to be applied", tokenList);
assertEquals("expecting WhitespaceTokenizer to produce 2 tokens", 2, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("fox", null, "word", 0, 3, 1, null, false));
- assertToken(tokenList.get(1), new TokenInfo("brown", null, "word", 4, 9, 2, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("fox", null, "word", 0, 3, 1, new int[]{1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("brown", null, "word", 4, 9, 2, new int[]{2}, null, false));
NamedList<NamedList> keywordtok = fieldNames.get("keywordtok");
assertNotNull("expecting result for field 'keywordtok'", keywordtok);
@@ -294,7 +294,7 @@ public class FieldAnalysisRequestHandler
tokenList = indexPart.get(KeywordTokenizer.class.getName());
assertNotNull("expecting only KeywordTokenizer to be applied", tokenList);
assertEquals("expecting KeywordTokenizer to produce 1 token", 1, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("the quick red fox jumped over the lazy brown dogs", null, "word", 0, 49, 1, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("the quick red fox jumped over the lazy brown dogs", null, "word", 0, 49, 1, new int[]{1}, null, false));
queryPart = keywordtok.get("query");
assertNotNull("expecting a query token analysis for field 'keywordtok'", queryPart);
@@ -302,7 +302,7 @@ public class FieldAnalysisRequestHandler
tokenList = queryPart.get(KeywordTokenizer.class.getName());
assertNotNull("expecting only KeywordTokenizer to be applied", tokenList);
assertEquals("expecting KeywordTokenizer to produce 1 token", 1, tokenList.size());
- assertToken(tokenList.get(0), new TokenInfo("fox brown", null, "word", 0, 9, 1, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("fox brown", null, "word", 0, 9, 1, new int[]{1}, null, false));
}
@@ -331,6 +331,53 @@ public class FieldAnalysisRequestHandler
List<NamedList> tokenList = (List<NamedList>)indexPart.get("org.apache.lucene.analysis.core.WhitespaceTokenizer");
assertNotNull("Expecting WhitespaceTokenizer analysis breakdown", tokenList);
assertEquals(tokenList.size(), 1);
- assertToken(tokenList.get(0), new TokenInfo("whatever", null, "word", 12, 20, 1, null, false));
+ assertToken(tokenList.get(0), new TokenInfo("whatever", null, "word", 12, 20, 1, new int[]{1}, null, false));
}
+
+ @Test
+ public void testPositionHistoryWithWDF() throws Exception {
+
+ FieldAnalysisRequest request = new FieldAnalysisRequest();
+ request.addFieldType("skutype1");
+ request.setFieldValue("hi, 3456-12 a Test");
+ request.setShowMatch(false);
+
+ NamedList<NamedList> result = handler.handleAnalysisRequest(request, h.getCore().getSchema());
+ assertTrue("result is null and it shouldn't be", result != null);
+
+ NamedList<NamedList> fieldTypes = result.get("field_types");
+ assertNotNull("field_types should never be null", fieldTypes);
+ NamedList<NamedList> textType = fieldTypes.get("skutype1");
+ assertNotNull("expecting result for field type 'skutype1'", textType);
+
+ NamedList<List<NamedList>> indexPart = textType.get("index");
+ assertNotNull("expecting an index token analysis for field type 'skutype1'", indexPart);
+
+ List<NamedList> tokenList = indexPart.get("org.apache.lucene.analysis.core.WhitespaceTokenizer");
+ assertNotNull("Expcting WhitespaceTokenizer analysis breakdown", tokenList);
+ assertEquals(4, tokenList.size());
+ assertToken(tokenList.get(0), new TokenInfo("hi,", null, "word", 0, 3, 1, new int[]{1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("3456-12", null, "word", 4, 11, 2, new int[]{2}, null, false));
+ assertToken(tokenList.get(2), new TokenInfo("a", null, "word", 12, 13, 3, new int[]{3}, null, false));
+ assertToken(tokenList.get(3), new TokenInfo("Test", null, "word", 14, 18, 4, new int[]{4}, null, false));
+ tokenList = indexPart.get("org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter");
+ assertNotNull("Expcting WordDelimiterFilter analysis breakdown", tokenList);
+ assertEquals(6, tokenList.size());
+ assertToken(tokenList.get(0), new TokenInfo("hi", null, "word", 0, 2, 1, new int[]{1,1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("3456", null, "word", 4, 8, 2, new int[]{2,2}, null, false));
+ assertToken(tokenList.get(2), new TokenInfo("12", null, "word", 9, 11, 3, new int[]{2,3}, null, false));
+ assertToken(tokenList.get(3), new TokenInfo("345612", null, "word", 4, 11, 3, new int[]{2,3}, null, false));
+ assertToken(tokenList.get(4), new TokenInfo("a", null, "word", 12, 13, 4, new int[]{3,4}, null, false));
+ assertToken(tokenList.get(5), new TokenInfo("Test", null, "word", 14, 18, 5, new int[]{4,5}, null, false));
+ tokenList = indexPart.get("org.apache.lucene.analysis.core.LowerCaseFilter");
+ assertNotNull("Expcting LowerCaseFilter analysis breakdown", tokenList);
+ assertEquals(6, tokenList.size());
+ assertToken(tokenList.get(0), new TokenInfo("hi", null, "word", 0, 2, 1, new int[]{1,1,1}, null, false));
+ assertToken(tokenList.get(1), new TokenInfo("3456", null, "word", 4, 8, 2, new int[]{2,2,2}, null, false));
+ assertToken(tokenList.get(2), new TokenInfo("12", null, "word", 9, 11, 3, new int[]{2,3,3}, null, false));
+ assertToken(tokenList.get(3), new TokenInfo("345612", null, "word", 4, 11, 3, new int[]{2,3,3}, null, false));
+ assertToken(tokenList.get(4), new TokenInfo("a", null, "word", 12, 13, 4, new int[]{3,4,4}, null, false));
+ assertToken(tokenList.get(5), new TokenInfo("test", null, "word", 14, 18, 5, new int[]{4,5,5}, null, false));
+ }
+
}