You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/05/07 13:21:14 UTC
svn commit: r1479862 [23/38] - in /lucene/dev/branches/lucene4258: ./
dev-tools/ dev-tools/idea/.idea/ dev-tools/idea/.idea/libraries/
dev-tools/maven/ dev-tools/maven/solr/ dev-tools/maven/solr/core/src/java/
dev-tools/maven/solr/solrj/src/java/ dev-t...
Modified: lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java Tue May 7 11:20:55 2013
@@ -83,7 +83,7 @@ public class TestFieldScoreQuery extends
private void doTestRank (ValueSource valueSource) throws Exception {
FunctionQuery functionQuery = new FunctionQuery(valueSource);
IndexReader r = DirectoryReader.open(dir);
- IndexSearcher s = new IndexSearcher(r);
+ IndexSearcher s = newSearcher(r);
log("test: "+ functionQuery);
QueryUtils.check(random(), functionQuery,s);
ScoreDoc[] h = s.search(functionQuery, null, 1000).scoreDocs;
@@ -132,7 +132,7 @@ public class TestFieldScoreQuery extends
private void doTestExactScore (ValueSource valueSource) throws Exception {
FunctionQuery functionQuery = new FunctionQuery(valueSource);
IndexReader r = DirectoryReader.open(dir);
- IndexSearcher s = new IndexSearcher(r);
+ IndexSearcher s = newSearcher(r);
TopDocs td = s.search(functionQuery,null,1000);
assertEquals("All docs should be matched!",N_DOCS,td.totalHits);
ScoreDoc sd[] = td.scoreDocs;
Modified: lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java Tue May 7 11:20:55 2013
@@ -59,7 +59,7 @@ public class TestFunctionQuerySort exten
// Open index
IndexReader reader = writer.getReader();
writer.close();
- IndexSearcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
// Get ValueSource from FieldCache
IntFieldSource src = new IntFieldSource("value");
@@ -69,7 +69,7 @@ public class TestFunctionQuerySort exten
// Get hits sorted by our FunctionValues (ascending values)
Query q = new MatchAllDocsQuery();
- TopDocs hits = searcher.search(q, Integer.MAX_VALUE, orderBy);
+ TopDocs hits = searcher.search(q, reader.maxDoc(), orderBy);
assertEquals(NUM_VALS, hits.scoreDocs.length);
// Verify that sorting works in general
int i = 0;
@@ -81,7 +81,7 @@ public class TestFunctionQuerySort exten
// Now get hits after hit #2 using IS.searchAfter()
int afterIdx = 1;
FieldDoc afterHit = (FieldDoc) hits.scoreDocs[afterIdx];
- hits = searcher.searchAfter(afterHit, q, Integer.MAX_VALUE, orderBy);
+ hits = searcher.searchAfter(afterHit, q, reader.maxDoc(), orderBy);
// Expected # of hits: NUM_VALS - 2
assertEquals(NUM_VALS - (afterIdx + 1), hits.scoreDocs.length);
Modified: lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestOrdValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestOrdValues.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestOrdValues.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestOrdValues.java Tue May 7 11:20:55 2013
@@ -66,7 +66,7 @@ public class TestOrdValues extends Funct
// Test that queries based on reverse/ordFieldScore scores correctly
private void doTestRank(String field, boolean inOrder) throws Exception {
IndexReader r = DirectoryReader.open(dir);
- IndexSearcher s = new IndexSearcher(r);
+ IndexSearcher s = newSearcher(r);
ValueSource vs;
if (inOrder) {
vs = new OrdFieldSource(field);
@@ -117,7 +117,7 @@ public class TestOrdValues extends Funct
// Test that queries based on reverse/ordFieldScore returns docs with expected score.
private void doTestExactScore(String field, boolean inOrder) throws Exception {
IndexReader r = DirectoryReader.open(dir);
- IndexSearcher s = new IndexSearcher(r);
+ IndexSearcher s = newSearcher(r);
ValueSource vs;
if (inOrder) {
vs = new OrdFieldSource(field);
Modified: lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java Tue May 7 11:20:55 2013
@@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
@@ -32,6 +33,8 @@ import org.apache.lucene.index.RandomInd
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryUtils;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -134,4 +137,11 @@ public class TestMoreLikeThis extends Lu
mlt.setFieldNames(new String[] {"text", "foobar"});
mlt.like(new StringReader("this is a test"), "foobar");
}
+
+ // just basic equals/hashcode etc
+ public void testMoreLikeThisQuery() throws Exception {
+ Query query = new MoreLikeThisQuery("this is a test", new String[] { "text" }, new MockAnalyzer(random()), "text");
+ QueryUtils.check(random(), query, searcher);
+ }
+ // TODO: add tests for the MoreLikeThisQuery
}
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java Tue May 7 11:20:55 2013
@@ -19,8 +19,8 @@ package org.apache.lucene.queryparser.an
import java.io.IOException;
import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
@@ -31,36 +31,29 @@ import org.apache.lucene.util.Version;
/**
* Overrides Lucene's default QueryParser so that Fuzzy-, Prefix-, Range-, and WildcardQuerys
- * are also passed through the given analyzer, but wild card characters (like <code>*</code>)
- * don't get removed from the search terms.
+ * are also passed through the given analyzer, but wildcard characters <code>*</code> and
+ * <code>?</code> don't get removed from the search terms.
*
* <p><b>Warning:</b> This class should only be used with analyzers that do not use stopwords
* or that add tokens. Also, several stemming analyzers are inappropriate: for example, GermanAnalyzer
* will turn <code>Häuser</code> into <code>hau</code>, but <code>H?user</code> will
* become <code>h?user</code> when using this parser and thus no match would be found (i.e.
* using this parser will be no improvement over QueryParser in such cases).
- *
*/
public class AnalyzingQueryParser extends org.apache.lucene.queryparser.classic.QueryParser {
-
- /**
- * Constructs a query parser.
- * @param field the default field for query terms.
- * @param analyzer used to find terms in the query text.
- */
+ // gobble escaped chars or find a wildcard character
+ private final Pattern wildcardPattern = Pattern.compile("(\\.)|([?*]+)");
public AnalyzingQueryParser(Version matchVersion, String field, Analyzer analyzer) {
super(matchVersion, field, analyzer);
setAnalyzeRangeTerms(true);
}
/**
- * Called when parser
- * parses an input term token that contains one or more wildcard
- * characters (like <code>*</code>), but is not a prefix term token (one
- * that has just a single * character at the end).
+ * Called when parser parses an input term that contains one or more wildcard
+ * characters (like <code>*</code>), but is not a prefix term (one that has
+ * just a single <code>*</code> character at the end).
* <p>
- * Example: will be called for <code>H?user</code> or for <code>H*user</code>
- * but not for <code>*user</code>.
+ * Example: will be called for <code>H?user</code> or for <code>H*user</code>.
* <p>
* Depending on analyzer and settings, a wildcard term may (most probably will)
* be lower-cased automatically. It <b>will</b> go through the default Analyzer.
@@ -68,113 +61,52 @@ public class AnalyzingQueryParser extend
* Overrides super class, by passing terms through analyzer.
*
* @param field Name of the field query will use.
- * @param termStr Term token that contains one or more wild card
+ * @param termStr Term that contains one or more wildcard
* characters (? or *), but is not simple prefix term
*
* @return Resulting {@link Query} built for the term
*/
@Override
protected Query getWildcardQuery(String field, String termStr) throws ParseException {
- List<String> tlist = new ArrayList<String>();
- List<String> wlist = new ArrayList<String>();
- /* somewhat a hack: find/store wildcard chars
- * in order to put them back after analyzing */
- boolean isWithinToken = (!termStr.startsWith("?") && !termStr.startsWith("*"));
- StringBuilder tmpBuffer = new StringBuilder();
- char[] chars = termStr.toCharArray();
- for (int i = 0; i < termStr.length(); i++) {
- if (chars[i] == '?' || chars[i] == '*') {
- if (isWithinToken) {
- tlist.add(tmpBuffer.toString());
- tmpBuffer.setLength(0);
- }
- isWithinToken = false;
- } else {
- if (!isWithinToken) {
- wlist.add(tmpBuffer.toString());
- tmpBuffer.setLength(0);
- }
- isWithinToken = true;
- }
- tmpBuffer.append(chars[i]);
- }
- if (isWithinToken) {
- tlist.add(tmpBuffer.toString());
- } else {
- wlist.add(tmpBuffer.toString());
- }
- // get Analyzer from superclass and tokenize the term
- TokenStream source;
-
- int countTokens = 0;
- try {
- source = getAnalyzer().tokenStream(field, new StringReader(termStr));
- source.reset();
- } catch (IOException e1) {
- throw new RuntimeException(e1);
+ if (termStr == null){
+ //can't imagine this would ever happen
+ throw new ParseException("Passed null value as term to getWildcardQuery");
+ }
+ if ( ! getAllowLeadingWildcard() && (termStr.startsWith("*") || termStr.startsWith("?"))) {
+ throw new ParseException("'*' or '?' not allowed as first character in WildcardQuery"
+ + " unless getAllowLeadingWildcard() returns true");
}
- CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
- while (true) {
- try {
- if (!source.incrementToken()) break;
- } catch (IOException e) {
- break;
- }
- String term = termAtt.toString();
- if (!"".equals(term)) {
- try {
- tlist.set(countTokens++, term);
- } catch (IndexOutOfBoundsException ioobe) {
- countTokens = -1;
- }
- }
- }
- try {
- source.end();
- source.close();
- } catch (IOException e) {
- // ignore
- }
-
- if (countTokens != tlist.size()) {
- /* this means that the analyzer used either added or consumed
- * (common for a stemmer) tokens, and we can't build a WildcardQuery */
- throw new ParseException("Cannot build WildcardQuery with analyzer "
- + getAnalyzer().getClass() + " - tokens added or lost");
+
+ Matcher wildcardMatcher = wildcardPattern.matcher(termStr);
+ StringBuilder sb = new StringBuilder();
+ int last = 0;
+
+ while (wildcardMatcher.find()){
+ // continue if escaped char
+ if (wildcardMatcher.group(1) != null){
+ continue;
+ }
+
+ if (wildcardMatcher.start() > 0){
+ String chunk = termStr.substring(last, wildcardMatcher.start());
+ String analyzed = analyzeSingleChunk(field, termStr, chunk);
+ sb.append(analyzed);
+ }
+ //append the wildcard character
+ sb.append(wildcardMatcher.group(2));
+
+ last = wildcardMatcher.end();
}
-
- if (tlist.size() == 0) {
- return null;
- } else if (tlist.size() == 1) {
- if (wlist != null && wlist.size() == 1) {
- /* if wlist contains one wildcard, it must be at the end, because:
- * 1) wildcards are not allowed in 1st position of a term by QueryParser
- * 2) if wildcard was *not* in end, there would be *two* or more tokens */
- return super.getWildcardQuery(field, tlist.get(0)
- + wlist.get(0).toString());
- } else {
- /* we should never get here! if so, this method was called
- * with a termStr containing no wildcard ... */
- throw new IllegalArgumentException("getWildcardQuery called without wildcard");
- }
- } else {
- /* the term was tokenized, let's rebuild to one token
- * with wildcards put back in postion */
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < tlist.size(); i++) {
- sb.append( tlist.get(i));
- if (wlist != null && wlist.size() > i) {
- sb.append(wlist.get(i));
- }
- }
- return super.getWildcardQuery(field, sb.toString());
+ if (last < termStr.length()){
+ sb.append(analyzeSingleChunk(field, termStr, termStr.substring(last)));
}
+ return super.getWildcardQuery(field, sb.toString());
}
-
+
/**
* Called when parser parses an input term
- * token that uses prefix notation; that is, contains a single '*' wildcard
+ * that uses prefix notation; that is, contains a single '*' wildcard
* character as its last character. Since this is a special case
* of generic wildcard term, and such a query can be optimized easily,
* this usually results in a different query object.
@@ -185,52 +117,19 @@ public class AnalyzingQueryParser extend
* Overrides super class, by passing terms through analyzer.
*
* @param field Name of the field query will use.
- * @param termStr Term token to use for building term for the query
+ * @param termStr Term to use for building term for the query
* (<b>without</b> trailing '*' character!)
*
* @return Resulting {@link Query} built for the term
*/
@Override
protected Query getPrefixQuery(String field, String termStr) throws ParseException {
- // get Analyzer from superclass and tokenize the term
- TokenStream source;
- List<String> tlist = new ArrayList<String>();
- try {
- source = getAnalyzer().tokenStream(field, new StringReader(termStr));
- source.reset();
- } catch (IOException e1) {
- throw new RuntimeException(e1);
- }
- CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
- while (true) {
- try {
- if (!source.incrementToken()) break;
- } catch (IOException e) {
- break;
- }
- tlist.add(termAtt.toString());
- }
-
- try {
- source.end();
- source.close();
- } catch (IOException e) {
- // ignore
- }
-
- if (tlist.size() == 1) {
- return super.getPrefixQuery(field, tlist.get(0));
- } else {
- /* this means that the analyzer used either added or consumed
- * (common for a stemmer) tokens, and we can't build a PrefixQuery */
- throw new ParseException("Cannot build PrefixQuery with analyzer "
- + getAnalyzer().getClass()
- + (tlist.size() > 1 ? " - token(s) added" : " - token consumed"));
- }
+ String analyzed = analyzeSingleChunk(field, termStr, termStr);
+ return super.getPrefixQuery(field, analyzed);
}
/**
- * Called when parser parses an input term token that has the fuzzy suffix (~) appended.
+ * Called when parser parses an input term that has the fuzzy suffix (~) appended.
* <p>
* Depending on analyzer and settings, a fuzzy term may (most probably will)
* be lower-cased automatically. It <b>will</b> go through the default Analyzer.
@@ -238,42 +137,73 @@ public class AnalyzingQueryParser extend
* Overrides super class, by passing terms through analyzer.
*
* @param field Name of the field query will use.
- * @param termStr Term token to use for building term for the query
+ * @param termStr Term to use for building term for the query
*
* @return Resulting {@link Query} built for the term
*/
@Override
protected Query getFuzzyQuery(String field, String termStr, float minSimilarity)
throws ParseException {
- // get Analyzer from superclass and tokenize the term
- TokenStream source = null;
- String nextToken = null;
- boolean multipleTokens = false;
-
- try {
- source = getAnalyzer().tokenStream(field, new StringReader(termStr));
- CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
- source.reset();
- if (source.incrementToken()) {
- nextToken = termAtt.toString();
- }
- multipleTokens = source.incrementToken();
- } catch (IOException e) {
- nextToken = null;
- }
-
- try {
- source.end();
- source.close();
- } catch (IOException e) {
- // ignore
- }
+
+ String analyzed = analyzeSingleChunk(field, termStr, termStr);
+ return super.getFuzzyQuery(field, analyzed, minSimilarity);
+ }
- if (multipleTokens) {
- throw new ParseException("Cannot build FuzzyQuery with analyzer " + getAnalyzer().getClass()
- + " - tokens were added");
+ /**
+ * Returns the analyzed form for the given chunk
+ *
+ * If the analyzer produces more than one output token from the given chunk,
+ * a ParseException is thrown.
+ *
+ * @param field The target field
+ * @param termStr The full term from which the given chunk is excerpted
+ * @param chunk The portion of the given termStr to be analyzed
+ * @return The result of analyzing the given chunk
+ * @throws ParseException when analysis returns other than one output token
+ */
+ protected String analyzeSingleChunk(String field, String termStr, String chunk) throws ParseException{
+ String analyzed = null;
+ TokenStream stream = null;
+ try{
+ stream = getAnalyzer().tokenStream(field, new StringReader(chunk));
+ stream.reset();
+ CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
+ // get first and hopefully only output token
+ if (stream.incrementToken()) {
+ analyzed = termAtt.toString();
+
+ // try to increment again, there should only be one output token
+ StringBuilder multipleOutputs = null;
+ while (stream.incrementToken()) {
+ if (null == multipleOutputs) {
+ multipleOutputs = new StringBuilder();
+ multipleOutputs.append('"');
+ multipleOutputs.append(analyzed);
+ multipleOutputs.append('"');
+ }
+ multipleOutputs.append(',');
+ multipleOutputs.append('"');
+ multipleOutputs.append(termAtt.toString());
+ multipleOutputs.append('"');
+ }
+ stream.end();
+ stream.close();
+ if (null != multipleOutputs) {
+ throw new ParseException(
+ String.format(getLocale(),
+ "Analyzer created multiple terms for \"%s\": %s", chunk, multipleOutputs.toString()));
+ }
+ } else {
+ // nothing returned by analyzer. Was it a stop word and the user accidentally
+ // used an analyzer with stop words?
+ stream.end();
+ stream.close();
+ throw new ParseException(String.format(getLocale(), "Analyzer returned nothing for \"%s\"", chunk));
+ }
+ } catch (IOException e){
+ throw new ParseException(
+ String.format(getLocale(), "IO error while trying to analyze single term: \"%s\"", termStr));
}
-
- return (nextToken == null) ? null : super.getFuzzyQuery(field, nextToken, minSimilarity);
+ return analyzed;
}
}
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/MultiFieldQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/MultiFieldQueryParser.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/MultiFieldQueryParser.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/MultiFieldQueryParser.java Tue May 7 11:20:55 2013
@@ -213,6 +213,22 @@ public class MultiFieldQueryParser exten
}
return super.getRangeQuery(field, part1, part2, startInclusive, endInclusive);
}
+
+
+
+ @Override
+ protected Query getRegexpQuery(String field, String termStr)
+ throws ParseException {
+ if (field == null) {
+ List<BooleanClause> clauses = new ArrayList<BooleanClause>();
+ for (int i = 0; i < fields.length; i++) {
+ clauses.add(new BooleanClause(getRegexpQuery(fields[i], termStr),
+ BooleanClause.Occur.SHOULD));
+ }
+ return getBooleanQuery(clauses, true);
+ }
+ return super.getRegexpQuery(field, termStr);
+ }
/**
* Parses a query which searches on the fields specified.
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/complexPhrase/ComplexPhraseQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/complexPhrase/ComplexPhraseQueryParser.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/complexPhrase/ComplexPhraseQueryParser.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/complexPhrase/ComplexPhraseQueryParser.java Tue May 7 11:20:55 2013
@@ -363,7 +363,7 @@ public class ComplexPhraseQueryParser ex
@Override
public int hashCode() {
final int prime = 31;
- int result = 1;
+ int result = super.hashCode();
result = prime * result + ((field == null) ? 0 : field.hashCode());
result = prime
* result
@@ -381,6 +381,9 @@ public class ComplexPhraseQueryParser ex
return false;
if (getClass() != obj.getClass())
return false;
+ if (!super.equals(obj)) {
+ return false;
+ }
ComplexPhraseQuery other = (ComplexPhraseQuery) obj;
if (field == null) {
if (other.field != null)
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java Tue May 7 11:20:55 2013
@@ -19,8 +19,17 @@ package org.apache.lucene.queryparser.an
import java.io.IOException;
import java.io.Reader;
+import java.util.Map;
+import java.util.TreeMap;
-import org.apache.lucene.analysis.*;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockBytesAnalyzer;
+import org.apache.lucene.analysis.MockTokenFilter;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -33,11 +42,14 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.Query;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
/**
*/
+@SuppressCodecs("Lucene3x") // binary terms
public class TestAnalyzingQueryParser extends LuceneTestCase {
-
+ private final static String FIELD = "field";
+
private Analyzer a;
private String[] wildcardInput;
@@ -49,12 +61,15 @@ public class TestAnalyzingQueryParser ex
private String[] fuzzyInput;
private String[] fuzzyExpected;
+ private Map<String, String> wildcardEscapeHits = new TreeMap<String, String>();
+ private Map<String, String> wildcardEscapeMisses = new TreeMap<String, String>();
+
@Override
public void setUp() throws Exception {
super.setUp();
- wildcardInput = new String[] { "übersetzung über*ung",
+ wildcardInput = new String[] { "*bersetzung über*ung",
"Mötley Cr\u00fce Mötl?* Crü?", "Renée Zellweger Ren?? Zellw?ger" };
- wildcardExpected = new String[] { "ubersetzung uber*ung", "motley crue motl?* cru?",
+ wildcardExpected = new String[] { "*bersetzung uber*ung", "motley crue motl?* cru?",
"renee zellweger ren?? zellw?ger" };
prefixInput = new String[] { "übersetzung übersetz*",
@@ -71,43 +86,138 @@ public class TestAnalyzingQueryParser ex
fuzzyExpected = new String[] { "ubersetzung ubersetzung~1",
"motley crue motley~1 crue~2", "renee zellweger renee~0 zellweger~2" };
+ wildcardEscapeHits.put("mö*tley", "moatley");
+
+ // need to have at least one genuine wildcard to trigger the wildcard analysis
+ // hence the * before the y
+ wildcardEscapeHits.put("mö\\*tl*y", "mo*tley");
+
+ // escaped backslash then true wildcard
+ wildcardEscapeHits.put("mö\\\\*tley", "mo\\atley");
+
+ // escaped wildcard then true wildcard
+ wildcardEscapeHits.put("mö\\??ley", "mo?tley");
+
+ // the first is an escaped * which should yield a miss
+ wildcardEscapeMisses.put("mö\\*tl*y", "moatley");
+
a = new ASCIIAnalyzer();
}
+ public void testSingleChunkExceptions() {
+ boolean ex = false;
+ String termStr = "the*tre";
+
+ Analyzer stopsAnalyzer = new MockAnalyzer
+ (random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
+ try {
+ String q = parseWithAnalyzingQueryParser(termStr, stopsAnalyzer, true);
+ } catch (ParseException e){
+ if (e.getMessage().contains("returned nothing")){
+ ex = true;
+ }
+ }
+ assertEquals("Should have returned nothing", true, ex);
+ ex = false;
+
+ AnalyzingQueryParser qp = new AnalyzingQueryParser(TEST_VERSION_CURRENT, FIELD, a);
+ try{
+ qp.analyzeSingleChunk(FIELD, "", "not a single chunk");
+ } catch (ParseException e){
+ if (e.getMessage().contains("multiple terms")){
+ ex = true;
+ }
+ }
+ assertEquals("Should have produced multiple terms", true, ex);
+ }
+
+ public void testWildcardAlone() throws ParseException {
+ //seems like crazy edge case, but can be useful in concordance
+ boolean pex = false;
+ try{
+ Query q = getAnalyzedQuery("*", a, false);
+ } catch (ParseException e){
+ pex = true;
+ }
+ assertEquals("Wildcard alone with allowWildcard=false", true, pex);
+
+ pex = false;
+ try {
+ String qString = parseWithAnalyzingQueryParser("*", a, true);
+ assertEquals("Every word", "*", qString);
+ } catch (ParseException e){
+ pex = true;
+ }
+
+ assertEquals("Wildcard alone with allowWildcard=true", false, pex);
+
+ }
+ public void testWildCardEscapes() throws ParseException, IOException {
+
+ for (Map.Entry<String, String> entry : wildcardEscapeHits.entrySet()){
+ Query q = getAnalyzedQuery(entry.getKey(), a, false);
+ assertEquals("WildcardEscapeHits: " + entry.getKey(), true, isAHit(q, entry.getValue(), a));
+ }
+ for (Map.Entry<String, String> entry : wildcardEscapeMisses.entrySet()){
+ Query q = getAnalyzedQuery(entry.getKey(), a, false);
+ assertEquals("WildcardEscapeMisses: " + entry.getKey(), false, isAHit(q, entry.getValue(), a));
+ }
+
+ }
+ public void testWildCardQueryNoLeadingAllowed() {
+ boolean ex = false;
+ try{
+ String q = parseWithAnalyzingQueryParser(wildcardInput[0], a, false);
+
+ } catch (ParseException e){
+ ex = true;
+ }
+ assertEquals("Testing initial wildcard not allowed",
+ true, ex);
+ }
+
public void testWildCardQuery() throws ParseException {
for (int i = 0; i < wildcardInput.length; i++) {
assertEquals("Testing wildcards with analyzer " + a.getClass() + ", input string: "
- + wildcardInput[i], wildcardExpected[i], parseWithAnalyzingQueryParser(wildcardInput[i], a));
+ + wildcardInput[i], wildcardExpected[i], parseWithAnalyzingQueryParser(wildcardInput[i], a, true));
}
}
+
public void testPrefixQuery() throws ParseException {
for (int i = 0; i < prefixInput.length; i++) {
assertEquals("Testing prefixes with analyzer " + a.getClass() + ", input string: "
- + prefixInput[i], prefixExpected[i], parseWithAnalyzingQueryParser(prefixInput[i], a));
+ + prefixInput[i], prefixExpected[i], parseWithAnalyzingQueryParser(prefixInput[i], a, false));
}
}
public void testRangeQuery() throws ParseException {
for (int i = 0; i < rangeInput.length; i++) {
assertEquals("Testing ranges with analyzer " + a.getClass() + ", input string: "
- + rangeInput[i], rangeExpected[i], parseWithAnalyzingQueryParser(rangeInput[i], a));
+ + rangeInput[i], rangeExpected[i], parseWithAnalyzingQueryParser(rangeInput[i], a, false));
}
}
public void testFuzzyQuery() throws ParseException {
for (int i = 0; i < fuzzyInput.length; i++) {
assertEquals("Testing fuzzys with analyzer " + a.getClass() + ", input string: "
- + fuzzyInput[i], fuzzyExpected[i], parseWithAnalyzingQueryParser(fuzzyInput[i], a));
+ + fuzzyInput[i], fuzzyExpected[i], parseWithAnalyzingQueryParser(fuzzyInput[i], a, false));
}
}
- private String parseWithAnalyzingQueryParser(String s, Analyzer a) throws ParseException {
- AnalyzingQueryParser qp = new AnalyzingQueryParser(TEST_VERSION_CURRENT, "field", a);
+
+ private String parseWithAnalyzingQueryParser(String s, Analyzer a, boolean allowLeadingWildcard) throws ParseException {
+ Query q = getAnalyzedQuery(s, a, allowLeadingWildcard);
+ return q.toString(FIELD);
+ }
+
+ private Query getAnalyzedQuery(String s, Analyzer a, boolean allowLeadingWildcard) throws ParseException {
+ AnalyzingQueryParser qp = new AnalyzingQueryParser(TEST_VERSION_CURRENT, FIELD, a);
+ qp.setAllowLeadingWildcard(allowLeadingWildcard);
org.apache.lucene.search.Query q = qp.parse(s);
- return q.toString("field");
+ return q;
}
-
+
final static class FoldingFilter extends TokenFilter {
final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
@@ -144,31 +254,45 @@ public class TestAnalyzingQueryParser ex
final static class ASCIIAnalyzer extends Analyzer {
@Override
public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ Tokenizer result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(result, new FoldingFilter(result));
}
}
-
+
+
// LUCENE-4176
public void testByteTerms() throws Exception {
- Directory ramDir = newDirectory();
+ String s = "à¹à¸";
Analyzer analyzer = new MockBytesAnalyzer();
+ QueryParser qp = new AnalyzingQueryParser(TEST_VERSION_CURRENT, FIELD, analyzer);
+ Query q = qp.parse("[à¹à¸ TO à¹à¸]");
+ assertEquals(true, isAHit(q, s, analyzer));
+ }
+
+
+ private boolean isAHit(Query q, String content, Analyzer analyzer) throws IOException{
+ Directory ramDir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), ramDir, analyzer);
Document doc = new Document();
FieldType fieldType = new FieldType();
fieldType.setIndexed(true);
fieldType.setTokenized(true);
fieldType.setStored(true);
- Field field = new Field("content","à¹à¸", fieldType);
+ Field field = new Field(FIELD, content, fieldType);
doc.add(field);
writer.addDocument(doc);
writer.close();
DirectoryReader ir = DirectoryReader.open(ramDir);
IndexSearcher is = new IndexSearcher(ir);
- QueryParser qp = new AnalyzingQueryParser(TEST_VERSION_CURRENT, "content", analyzer);
- Query q = qp.parse("[à¹à¸ TO à¹à¸]");
- assertEquals(1, is.search(q, 10).totalHits);
+
+ int hits = is.search(q, 10).totalHits;
ir.close();
ramDir.close();
+ if (hits == 1){
+ return true;
+ } else {
+ return false;
+ }
+
}
}
\ No newline at end of file
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java Tue May 7 11:20:55 2013
@@ -28,10 +28,13 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.RegexpQuery;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
@@ -294,7 +297,7 @@ public class TestMultiFieldQueryParser e
mfqp.setDefaultOperator(QueryParser.Operator.AND);
Query q = mfqp.parse("the footest");
IndexReader ir = DirectoryReader.open(ramDir);
- IndexSearcher is = new IndexSearcher(ir);
+ IndexSearcher is = newSearcher(ir);
ScoreDoc[] hits = is.search(q, null, 1000).scoreDocs;
assertEquals(1, hits.length);
ir.close();
@@ -328,5 +331,15 @@ public class TestMultiFieldQueryParser e
return stdAnalyzer.createComponents(fieldName, reader);
}
}
+
+ public void testSimpleRegex() throws ParseException {
+ String[] fields = new String[] {"a", "b"};
+ MultiFieldQueryParser mfqp = new MultiFieldQueryParser(TEST_VERSION_CURRENT, fields, new MockAnalyzer(random()));
+
+ BooleanQuery bq = new BooleanQuery(true);
+ bq.add(new RegexpQuery(new Term("a", "[a-z][123]")), Occur.SHOULD);
+ bq.add(new RegexpQuery(new Term("b", "[a-z][123]")), Occur.SHOULD);
+ assertEquals(bq, mfqp.parse("/[a-z][123]/"));
+ }
}
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/complexPhrase/TestComplexPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/complexPhrase/TestComplexPhraseQuery.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/complexPhrase/TestComplexPhraseQuery.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/complexPhrase/TestComplexPhraseQuery.java Tue May 7 11:20:55 2013
@@ -127,7 +127,7 @@ public class TestComplexPhraseQuery exte
}
w.close();
reader = DirectoryReader.open(rd);
- searcher = new IndexSearcher(reader);
+ searcher = newSearcher(reader);
}
@Override
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java Tue May 7 11:20:55 2013
@@ -546,7 +546,7 @@ public class TestPrecedenceQueryParser e
public void testBoost() throws Exception {
CharacterRunAutomaton stopSet = new CharacterRunAutomaton(BasicAutomata.makeString("on"));
- Analyzer oneStopAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopSet, true);
+ Analyzer oneStopAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopSet);
PrecedenceQueryParser qp = new PrecedenceQueryParser();
qp.setAnalyzer(oneStopAnalyzer);
@@ -561,7 +561,7 @@ public class TestPrecedenceQueryParser e
q = qp.parse("\"on\"^1.0", "field");
assertNotNull(q);
- q = getParser(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true)).parse("the^3",
+ q = getParser(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET)).parse("the^3",
"field");
assertNotNull(q);
}
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java Tue May 7 11:20:55 2013
@@ -333,7 +333,7 @@ public class TestMultiFieldQPHelper exte
mfqp.setDefaultOperator(StandardQueryConfigHandler.Operator.AND);
Query q = mfqp.parse("the footest", null);
IndexReader ir = DirectoryReader.open(ramDir);
- IndexSearcher is = new IndexSearcher(ir);
+ IndexSearcher is = newSearcher(ir);
ScoreDoc[] hits = is.search(q, null, 1000).scoreDocs;
assertEquals(1, hits.length);
ir.close();
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java Tue May 7 11:20:55 2013
@@ -946,7 +946,7 @@ public class TestQPHelper extends Lucene
public void testBoost() throws Exception {
CharacterRunAutomaton stopSet = new CharacterRunAutomaton(BasicAutomata.makeString("on"));
- Analyzer oneStopAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopSet, true);
+ Analyzer oneStopAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopSet);
StandardQueryParser qp = new StandardQueryParser();
qp.setAnalyzer(oneStopAnalyzer);
@@ -962,7 +962,7 @@ public class TestQPHelper extends Lucene
assertNotNull(q);
StandardQueryParser qp2 = new StandardQueryParser();
- qp2.setAnalyzer(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true));
+ qp2.setAnalyzer(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET));
q = qp2.parse("the^3", "field");
// "the" is a stop word so the result is an empty query:
@@ -1179,7 +1179,7 @@ public class TestQPHelper extends Lucene
public void testStopwords() throws Exception {
StandardQueryParser qp = new StandardQueryParser();
CharacterRunAutomaton stopSet = new CharacterRunAutomaton(new RegExp("the|foo").toAutomaton());
- qp.setAnalyzer(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopSet, true));
+ qp.setAnalyzer(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopSet));
Query result = qp.parse("a:the OR a:foo", "a");
assertNotNull("result is null and it shouldn't be", result);
@@ -1203,7 +1203,7 @@ public class TestQPHelper extends Lucene
public void testPositionIncrement() throws Exception {
StandardQueryParser qp = new StandardQueryParser();
qp.setAnalyzer(
- new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true));
+ new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET));
qp.setEnablePositionIncrements(true);
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java Tue May 7 11:20:55 2013
@@ -852,7 +852,7 @@ public abstract class QueryParserTestBas
public void testBoost()
throws Exception {
CharacterRunAutomaton stopWords = new CharacterRunAutomaton(BasicAutomata.makeString("on"));
- Analyzer oneStopAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopWords, true);
+ Analyzer oneStopAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopWords);
CommonQueryParserConfiguration qp = getParserConfig(oneStopAnalyzer);
Query q = getQuery("on^1.0",qp);
assertNotNull(q);
@@ -865,7 +865,7 @@ public abstract class QueryParserTestBas
q = getQuery("\"on\"^1.0",qp);
assertNotNull(q);
- Analyzer a2 = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true);
+ Analyzer a2 = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
CommonQueryParserConfiguration qp2 = getParserConfig(a2);
q = getQuery("the^3", qp2);
// "the" is a stop word so the result is an empty query:
@@ -1007,7 +1007,7 @@ public abstract class QueryParserTestBas
public void testStopwords() throws Exception {
CharacterRunAutomaton stopSet = new CharacterRunAutomaton(new RegExp("the|foo").toAutomaton());
- CommonQueryParserConfiguration qp = getParserConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopSet, true));
+ CommonQueryParserConfiguration qp = getParserConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopSet));
Query result = getQuery("field:the OR field:foo",qp);
assertNotNull("result is null and it shouldn't be", result);
assertTrue("result is not a BooleanQuery", result instanceof BooleanQuery);
@@ -1023,7 +1023,7 @@ public abstract class QueryParserTestBas
}
public void testPositionIncrement() throws Exception {
- CommonQueryParserConfiguration qp = getParserConfig( new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true));
+ CommonQueryParserConfiguration qp = getParserConfig( new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET));
qp.setEnablePositionIncrements(true);
String qtxt = "\"the words in poisitions pos02578 are stopped in this phrasequery\"";
// 0 2 5 7 8
@@ -1070,7 +1070,7 @@ public abstract class QueryParserTestBas
// "match"
public void testPositionIncrements() throws Exception {
Directory dir = newDirectory();
- Analyzer a = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true);
+ Analyzer a = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, a));
Document doc = new Document();
doc.add(newTextField("field", "the wizard of ozzy", Field.Store.NO));
@@ -1185,7 +1185,7 @@ public abstract class QueryParserTestBas
}
public void testPhraseQueryToString() throws Exception {
- Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true);
+ Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
CommonQueryParserConfiguration qp = getParserConfig(analyzer);
qp.setEnablePositionIncrements(true);
PhraseQuery q = (PhraseQuery)getQuery("\"this hi this is a test is\"", qp);
@@ -1235,26 +1235,13 @@ public abstract class QueryParserTestBas
CharacterRunAutomaton stopStopList =
new CharacterRunAutomaton(new RegExp("[sS][tT][oO][pP]").toAutomaton());
- CommonQueryParserConfiguration qp = getParserConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false, stopStopList, false));
-
- PhraseQuery phraseQuery = new PhraseQuery();
- phraseQuery.add(new Term("field", "1"));
- phraseQuery.add(new Term("field", "2"));
-
- assertEquals(phraseQuery, getQuery("\"1 2\"",qp));
- assertEquals(phraseQuery, getQuery("\"1 stop 2\"",qp));
-
- qp.setEnablePositionIncrements(true);
- assertEquals(phraseQuery, getQuery("\"1 stop 2\"",qp));
-
- qp.setEnablePositionIncrements(false);
- assertEquals(phraseQuery, getQuery("\"1 stop 2\"",qp));
+ CommonQueryParserConfiguration qp = getParserConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false, stopStopList));
qp = getParserConfig(
- new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false, stopStopList, true));
+ new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false, stopStopList));
qp.setEnablePositionIncrements(true);
- phraseQuery = new PhraseQuery();
+ PhraseQuery phraseQuery = new PhraseQuery();
phraseQuery.add(new Term("field", "1"));
phraseQuery.add(new Term("field", "2"), 2);
assertEquals(phraseQuery, getQuery("\"1 stop 2\"",qp));
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java Tue May 7 11:20:55 2013
@@ -58,7 +58,7 @@ public class TestParser extends LuceneTe
@BeforeClass
public static void beforeClass() throws Exception {
// TODO: rewrite test (this needs to set QueryParser.enablePositionIncrements, too, for work with CURRENT):
- Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET, false);
+ Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
//initialize the parser
builder = new CorePlusExtensionsParser("contents", analyzer);
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestQueryTemplateManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestQueryTemplateManager.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestQueryTemplateManager.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestQueryTemplateManager.java Tue May 7 11:20:55 2013
@@ -152,7 +152,7 @@ public class TestQueryTemplateManager ex
w.forceMerge(1);
w.close();
reader = DirectoryReader.open(dir);
- searcher = new IndexSearcher(reader);
+ searcher = newSearcher(reader);
//initialize the parser
builder = new CorePlusExtensionsParser("artist", analyzer);
Modified: lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java (original)
+++ lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java Tue May 7 11:20:55 2013
@@ -72,7 +72,7 @@ public class FuzzyLikeThisQuery extends
@Override
public int hashCode() {
final int prime = 31;
- int result = 1;
+ int result = super.hashCode();
result = prime * result + ((analyzer == null) ? 0 : analyzer.hashCode());
result = prime * result
+ ((fieldVals == null) ? 0 : fieldVals.hashCode());
@@ -89,6 +89,9 @@ public class FuzzyLikeThisQuery extends
return false;
if (getClass() != obj.getClass())
return false;
+ if (!super.equals(obj)) {
+ return false;
+ }
FuzzyLikeThisQuery other = (FuzzyLikeThisQuery) obj;
if (analyzer == null) {
if (other.analyzer != null)
Modified: lucene/dev/branches/lucene4258/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java (original)
+++ lucene/dev/branches/lucene4258/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java Tue May 7 11:20:55 2013
@@ -69,7 +69,7 @@ public class TestSpanRegexQuery extends
writer.close();
IndexReader reader = DirectoryReader.open(directory);
- IndexSearcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
SpanQuery srq = new SpanMultiTermQueryWrapper<RegexQuery>(new RegexQuery(new Term("field", "aut.*")));
SpanFirstQuery sfq = new SpanFirstQuery(srq, 1);
// SpanNearQuery query = new SpanNearQuery(new SpanQuery[] {srq, stq}, 6,
Modified: lucene/dev/branches/lucene4258/lucene/site/changes/changes2html.pl
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/site/changes/changes2html.pl?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/site/changes/changes2html.pl (original)
+++ lucene/dev/branches/lucene4258/lucene/site/changes/changes2html.pl Tue May 7 11:20:55 2013
@@ -78,9 +78,9 @@ for (my $line_num = 0 ; $line_num <= $#l
($reldate, $relinfo) = get_release_date($release, $relinfo);
$sections = [];
push @releases, [ $release, $reldate, $relinfo, $sections ];
- ($first_relid = lc($release)) =~ s/\s+/_/g
+ ($first_relid = 'v'.lc($release)) =~ s/\s+/_/g
if ($#releases == 0 or ($#releases == 1 and not ($releases[0][0])));
- ($second_relid = lc($release)) =~ s/\s+/_/g
+ ($second_relid = 'v'.lc($release)) =~ s/\s+/_/g
if ( ($#releases == 1 and $releases[0][0])
or ($#releases == 2 and not $releases[0][0]));
$items = undef;
@@ -123,6 +123,11 @@ for (my $line_num = 0 ; $line_num <= $#l
# Make a fake release to hold pre-release sections
push @releases, [ undef, undef, undef, $sections ];
}
+ for my $section (@$sections) {
+ if ($heading eq $section->[0]) {
+ die "Section '$heading' appears more than once under release '$releases[-1][0]'";
+ }
+ }
push @$sections, [ $heading, $items ];
$in_major_component_versions_section
= ($heading =~ /Versions of Major Components/i);
@@ -416,7 +421,7 @@ for my $rel (@releases) {
my $relid = '';
if ($release) { # Pre-release sections have no release ID
++$relcnt;
- ($relid = lc($release)) =~ s/\s+/_/g;
+ ($relid = 'v'.lc($release)) =~ s/\s+/_/g;
print "<$header>";
print "<a id=\"$relid\" href=\"javascript:toggleList('$relid')\">"
unless ($release =~ /^20\d\d/);
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeFilter.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeFilter.java Tue May 7 11:20:55 2013
@@ -27,14 +27,14 @@ import org.apache.lucene.search.DocIdSet
import org.apache.lucene.search.Filter;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.FixedBitSet;
import java.io.IOException;
/**
* Base class for Lucene Filters on SpatialPrefixTree fields.
*
- * @lucene.internal
+ * @lucene.experimental
*/
public abstract class AbstractPrefixTreeFilter extends Filter {
@@ -93,13 +93,13 @@ public abstract class AbstractPrefixTree
this.termsEnum = terms.iterator(null);
}
- protected void collectDocs(OpenBitSet bitSet) throws IOException {
+ protected void collectDocs(FixedBitSet bitSet) throws IOException {
//WARN: keep this specialization in sync
assert termsEnum != null;
docsEnum = termsEnum.docs(acceptDocs, docsEnum, DocsEnum.FLAG_NONE);
int docid;
while ((docid = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- bitSet.fastSet(docid);
+ bitSet.set(docid);
}
}
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeFilter.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeFilter.java Tue May 7 11:20:55 2013
@@ -21,7 +21,7 @@ import com.spatial4j.core.shape.Shape;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSet;
-import org.apache.lucene.spatial.prefix.tree.Node;
+import org.apache.lucene.spatial.prefix.tree.Cell;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -52,7 +52,7 @@ public abstract class AbstractVisitingPr
public AbstractVisitingPrefixTreeFilter(Shape queryShape, String fieldName, SpatialPrefixTree grid,
int detailLevel, int prefixGridScanLevel) {
super(queryShape, fieldName, grid, detailLevel);
- this.prefixGridScanLevel = Math.max(1, Math.min(prefixGridScanLevel, grid.getMaxLevels() - 1));
+ this.prefixGridScanLevel = Math.max(0, Math.min(prefixGridScanLevel, grid.getMaxLevels() - 1));
assert detailLevel <= grid.getMaxLevels();
}
@@ -84,13 +84,13 @@ public abstract class AbstractVisitingPr
* that there are indexed terms; if not it quickly returns null. Then it calls
* {@link #start()} so a subclass can set up a return value, like an
* {@link org.apache.lucene.util.OpenBitSet}. Then it starts the traversal
- * process, calling {@link #findSubCellsToVisit(org.apache.lucene.spatial.prefix.tree.Node)}
+ * process, calling {@link #findSubCellsToVisit(org.apache.lucene.spatial.prefix.tree.Cell)}
* which by default finds the top cells that intersect {@code queryShape}. If
* there isn't an indexed cell for a corresponding cell returned for this
* method then it's short-circuited until it finds one, at which point
- * {@link #visit(org.apache.lucene.spatial.prefix.tree.Node)} is called. At
+ * {@link #visit(org.apache.lucene.spatial.prefix.tree.Cell)} is called. At
* some depths, of the tree, the algorithm switches to a scanning mode that
- * finds calls {@link #visitScanned(org.apache.lucene.spatial.prefix.tree.Node, com.spatial4j.core.shape.Shape)}
+ * finds calls {@link #visitScanned(org.apache.lucene.spatial.prefix.tree.Cell)}
* for each leaf cell found.
*
* @lucene.internal
@@ -113,7 +113,7 @@ public abstract class AbstractVisitingPr
private VNode curVNode;//current pointer, derived from query shape
private BytesRef curVNodeTerm = new BytesRef();//curVNode.cell's term.
- private Node scanCell;
+ private Cell scanCell;
private BytesRef thisTerm;//the result of termsEnum.term()
@@ -132,7 +132,7 @@ public abstract class AbstractVisitingPr
return null; // all done
curVNode = new VNode(null);
- curVNode.reset(grid.getWorldNode());
+ curVNode.reset(grid.getWorldCell());
start();
@@ -198,21 +198,20 @@ public abstract class AbstractVisitingPr
return finish();
}
- /** Called initially, and whenever {@link #visit(org.apache.lucene.spatial.prefix.tree.Node)}
+ /** Called initially, and whenever {@link #visit(org.apache.lucene.spatial.prefix.tree.Cell)}
* returns true. */
private void addIntersectingChildren() throws IOException {
assert thisTerm != null;
- Node cell = curVNode.cell;
+ Cell cell = curVNode.cell;
if (cell.getLevel() >= detailLevel)
throw new IllegalStateException("Spatial logic error");
//Check for adjacent leaf (happens for indexed non-point shapes)
- assert !cell.isLeaf();
if (hasIndexedLeaves && cell.getLevel() != 0) {
//If the next indexed term just adds a leaf marker ('+') to cell,
// then add all of those docs
assert StringHelper.startsWith(thisTerm, curVNodeTerm);
- scanCell = grid.getNode(thisTerm.bytes, thisTerm.offset, thisTerm.length, scanCell);
+ scanCell = grid.getCell(thisTerm.bytes, thisTerm.offset, thisTerm.length, scanCell);
if (scanCell.getLevel() == cell.getLevel() && scanCell.isLeaf()) {
visitLeaf(scanCell);
//advance
@@ -231,7 +230,7 @@ public abstract class AbstractVisitingPr
if (!scan) {
//Divide & conquer (ultimately termsEnum.seek())
- Iterator<Node> subCellsIter = findSubCellsToVisit(cell);
+ Iterator<Cell> subCellsIter = findSubCellsToVisit(cell);
if (!subCellsIter.hasNext())//not expected
return;
curVNode.children = new VNodeCellIterator(subCellsIter, new VNode(curVNode));
@@ -249,7 +248,7 @@ public abstract class AbstractVisitingPr
* guaranteed to have an intersection and thus this must return some number
* of nodes.
*/
- protected Iterator<Node> findSubCellsToVisit(Node cell) {
+ protected Iterator<Cell> findSubCellsToVisit(Cell cell) {
return cell.getSubCells(queryShape).iterator();
}
@@ -257,28 +256,19 @@ public abstract class AbstractVisitingPr
* Scans ({@code termsEnum.next()}) terms until a term is found that does
* not start with curVNode's cell. If it finds a leaf cell or a cell at
* level {@code scanDetailLevel} then it calls {@link
- * #visitScanned(org.apache.lucene.spatial.prefix.tree.Node,
- * com.spatial4j.core.shape.Shape)}.
+ * #visitScanned(org.apache.lucene.spatial.prefix.tree.Cell)}.
*/
protected void scan(int scanDetailLevel) throws IOException {
for (;
thisTerm != null && StringHelper.startsWith(thisTerm, curVNodeTerm);
thisTerm = termsEnum.next()) {
- scanCell = grid.getNode(thisTerm.bytes, thisTerm.offset, thisTerm.length, scanCell);
+ scanCell = grid.getCell(thisTerm.bytes, thisTerm.offset, thisTerm.length, scanCell);
int termLevel = scanCell.getLevel();
if (termLevel > scanDetailLevel)
continue;
if (termLevel == scanDetailLevel || scanCell.isLeaf()) {
- Shape cShape;
- //if this cell represents a point, use the cell center vs the box
- // (points never have isLeaf())
- if (termLevel == grid.getMaxLevels() && !scanCell.isLeaf())
- cShape = scanCell.getCenter();
- else
- cShape = scanCell.getShape();
-
- visitScanned(scanCell, cShape);
+ visitScanned(scanCell);
}
}//term loop
}
@@ -286,10 +276,10 @@ public abstract class AbstractVisitingPr
/** Used for {@link VNode#children}. */
private class VNodeCellIterator implements Iterator<VNode> {
- final Iterator<Node> cellIter;
+ final Iterator<Cell> cellIter;
private final VNode vNode;
- VNodeCellIterator(Iterator<Node> cellIter, VNode vNode) {
+ VNodeCellIterator(Iterator<Cell> cellIter, VNode vNode) {
this.cellIter = cellIter;
this.vNode = vNode;
}
@@ -319,28 +309,26 @@ public abstract class AbstractVisitingPr
/**
* Visit an indexed cell returned from
- * {@link #findSubCellsToVisit(org.apache.lucene.spatial.prefix.tree.Node)}.
+ * {@link #findSubCellsToVisit(org.apache.lucene.spatial.prefix.tree.Cell)}.
*
* @param cell An intersecting cell.
* @return true to descend to more levels. It is an error to return true
* if cell.level == detailLevel
*/
- protected abstract boolean visit(Node cell) throws IOException;
+ protected abstract boolean visit(Cell cell) throws IOException;
/**
* Called after visit() returns true and an indexed leaf cell is found. An
* indexed leaf cell means associated documents generally won't be found at
* further detail levels.
*/
- protected abstract void visitLeaf(Node cell) throws IOException;
+ protected abstract void visitLeaf(Cell cell) throws IOException;
/**
* The cell is either indexed as a leaf or is the last level of detail. It
* might not even intersect the query shape, so be sure to check for that.
- * Use {@code cellShape} instead of {@code cell.getCellShape} for the cell's
- * shape.
*/
- protected abstract void visitScanned(Node cell, Shape cellShape) throws IOException;
+ protected abstract void visitScanned(Cell cell) throws IOException;
protected void preSiblings(VNode vNode) throws IOException {
@@ -351,7 +339,7 @@ public abstract class AbstractVisitingPr
}//class VisitorTemplate
/**
- * A Visitor Node/Cell found via the query shape for {@link VisitorTemplate}.
+ * A Visitor Cell/Cell found via the query shape for {@link VisitorTemplate}.
* Sometimes these are reset(cell). It's like a LinkedList node but forms a
* tree.
*
@@ -365,7 +353,7 @@ public abstract class AbstractVisitingPr
final VNode parent;//only null at the root
Iterator<VNode> children;//null, then sometimes set, then null
- Node cell;//not null (except initially before reset())
+ Cell cell;//not null (except initially before reset())
/**
* call reset(cell) after to set the cell.
@@ -374,7 +362,7 @@ public abstract class AbstractVisitingPr
this.parent = parent;
}
- void reset(Node cell) {
+ void reset(Cell cell) {
assert cell != null;
this.cell = cell;
assert children == null;
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/IntersectsPrefixTreeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/IntersectsPrefixTreeFilter.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/IntersectsPrefixTreeFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/IntersectsPrefixTreeFilter.java Tue May 7 11:20:55 2013
@@ -21,10 +21,10 @@ import com.spatial4j.core.shape.Shape;
import com.spatial4j.core.shape.SpatialRelation;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.DocIdSet;
-import org.apache.lucene.spatial.prefix.tree.Node;
+import org.apache.lucene.spatial.prefix.tree.Cell;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.FixedBitSet;
import java.io.IOException;
@@ -53,11 +53,11 @@ public class IntersectsPrefixTreeFilter
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
return new VisitorTemplate(context, acceptDocs, hasIndexedLeaves) {
- private OpenBitSet results;
+ private FixedBitSet results;
@Override
protected void start() {
- results = new OpenBitSet(maxDoc);
+ results = new FixedBitSet(maxDoc);
}
@Override
@@ -66,7 +66,7 @@ public class IntersectsPrefixTreeFilter
}
@Override
- protected boolean visit(Node cell) throws IOException {
+ protected boolean visit(Cell cell) throws IOException {
if (cell.getShapeRel() == SpatialRelation.WITHIN || cell.getLevel() == detailLevel) {
collectDocs(results);
return false;
@@ -75,13 +75,21 @@ public class IntersectsPrefixTreeFilter
}
@Override
- protected void visitLeaf(Node cell) throws IOException {
+ protected void visitLeaf(Cell cell) throws IOException {
collectDocs(results);
}
@Override
- protected void visitScanned(Node cell, Shape cellShape) throws IOException {
- if (queryShape.relate(cellShape).intersects())
+ protected void visitScanned(Cell cell) throws IOException {
+ Shape cShape;
+ //if this cell represents a point, use the cell center vs the box
+ // TODO this behavior is debatable; might want to be configurable
+ // (points never have isLeaf())
+ if (cell.getLevel() == grid.getMaxLevels() && !cell.isLeaf())
+ cShape = cell.getCenter();
+ else
+ cShape = cell.getShape();
+ if (queryShape.relate(cShape).intersects())
collectDocs(results);
}
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java Tue May 7 11:20:55 2013
@@ -1,3 +1,5 @@
+package org.apache.lucene.spatial.prefix;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -15,10 +17,8 @@
* limitations under the License.
*/
-package org.apache.lucene.spatial.prefix;
-
import com.spatial4j.core.shape.Point;
-import org.apache.lucene.spatial.prefix.tree.Node;
+import org.apache.lucene.spatial.prefix.tree.Cell;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.spatial.util.ShapeFieldCacheProvider;
import org.apache.lucene.util.BytesRef;
@@ -40,11 +40,11 @@ public class PointPrefixTreeFieldCachePr
this.grid = grid;
}
- private Node scanCell = null;//re-used in readShape to save GC
+ private Cell scanCell = null;//re-used in readShape to save GC
@Override
protected Point readShape(BytesRef term) {
- scanCell = grid.getNode(term.bytes, term.offset, term.length, scanCell);
+ scanCell = grid.getCell(term.bytes, term.offset, term.length, scanCell);
if (scanCell.getLevel() == grid.getMaxLevels() && !scanCell.isLeaf())
return scanCell.getCenter();
return null;
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java Tue May 7 11:20:55 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.document.FieldT
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.spatial.SpatialStrategy;
-import org.apache.lucene.spatial.prefix.tree.Node;
+import org.apache.lucene.spatial.prefix.tree.Cell;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.spatial.query.SpatialArgs;
import org.apache.lucene.spatial.util.ShapeFieldCacheDistanceValueSource;
@@ -125,7 +125,7 @@ public abstract class PrefixTreeStrategy
public Field[] createIndexableFields(Shape shape, double distErr) {
int detailLevel = grid.getLevelForDistance(distErr);
- List<Node> cells = grid.getNodes(shape, detailLevel, true, simplifyIndexedCells);//intermediates cells
+ List<Cell> cells = grid.getCells(shape, detailLevel, true, simplifyIndexedCells);//intermediates cells
//TODO is CellTokenStream supposed to be re-used somehow? see Uwe's comments:
// http://code.google.com/p/lucene-spatial-playground/issues/detail?id=4
@@ -151,9 +151,9 @@ public abstract class PrefixTreeStrategy
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
- private Iterator<Node> iter = null;
+ private Iterator<Cell> iter = null;
- public CellTokenStream(Iterator<Node> tokens) {
+ public CellTokenStream(Iterator<Cell> tokens) {
this.iter = tokens;
}
@@ -164,12 +164,12 @@ public abstract class PrefixTreeStrategy
clearAttributes();
if (nextTokenStringNeedingLeaf != null) {
termAtt.append(nextTokenStringNeedingLeaf);
- termAtt.append((char) Node.LEAF_BYTE);
+ termAtt.append((char) Cell.LEAF_BYTE);
nextTokenStringNeedingLeaf = null;
return true;
}
if (iter.hasNext()) {
- Node cell = iter.next();
+ Cell cell = iter.next();
CharSequence token = cell.getTokenString();
termAtt.append(token);
if (cell.isLeaf())
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java Tue May 7 11:20:55 2013
@@ -19,6 +19,7 @@ package org.apache.lucene.spatial.prefix
import com.spatial4j.core.shape.Shape;
import org.apache.lucene.search.Filter;
+import org.apache.lucene.spatial.DisjointSpatialFilter;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.spatial.query.SpatialArgs;
import org.apache.lucene.spatial.query.SpatialOperation;
@@ -63,16 +64,25 @@ public class RecursivePrefixTreeStrategy
@Override
public Filter makeFilter(SpatialArgs args) {
final SpatialOperation op = args.getOperation();
- if (op != SpatialOperation.Intersects)
- throw new UnsupportedSpatialOperation(op);
+ if (op == SpatialOperation.IsDisjointTo)
+ return new DisjointSpatialFilter(this, args, getFieldName());
Shape shape = args.getShape();
-
int detailLevel = grid.getLevelForDistance(args.resolveDistErr(ctx, distErrPct));
+ final boolean hasIndexedLeaves = true;
- return new IntersectsPrefixTreeFilter(
- shape, getFieldName(), grid, detailLevel, prefixGridScanLevel,
- true);//hasIndexedLeaves
+ if (op == SpatialOperation.Intersects) {
+ return new IntersectsPrefixTreeFilter(
+ shape, getFieldName(), grid, detailLevel, prefixGridScanLevel,
+ hasIndexedLeaves);
+ } else if (op == SpatialOperation.IsWithin) {
+ return new WithinPrefixTreeFilter(
+ shape, getFieldName(), grid, detailLevel, prefixGridScanLevel,
+ -1);//-1 flag is slower but ensures correct results
+ } else if (op == SpatialOperation.Contains) {
+ return new ContainsPrefixTreeFilter(shape, getFieldName(), grid, detailLevel);
+ }
+ throw new UnsupportedSpatialOperation(op);
}
}
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java Tue May 7 11:20:55 2013
@@ -20,7 +20,7 @@ package org.apache.lucene.spatial.prefix
import com.spatial4j.core.shape.Shape;
import org.apache.lucene.queries.TermsFilter;
import org.apache.lucene.search.Filter;
-import org.apache.lucene.spatial.prefix.tree.Node;
+import org.apache.lucene.spatial.prefix.tree.Cell;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.spatial.query.SpatialArgs;
import org.apache.lucene.spatial.query.SpatialOperation;
@@ -31,7 +31,7 @@ import java.util.List;
/**
* A basic implementation of {@link PrefixTreeStrategy} using a large {@link
- * TermsFilter} of all the nodes from {@link SpatialPrefixTree#getNodes(com.spatial4j.core.shape.Shape,
+ * TermsFilter} of all the cells from {@link SpatialPrefixTree#getCells(com.spatial4j.core.shape.Shape,
* int, boolean, boolean)}. It only supports the search of indexed Point shapes.
* <p/>
* The precision of query shapes (distErrPct) is an important factor in using
@@ -55,12 +55,12 @@ public class TermQueryPrefixTreeStrategy
Shape shape = args.getShape();
int detailLevel = grid.getLevelForDistance(args.resolveDistErr(ctx, distErrPct));
- List<Node> cells = grid.getNodes(shape, detailLevel,
+ List<Cell> cells = grid.getCells(shape, detailLevel,
false,//no parents
true);//simplify
BytesRef[] terms = new BytesRef[cells.size()];
int i = 0;
- for (Node cell : cells) {
+ for (Cell cell : cells) {
terms[i++] = new BytesRef(cell.getTokenString());
}
return new TermsFilter(getFieldName(), terms);
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java Tue May 7 11:20:55 2013
@@ -79,21 +79,21 @@ public class GeohashPrefixTree extends S
}
@Override
- public Node getNode(Point p, int level) {
+ public Cell getCell(Point p, int level) {
return new GhCell(GeohashUtils.encodeLatLon(p.getY(), p.getX(), level));//args are lat,lon (y,x)
}
@Override
- public Node getNode(String token) {
+ public Cell getCell(String token) {
return new GhCell(token);
}
@Override
- public Node getNode(byte[] bytes, int offset, int len) {
+ public Cell getCell(byte[] bytes, int offset, int len) {
return new GhCell(bytes, offset, len);
}
- class GhCell extends Node {
+ class GhCell extends Cell {
GhCell(String token) {
super(token);
}
@@ -109,9 +109,9 @@ public class GeohashPrefixTree extends S
}
@Override
- public Collection<Node> getSubCells() {
+ public Collection<Cell> getSubCells() {
String[] hashes = GeohashUtils.getSubGeohashes(getGeohash());//sorted
- List<Node> cells = new ArrayList<Node>(hashes.length);
+ List<Cell> cells = new ArrayList<Cell>(hashes.length);
for (String hash : hashes) {
cells.add(new GhCell(hash));
}
@@ -124,8 +124,8 @@ public class GeohashPrefixTree extends S
}
@Override
- public Node getSubCell(Point p) {
- return GeohashPrefixTree.this.getNode(p,getLevel()+1);//not performant!
+ public Cell getSubCell(Point p) {
+ return GeohashPrefixTree.this.getCell(p, getLevel() + 1);//not performant!
}
private Shape shape;//cache
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java Tue May 7 11:20:55 2013
@@ -33,7 +33,7 @@ import java.util.Locale;
/**
* A {@link SpatialPrefixTree} which uses a
* <a href="http://en.wikipedia.org/wiki/Quadtree">quad tree</a> in which an
- * indexed term will be generated for each node, 'A', 'B', 'C', 'D'.
+ * indexed term will be generated for each cell, 'A', 'B', 'C', 'D'.
*
* @lucene.experimental
*/
@@ -140,19 +140,19 @@ public class QuadPrefixTree extends Spat
}
@Override
- public Node getNode(Point p, int level) {
- List<Node> cells = new ArrayList<Node>(1);
+ public Cell getCell(Point p, int level) {
+ List<Cell> cells = new ArrayList<Cell>(1);
build(xmid, ymid, 0, cells, new StringBuilder(), ctx.makePoint(p.getX(),p.getY()), level);
return cells.get(0);//note cells could be longer if p on edge
}
@Override
- public Node getNode(String token) {
+ public Cell getCell(String token) {
return new QuadCell(token);
}
@Override
- public Node getNode(byte[] bytes, int offset, int len) {
+ public Cell getCell(byte[] bytes, int offset, int len) {
return new QuadCell(bytes, offset, len);
}
@@ -160,7 +160,7 @@ public class QuadPrefixTree extends Spat
double x,
double y,
int level,
- List<Node> matches,
+ List<Cell> matches,
StringBuilder str,
Shape shape,
int maxLevel) {
@@ -186,7 +186,7 @@ public class QuadPrefixTree extends Spat
double cx,
double cy,
int level,
- List<Node> matches,
+ List<Cell> matches,
StringBuilder str,
Shape shape,
int maxLevel) {
@@ -217,7 +217,7 @@ public class QuadPrefixTree extends Spat
str.setLength(strlen);
}
- class QuadCell extends Node {
+ class QuadCell extends Cell {
public QuadCell(String token) {
super(token);
@@ -239,8 +239,8 @@ public class QuadPrefixTree extends Spat
}
@Override
- public Collection<Node> getSubCells() {
- List<Node> cells = new ArrayList<Node>(4);
+ public Collection<Cell> getSubCells() {
+ List<Cell> cells = new ArrayList<Cell>(4);
cells.add(new QuadCell(getTokenString()+"A"));
cells.add(new QuadCell(getTokenString()+"B"));
cells.add(new QuadCell(getTokenString()+"C"));
@@ -254,8 +254,8 @@ public class QuadPrefixTree extends Spat
}
@Override
- public Node getSubCell(Point p) {
- return QuadPrefixTree.this.getNode(p,getLevel()+1);//not performant!
+ public Cell getSubCell(Point p) {
+ return QuadPrefixTree.this.getCell(p, getLevel() + 1);//not performant!
}
private Shape shape;//cache