You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ja...@apache.org on 2013/05/30 09:53:46 UTC
svn commit: r1487777 [30/50] - in /lucene/dev/branches/security: ./
dev-tools/ dev-tools/eclipse/dot.settings/ dev-tools/idea/.idea/
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/replicator/
dev-tools/maven/ dev-tools/maven/lucene/ dev-tools/ma...
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java Thu May 30 07:53:18 2013
@@ -19,8 +19,8 @@ package org.apache.lucene.queryparser.an
import java.io.IOException;
import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
@@ -31,36 +31,29 @@ import org.apache.lucene.util.Version;
/**
* Overrides Lucene's default QueryParser so that Fuzzy-, Prefix-, Range-, and WildcardQuerys
- * are also passed through the given analyzer, but wild card characters (like <code>*</code>)
- * don't get removed from the search terms.
+ * are also passed through the given analyzer, but wildcard characters <code>*</code> and
+ * <code>?</code> don't get removed from the search terms.
*
* <p><b>Warning:</b> This class should only be used with analyzers that do not use stopwords
* or that add tokens. Also, several stemming analyzers are inappropriate: for example, GermanAnalyzer
* will turn <code>Häuser</code> into <code>hau</code>, but <code>H?user</code> will
* become <code>h?user</code> when using this parser and thus no match would be found (i.e.
* using this parser will be no improvement over QueryParser in such cases).
- *
*/
public class AnalyzingQueryParser extends org.apache.lucene.queryparser.classic.QueryParser {
-
- /**
- * Constructs a query parser.
- * @param field the default field for query terms.
- * @param analyzer used to find terms in the query text.
- */
+ // gobble escaped chars or find a wildcard character
+ private final Pattern wildcardPattern = Pattern.compile("(\\.)|([?*]+)");
public AnalyzingQueryParser(Version matchVersion, String field, Analyzer analyzer) {
super(matchVersion, field, analyzer);
setAnalyzeRangeTerms(true);
}
/**
- * Called when parser
- * parses an input term token that contains one or more wildcard
- * characters (like <code>*</code>), but is not a prefix term token (one
- * that has just a single * character at the end).
+ * Called when parser parses an input term that contains one or more wildcard
+ * characters (like <code>*</code>), but is not a prefix term (one that has
+ * just a single <code>*</code> character at the end).
* <p>
- * Example: will be called for <code>H?user</code> or for <code>H*user</code>
- * but not for <code>*user</code>.
+ * Example: will be called for <code>H?user</code> or for <code>H*user</code>.
* <p>
* Depending on analyzer and settings, a wildcard term may (most probably will)
* be lower-cased automatically. It <b>will</b> go through the default Analyzer.
@@ -68,113 +61,52 @@ public class AnalyzingQueryParser extend
* Overrides super class, by passing terms through analyzer.
*
* @param field Name of the field query will use.
- * @param termStr Term token that contains one or more wild card
+ * @param termStr Term that contains one or more wildcard
* characters (? or *), but is not simple prefix term
*
* @return Resulting {@link Query} built for the term
*/
@Override
protected Query getWildcardQuery(String field, String termStr) throws ParseException {
- List<String> tlist = new ArrayList<String>();
- List<String> wlist = new ArrayList<String>();
- /* somewhat a hack: find/store wildcard chars
- * in order to put them back after analyzing */
- boolean isWithinToken = (!termStr.startsWith("?") && !termStr.startsWith("*"));
- StringBuilder tmpBuffer = new StringBuilder();
- char[] chars = termStr.toCharArray();
- for (int i = 0; i < termStr.length(); i++) {
- if (chars[i] == '?' || chars[i] == '*') {
- if (isWithinToken) {
- tlist.add(tmpBuffer.toString());
- tmpBuffer.setLength(0);
- }
- isWithinToken = false;
- } else {
- if (!isWithinToken) {
- wlist.add(tmpBuffer.toString());
- tmpBuffer.setLength(0);
- }
- isWithinToken = true;
- }
- tmpBuffer.append(chars[i]);
- }
- if (isWithinToken) {
- tlist.add(tmpBuffer.toString());
- } else {
- wlist.add(tmpBuffer.toString());
- }
- // get Analyzer from superclass and tokenize the term
- TokenStream source;
-
- int countTokens = 0;
- try {
- source = getAnalyzer().tokenStream(field, new StringReader(termStr));
- source.reset();
- } catch (IOException e1) {
- throw new RuntimeException(e1);
+ if (termStr == null){
+ //can't imagine this would ever happen
+ throw new ParseException("Passed null value as term to getWildcardQuery");
+ }
+ if ( ! getAllowLeadingWildcard() && (termStr.startsWith("*") || termStr.startsWith("?"))) {
+ throw new ParseException("'*' or '?' not allowed as first character in WildcardQuery"
+ + " unless getAllowLeadingWildcard() returns true");
}
- CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
- while (true) {
- try {
- if (!source.incrementToken()) break;
- } catch (IOException e) {
- break;
- }
- String term = termAtt.toString();
- if (!"".equals(term)) {
- try {
- tlist.set(countTokens++, term);
- } catch (IndexOutOfBoundsException ioobe) {
- countTokens = -1;
- }
- }
- }
- try {
- source.end();
- source.close();
- } catch (IOException e) {
- // ignore
- }
-
- if (countTokens != tlist.size()) {
- /* this means that the analyzer used either added or consumed
- * (common for a stemmer) tokens, and we can't build a WildcardQuery */
- throw new ParseException("Cannot build WildcardQuery with analyzer "
- + getAnalyzer().getClass() + " - tokens added or lost");
+
+ Matcher wildcardMatcher = wildcardPattern.matcher(termStr);
+ StringBuilder sb = new StringBuilder();
+ int last = 0;
+
+ while (wildcardMatcher.find()){
+ // continue if escaped char
+ if (wildcardMatcher.group(1) != null){
+ continue;
+ }
+
+ if (wildcardMatcher.start() > 0){
+ String chunk = termStr.substring(last, wildcardMatcher.start());
+ String analyzed = analyzeSingleChunk(field, termStr, chunk);
+ sb.append(analyzed);
+ }
+ //append the wildcard character
+ sb.append(wildcardMatcher.group(2));
+
+ last = wildcardMatcher.end();
}
-
- if (tlist.size() == 0) {
- return null;
- } else if (tlist.size() == 1) {
- if (wlist != null && wlist.size() == 1) {
- /* if wlist contains one wildcard, it must be at the end, because:
- * 1) wildcards are not allowed in 1st position of a term by QueryParser
- * 2) if wildcard was *not* in end, there would be *two* or more tokens */
- return super.getWildcardQuery(field, tlist.get(0)
- + wlist.get(0).toString());
- } else {
- /* we should never get here! if so, this method was called
- * with a termStr containing no wildcard ... */
- throw new IllegalArgumentException("getWildcardQuery called without wildcard");
- }
- } else {
- /* the term was tokenized, let's rebuild to one token
- * with wildcards put back in postion */
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < tlist.size(); i++) {
- sb.append( tlist.get(i));
- if (wlist != null && wlist.size() > i) {
- sb.append(wlist.get(i));
- }
- }
- return super.getWildcardQuery(field, sb.toString());
+ if (last < termStr.length()){
+ sb.append(analyzeSingleChunk(field, termStr, termStr.substring(last)));
}
+ return super.getWildcardQuery(field, sb.toString());
}
-
+
/**
* Called when parser parses an input term
- * token that uses prefix notation; that is, contains a single '*' wildcard
+ * that uses prefix notation; that is, contains a single '*' wildcard
* character as its last character. Since this is a special case
* of generic wildcard term, and such a query can be optimized easily,
* this usually results in a different query object.
@@ -185,52 +117,19 @@ public class AnalyzingQueryParser extend
* Overrides super class, by passing terms through analyzer.
*
* @param field Name of the field query will use.
- * @param termStr Term token to use for building term for the query
+ * @param termStr Term to use for building term for the query
* (<b>without</b> trailing '*' character!)
*
* @return Resulting {@link Query} built for the term
*/
@Override
protected Query getPrefixQuery(String field, String termStr) throws ParseException {
- // get Analyzer from superclass and tokenize the term
- TokenStream source;
- List<String> tlist = new ArrayList<String>();
- try {
- source = getAnalyzer().tokenStream(field, new StringReader(termStr));
- source.reset();
- } catch (IOException e1) {
- throw new RuntimeException(e1);
- }
- CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
- while (true) {
- try {
- if (!source.incrementToken()) break;
- } catch (IOException e) {
- break;
- }
- tlist.add(termAtt.toString());
- }
-
- try {
- source.end();
- source.close();
- } catch (IOException e) {
- // ignore
- }
-
- if (tlist.size() == 1) {
- return super.getPrefixQuery(field, tlist.get(0));
- } else {
- /* this means that the analyzer used either added or consumed
- * (common for a stemmer) tokens, and we can't build a PrefixQuery */
- throw new ParseException("Cannot build PrefixQuery with analyzer "
- + getAnalyzer().getClass()
- + (tlist.size() > 1 ? " - token(s) added" : " - token consumed"));
- }
+ String analyzed = analyzeSingleChunk(field, termStr, termStr);
+ return super.getPrefixQuery(field, analyzed);
}
/**
- * Called when parser parses an input term token that has the fuzzy suffix (~) appended.
+ * Called when parser parses an input term that has the fuzzy suffix (~) appended.
* <p>
* Depending on analyzer and settings, a fuzzy term may (most probably will)
* be lower-cased automatically. It <b>will</b> go through the default Analyzer.
@@ -238,42 +137,73 @@ public class AnalyzingQueryParser extend
* Overrides super class, by passing terms through analyzer.
*
* @param field Name of the field query will use.
- * @param termStr Term token to use for building term for the query
+ * @param termStr Term to use for building term for the query
*
* @return Resulting {@link Query} built for the term
*/
@Override
protected Query getFuzzyQuery(String field, String termStr, float minSimilarity)
throws ParseException {
- // get Analyzer from superclass and tokenize the term
- TokenStream source = null;
- String nextToken = null;
- boolean multipleTokens = false;
-
- try {
- source = getAnalyzer().tokenStream(field, new StringReader(termStr));
- CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
- source.reset();
- if (source.incrementToken()) {
- nextToken = termAtt.toString();
- }
- multipleTokens = source.incrementToken();
- } catch (IOException e) {
- nextToken = null;
- }
-
- try {
- source.end();
- source.close();
- } catch (IOException e) {
- // ignore
- }
+
+ String analyzed = analyzeSingleChunk(field, termStr, termStr);
+ return super.getFuzzyQuery(field, analyzed, minSimilarity);
+ }
- if (multipleTokens) {
- throw new ParseException("Cannot build FuzzyQuery with analyzer " + getAnalyzer().getClass()
- + " - tokens were added");
+ /**
+ * Returns the analyzed form for the given chunk
+ *
+ * If the analyzer produces more than one output token from the given chunk,
+ * a ParseException is thrown.
+ *
+ * @param field The target field
+ * @param termStr The full term from which the given chunk is excerpted
+ * @param chunk The portion of the given termStr to be analyzed
+ * @return The result of analyzing the given chunk
+ * @throws ParseException when analysis returns other than one output token
+ */
+ protected String analyzeSingleChunk(String field, String termStr, String chunk) throws ParseException{
+ String analyzed = null;
+ TokenStream stream = null;
+ try{
+ stream = getAnalyzer().tokenStream(field, new StringReader(chunk));
+ stream.reset();
+ CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
+ // get first and hopefully only output token
+ if (stream.incrementToken()) {
+ analyzed = termAtt.toString();
+
+ // try to increment again, there should only be one output token
+ StringBuilder multipleOutputs = null;
+ while (stream.incrementToken()) {
+ if (null == multipleOutputs) {
+ multipleOutputs = new StringBuilder();
+ multipleOutputs.append('"');
+ multipleOutputs.append(analyzed);
+ multipleOutputs.append('"');
+ }
+ multipleOutputs.append(',');
+ multipleOutputs.append('"');
+ multipleOutputs.append(termAtt.toString());
+ multipleOutputs.append('"');
+ }
+ stream.end();
+ stream.close();
+ if (null != multipleOutputs) {
+ throw new ParseException(
+ String.format(getLocale(),
+ "Analyzer created multiple terms for \"%s\": %s", chunk, multipleOutputs.toString()));
+ }
+ } else {
+ // nothing returned by analyzer. Was it a stop word and the user accidentally
+ // used an analyzer with stop words?
+ stream.end();
+ stream.close();
+ throw new ParseException(String.format(getLocale(), "Analyzer returned nothing for \"%s\"", chunk));
+ }
+ } catch (IOException e){
+ throw new ParseException(
+ String.format(getLocale(), "IO error while trying to analyze single term: \"%s\"", termStr));
}
-
- return (nextToken == null) ? null : super.getFuzzyQuery(field, nextToken, minSimilarity);
+ return analyzed;
}
}
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/CharStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/CharStream.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/CharStream.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/CharStream.java Thu May 30 07:53:18 2013
@@ -112,4 +112,4 @@ interface CharStream {
void Done();
}
-/* JavaCC - OriginalChecksum=30b94cad7b10d0d81e3a59a1083939d0 (do not edit this line) */
+/* JavaCC - OriginalChecksum=c847dd1920bf7901125a7244125682ad (do not edit this line) */
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/MultiFieldQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/MultiFieldQueryParser.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/MultiFieldQueryParser.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/MultiFieldQueryParser.java Thu May 30 07:53:18 2013
@@ -213,6 +213,22 @@ public class MultiFieldQueryParser exten
}
return super.getRangeQuery(field, part1, part2, startInclusive, endInclusive);
}
+
+
+
+ @Override
+ protected Query getRegexpQuery(String field, String termStr)
+ throws ParseException {
+ if (field == null) {
+ List<BooleanClause> clauses = new ArrayList<BooleanClause>();
+ for (int i = 0; i < fields.length; i++) {
+ clauses.add(new BooleanClause(getRegexpQuery(fields[i], termStr),
+ BooleanClause.Occur.SHOULD));
+ }
+ return getBooleanQuery(clauses, true);
+ }
+ return super.getRegexpQuery(field, termStr);
+ }
/**
* Parses a query which searches on the fields specified.
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/ParseException.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/ParseException.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/ParseException.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/ParseException.java Thu May 30 07:53:18 2013
@@ -184,4 +184,4 @@ public class ParseException extends Exce
}
}
-/* JavaCC - OriginalChecksum=b187d97d5bb75c3fc63d642c1c26ac6e (do not edit this line) */
+/* JavaCC - OriginalChecksum=61602edcb3a15810cbc58f5593eba40d (do not edit this line) */
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.java Thu May 30 07:53:18 2013
@@ -165,7 +165,6 @@ public class QueryParser extends QueryPa
}
// This makes sure that there is no garbage after the query string
- @Override
final public Query TopLevelQuery(String field) throws ParseException {
Query q;
q = Query(field);
@@ -538,7 +537,6 @@ public class QueryParser extends QueryPa
}
/** Reinitialise. */
- @Override
public void ReInit(CharStream stream) {
token_source.ReInit(stream);
token = new Token();
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java Thu May 30 07:53:18 2013
@@ -572,24 +572,53 @@ public abstract class QueryParserBase im
if (severalTokensAtSamePosition || (!quoted && !autoGeneratePhraseQueries)) {
if (positionCount == 1 || (!quoted && !autoGeneratePhraseQueries)) {
// no phrase query:
- BooleanQuery q = newBooleanQuery(positionCount == 1);
-
- BooleanClause.Occur occur = positionCount > 1 && operator == AND_OPERATOR ?
- BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD;
-
- for (int i = 0; i < numTokens; i++) {
- try {
- boolean hasNext = buffer.incrementToken();
- assert hasNext == true;
- termAtt.fillBytesRef();
- } catch (IOException e) {
- // safe to ignore, because we know the number of tokens
+
+ if (positionCount == 1) {
+ // simple case: only one position, with synonyms
+ BooleanQuery q = newBooleanQuery(true);
+ for (int i = 0; i < numTokens; i++) {
+ try {
+ boolean hasNext = buffer.incrementToken();
+ assert hasNext == true;
+ termAtt.fillBytesRef();
+ } catch (IOException e) {
+ // safe to ignore, because we know the number of tokens
+ }
+ Query currentQuery = newTermQuery(
+ new Term(field, BytesRef.deepCopyOf(bytes)));
+ q.add(currentQuery, BooleanClause.Occur.SHOULD);
+ }
+ return q;
+ } else {
+ // multiple positions
+ BooleanQuery q = newBooleanQuery(false);
+ final BooleanClause.Occur occur = operator == Operator.AND ? BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD;
+ Query currentQuery = null;
+ for (int i = 0; i < numTokens; i++) {
+ try {
+ boolean hasNext = buffer.incrementToken();
+ assert hasNext == true;
+ termAtt.fillBytesRef();
+ } catch (IOException e) {
+ // safe to ignore, because we know the number of tokens
+ }
+ if (posIncrAtt != null && posIncrAtt.getPositionIncrement() == 0) {
+ if (!(currentQuery instanceof BooleanQuery)) {
+ Query t = currentQuery;
+ currentQuery = newBooleanQuery(true);
+ ((BooleanQuery)currentQuery).add(t, BooleanClause.Occur.SHOULD);
+ }
+ ((BooleanQuery)currentQuery).add(newTermQuery(new Term(field, BytesRef.deepCopyOf(bytes))), BooleanClause.Occur.SHOULD);
+ } else {
+ if (currentQuery != null) {
+ q.add(currentQuery, occur);
+ }
+ currentQuery = newTermQuery(new Term(field, BytesRef.deepCopyOf(bytes)));
+ }
}
- Query currentQuery = newTermQuery(
- new Term(field, BytesRef.deepCopyOf(bytes)));
q.add(currentQuery, occur);
+ return q;
}
- return q;
}
else {
// phrase query:
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/Token.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/Token.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/Token.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/Token.java Thu May 30 07:53:18 2013
@@ -97,7 +97,6 @@ public class Token implements java.io.Se
/**
* Returns the image.
*/
- @Override
public String toString()
{
return image;
@@ -129,4 +128,4 @@ public class Token implements java.io.Se
}
}
-/* JavaCC - OriginalChecksum=405bb5d2fcd84e94ac1c8f0b12c1f914 (do not edit this line) */
+/* JavaCC - OriginalChecksum=c1e1418b35aa9e47ef8dc98b87423d70 (do not edit this line) */
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/TokenMgrError.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/TokenMgrError.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/TokenMgrError.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/TokenMgrError.java Thu May 30 07:53:18 2013
@@ -121,7 +121,6 @@ public class TokenMgrError extends Error
*
* from this method for such cases in the release version of your parser.
*/
- @Override
public String getMessage() {
return super.getMessage();
}
@@ -145,4 +144,4 @@ public class TokenMgrError extends Error
this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
}
}
-/* JavaCC - OriginalChecksum=f433e1a52b8eadbf12f3fbbbf87fd140 (do not edit this line) */
+/* JavaCC - OriginalChecksum=0c275864a1972d9a01601ab81426872d (do not edit this line) */
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/complexPhrase/ComplexPhraseQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/complexPhrase/ComplexPhraseQueryParser.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/complexPhrase/ComplexPhraseQueryParser.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/complexPhrase/ComplexPhraseQueryParser.java Thu May 30 07:53:18 2013
@@ -363,7 +363,7 @@ public class ComplexPhraseQueryParser ex
@Override
public int hashCode() {
final int prime = 31;
- int result = 1;
+ int result = super.hashCode();
result = prime * result + ((field == null) ? 0 : field.hashCode());
result = prime
* result
@@ -381,6 +381,9 @@ public class ComplexPhraseQueryParser ex
return false;
if (getClass() != obj.getClass())
return false;
+ if (!super.equals(obj)) {
+ return false;
+ }
ComplexPhraseQuery other = (ComplexPhraseQuery) obj;
if (field == null) {
if (other.field != null)
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/CommonQueryParserConfiguration.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/CommonQueryParserConfiguration.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/CommonQueryParserConfiguration.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/CommonQueryParserConfiguration.java Thu May 30 07:53:18 2013
@@ -33,13 +33,9 @@ import org.apache.lucene.search.MultiTer
public interface CommonQueryParserConfiguration {
/**
- * Set to <code>true</code> to allow leading wildcard characters.
- * <p>
- * When set, <code>*</code> or <code>?</code> are allowed as the first
- * character of a PrefixQuery and WildcardQuery. Note that this can produce
- * very slow queries on big indexes.
- * <p>
- * Default: false.
+ * Whether terms of multi-term queries (e.g., wildcard,
+ * prefix, fuzzy and range) should be automatically
+ * lower-cased or not. Default is <code>true</code>.
*/
public void setLowercaseExpandedTerms(boolean lowercaseExpandedTerms);
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/StandardQueryConfigHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/StandardQueryConfigHandler.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/StandardQueryConfigHandler.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/StandardQueryConfigHandler.java Thu May 30 07:53:18 2013
@@ -58,7 +58,7 @@ public class StandardQueryConfigHandler
final public static ConfigurationKey<Boolean> ENABLE_POSITION_INCREMENTS = ConfigurationKey.newInstance();
/**
- * Key used to set whether expanded terms should be expanded
+ * Key used to set whether expanded terms should be lower-cased
*
* @see StandardQueryParser#setLowercaseExpandedTerms(boolean)
* @see StandardQueryParser#getLowercaseExpandedTerms()
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/CharStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/CharStream.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/CharStream.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/CharStream.java Thu May 30 07:53:18 2013
@@ -112,4 +112,4 @@ interface CharStream {
void Done();
}
-/* JavaCC - OriginalChecksum=53b2ec7502d50e2290e86187a6c01270 (do not edit this line) */
+/* JavaCC - OriginalChecksum=c95f1720d9b38046dc5d294b741c44cb (do not edit this line) */
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/ParseException.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/ParseException.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/ParseException.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/ParseException.java Thu May 30 07:53:18 2013
@@ -187,4 +187,4 @@ public class ParseException extends Quer
}
}
-/* JavaCC - OriginalChecksum=4263a02db9988d7a863aa97ad2f6dc67 (do not edit this line) */
+/* JavaCC - OriginalChecksum=81401c29cf6f9909761c636b4778ccc0 (do not edit this line) */
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.java Thu May 30 07:53:18 2013
@@ -58,7 +58,6 @@ public class StandardSyntaxParser implem
* @param query the query string to be parsed.
* @throws ParseException if the parsing fails
*/
- @Override
public QueryNode parse(CharSequence query, CharSequence field) throws QueryNodeParseException {
ReInit(new FastCharStream(new StringReader(query.toString())));
try {
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/Token.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/Token.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/Token.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/Token.java Thu May 30 07:53:18 2013
@@ -97,7 +97,6 @@ public class Token implements java.io.Se
/**
* Returns the image.
*/
- @Override
public String toString()
{
return image;
@@ -129,4 +128,4 @@ public class Token implements java.io.Se
}
}
-/* JavaCC - OriginalChecksum=ea8b1e55950603be28e2f63dcd544ab4 (do not edit this line) */
+/* JavaCC - OriginalChecksum=30bbd23e0dec26f141130dc62a4f6e9d (do not edit this line) */
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/TokenMgrError.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/TokenMgrError.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/TokenMgrError.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/TokenMgrError.java Thu May 30 07:53:18 2013
@@ -121,7 +121,6 @@ public class TokenMgrError extends Error
*
* from this method for such cases in the release version of your parser.
*/
- @Override
public String getMessage() {
return super.getMessage();
}
@@ -145,4 +144,4 @@ public class TokenMgrError extends Error
this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
}
}
-/* JavaCC - OriginalChecksum=be88283d82a985d82a34dda46bcf42d5 (do not edit this line) */
+/* JavaCC - OriginalChecksum=3ca7fbf7de9f2424b131a5499b0a78d0 (do not edit this line) */
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/CharStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/CharStream.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/CharStream.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/CharStream.java Thu May 30 07:53:18 2013
@@ -112,4 +112,4 @@ interface CharStream {
void Done();
}
-/* JavaCC - OriginalChecksum=242ae59b965491e225a44534cbc73b42 (do not edit this line) */
+/* JavaCC - OriginalChecksum=5ca20c9145f29a0f8909470a7f949fe4 (do not edit this line) */
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/ParseException.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/ParseException.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/ParseException.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/ParseException.java Thu May 30 07:53:18 2013
@@ -184,4 +184,4 @@ public class ParseException extends Exce
}
}
-/* JavaCC - OriginalChecksum=bd8163f41bf2fd1bb00f025fce3dcaaf (do not edit this line) */
+/* JavaCC - OriginalChecksum=be6f55e3bf157e8c96b4c06cca5ec81b (do not edit this line) */
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.java Thu May 30 07:53:18 2013
@@ -52,7 +52,7 @@ public class QueryParser implements Quer
/* CHECKME: These should be the same as for the tokenizer. How? */
final char truncator = '*';
final char anyChar = '?';
- final char quote = '\u005c"';
+ final char quote = '"';
final char fieldOperator = ':';
final char comma = ','; /* prefix list separator */
final char carat = '^'; /* weight operator */
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.jj
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.jj?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.jj (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.jj Thu May 30 07:53:18 2013
@@ -81,7 +81,7 @@ public class QueryParser {
/* CHECKME: These should be the same as for the tokenizer. How? */
final char truncator = '*';
final char anyChar = '?';
- final char quote = '\"';
+ final char quote = '"';
final char fieldOperator = ':';
final char comma = ','; /* prefix list separator */
final char carat = '^'; /* weight operator */
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/Token.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/Token.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/Token.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/Token.java Thu May 30 07:53:18 2013
@@ -97,7 +97,6 @@ public class Token implements java.io.Se
/**
* Returns the image.
*/
- @Override
public String toString()
{
return image;
@@ -129,4 +128,4 @@ public class Token implements java.io.Se
}
}
-/* JavaCC - OriginalChecksum=f2df701e24da1cf2d025118ce6efdd2f (do not edit this line) */
+/* JavaCC - OriginalChecksum=db38f23b3674db52ff034369707a0ac3 (do not edit this line) */
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/TokenMgrError.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/TokenMgrError.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/TokenMgrError.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/TokenMgrError.java Thu May 30 07:53:18 2013
@@ -121,7 +121,6 @@ public class TokenMgrError extends Error
*
* from this method for such cases in the release version of your parser.
*/
- @Override
public String getMessage() {
return super.getMessage();
}
@@ -145,4 +144,4 @@ public class TokenMgrError extends Error
this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
}
}
-/* JavaCC - OriginalChecksum=8c69a370d9a9893140562c8bb911678c (do not edit this line) */
+/* JavaCC - OriginalChecksum=dcdd5ccde13b91bcd8f76a86ca618852 (do not edit this line) */
Modified: lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java Thu May 30 07:53:18 2013
@@ -19,8 +19,17 @@ package org.apache.lucene.queryparser.an
import java.io.IOException;
import java.io.Reader;
+import java.util.Map;
+import java.util.TreeMap;
-import org.apache.lucene.analysis.*;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockBytesAnalyzer;
+import org.apache.lucene.analysis.MockTokenFilter;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -33,11 +42,14 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.Query;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
/**
*/
+@SuppressCodecs("Lucene3x") // binary terms
public class TestAnalyzingQueryParser extends LuceneTestCase {
-
+ private final static String FIELD = "field";
+
private Analyzer a;
private String[] wildcardInput;
@@ -49,12 +61,15 @@ public class TestAnalyzingQueryParser ex
private String[] fuzzyInput;
private String[] fuzzyExpected;
+ private Map<String, String> wildcardEscapeHits = new TreeMap<String, String>();
+ private Map<String, String> wildcardEscapeMisses = new TreeMap<String, String>();
+
@Override
public void setUp() throws Exception {
super.setUp();
- wildcardInput = new String[] { "übersetzung über*ung",
+ wildcardInput = new String[] { "*bersetzung über*ung",
"Mötley Cr\u00fce Mötl?* Crü?", "Renée Zellweger Ren?? Zellw?ger" };
- wildcardExpected = new String[] { "ubersetzung uber*ung", "motley crue motl?* cru?",
+ wildcardExpected = new String[] { "*bersetzung uber*ung", "motley crue motl?* cru?",
"renee zellweger ren?? zellw?ger" };
prefixInput = new String[] { "übersetzung übersetz*",
@@ -71,43 +86,138 @@ public class TestAnalyzingQueryParser ex
fuzzyExpected = new String[] { "ubersetzung ubersetzung~1",
"motley crue motley~1 crue~2", "renee zellweger renee~0 zellweger~2" };
+ wildcardEscapeHits.put("mö*tley", "moatley");
+
+ // need to have at least one genuine wildcard to trigger the wildcard analysis
+ // hence the * before the y
+ wildcardEscapeHits.put("mö\\*tl*y", "mo*tley");
+
+ // escaped backslash then true wildcard
+ wildcardEscapeHits.put("mö\\\\*tley", "mo\\atley");
+
+ // escaped wildcard then true wildcard
+ wildcardEscapeHits.put("mö\\??ley", "mo?tley");
+
+ // the first is an escaped * which should yield a miss
+ wildcardEscapeMisses.put("mö\\*tl*y", "moatley");
+
a = new ASCIIAnalyzer();
}
+ public void testSingleChunkExceptions() {
+ boolean ex = false;
+ String termStr = "the*tre";
+
+ Analyzer stopsAnalyzer = new MockAnalyzer
+ (random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
+ try {
+ String q = parseWithAnalyzingQueryParser(termStr, stopsAnalyzer, true);
+ } catch (ParseException e){
+ if (e.getMessage().contains("returned nothing")){
+ ex = true;
+ }
+ }
+ assertEquals("Should have returned nothing", true, ex);
+ ex = false;
+
+ AnalyzingQueryParser qp = new AnalyzingQueryParser(TEST_VERSION_CURRENT, FIELD, a);
+ try{
+ qp.analyzeSingleChunk(FIELD, "", "not a single chunk");
+ } catch (ParseException e){
+ if (e.getMessage().contains("multiple terms")){
+ ex = true;
+ }
+ }
+ assertEquals("Should have produced multiple terms", true, ex);
+ }
+
+ public void testWildcardAlone() throws ParseException {
+ //seems like crazy edge case, but can be useful in concordance
+ boolean pex = false;
+ try{
+ Query q = getAnalyzedQuery("*", a, false);
+ } catch (ParseException e){
+ pex = true;
+ }
+ assertEquals("Wildcard alone with allowWildcard=false", true, pex);
+
+ pex = false;
+ try {
+ String qString = parseWithAnalyzingQueryParser("*", a, true);
+ assertEquals("Every word", "*", qString);
+ } catch (ParseException e){
+ pex = true;
+ }
+
+ assertEquals("Wildcard alone with allowWildcard=true", false, pex);
+
+ }
+ public void testWildCardEscapes() throws ParseException, IOException {
+
+ for (Map.Entry<String, String> entry : wildcardEscapeHits.entrySet()){
+ Query q = getAnalyzedQuery(entry.getKey(), a, false);
+ assertEquals("WildcardEscapeHits: " + entry.getKey(), true, isAHit(q, entry.getValue(), a));
+ }
+ for (Map.Entry<String, String> entry : wildcardEscapeMisses.entrySet()){
+ Query q = getAnalyzedQuery(entry.getKey(), a, false);
+ assertEquals("WildcardEscapeMisses: " + entry.getKey(), false, isAHit(q, entry.getValue(), a));
+ }
+
+ }
+ public void testWildCardQueryNoLeadingAllowed() {
+ boolean ex = false;
+ try{
+ String q = parseWithAnalyzingQueryParser(wildcardInput[0], a, false);
+
+ } catch (ParseException e){
+ ex = true;
+ }
+ assertEquals("Testing initial wildcard not allowed",
+ true, ex);
+ }
+
public void testWildCardQuery() throws ParseException {
for (int i = 0; i < wildcardInput.length; i++) {
assertEquals("Testing wildcards with analyzer " + a.getClass() + ", input string: "
- + wildcardInput[i], wildcardExpected[i], parseWithAnalyzingQueryParser(wildcardInput[i], a));
+ + wildcardInput[i], wildcardExpected[i], parseWithAnalyzingQueryParser(wildcardInput[i], a, true));
}
}
+
public void testPrefixQuery() throws ParseException {
for (int i = 0; i < prefixInput.length; i++) {
assertEquals("Testing prefixes with analyzer " + a.getClass() + ", input string: "
- + prefixInput[i], prefixExpected[i], parseWithAnalyzingQueryParser(prefixInput[i], a));
+ + prefixInput[i], prefixExpected[i], parseWithAnalyzingQueryParser(prefixInput[i], a, false));
}
}
public void testRangeQuery() throws ParseException {
for (int i = 0; i < rangeInput.length; i++) {
assertEquals("Testing ranges with analyzer " + a.getClass() + ", input string: "
- + rangeInput[i], rangeExpected[i], parseWithAnalyzingQueryParser(rangeInput[i], a));
+ + rangeInput[i], rangeExpected[i], parseWithAnalyzingQueryParser(rangeInput[i], a, false));
}
}
public void testFuzzyQuery() throws ParseException {
for (int i = 0; i < fuzzyInput.length; i++) {
assertEquals("Testing fuzzys with analyzer " + a.getClass() + ", input string: "
- + fuzzyInput[i], fuzzyExpected[i], parseWithAnalyzingQueryParser(fuzzyInput[i], a));
+ + fuzzyInput[i], fuzzyExpected[i], parseWithAnalyzingQueryParser(fuzzyInput[i], a, false));
}
}
- private String parseWithAnalyzingQueryParser(String s, Analyzer a) throws ParseException {
- AnalyzingQueryParser qp = new AnalyzingQueryParser(TEST_VERSION_CURRENT, "field", a);
+
+ private String parseWithAnalyzingQueryParser(String s, Analyzer a, boolean allowLeadingWildcard) throws ParseException {
+ Query q = getAnalyzedQuery(s, a, allowLeadingWildcard);
+ return q.toString(FIELD);
+ }
+
+ private Query getAnalyzedQuery(String s, Analyzer a, boolean allowLeadingWildcard) throws ParseException {
+ AnalyzingQueryParser qp = new AnalyzingQueryParser(TEST_VERSION_CURRENT, FIELD, a);
+ qp.setAllowLeadingWildcard(allowLeadingWildcard);
org.apache.lucene.search.Query q = qp.parse(s);
- return q.toString("field");
+ return q;
}
-
+
final static class FoldingFilter extends TokenFilter {
final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
@@ -144,31 +254,45 @@ public class TestAnalyzingQueryParser ex
final static class ASCIIAnalyzer extends Analyzer {
@Override
public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ Tokenizer result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(result, new FoldingFilter(result));
}
}
-
+
+
// LUCENE-4176
public void testByteTerms() throws Exception {
- Directory ramDir = newDirectory();
+ String s = "à¹à¸";
Analyzer analyzer = new MockBytesAnalyzer();
+ QueryParser qp = new AnalyzingQueryParser(TEST_VERSION_CURRENT, FIELD, analyzer);
+ Query q = qp.parse("[à¹à¸ TO à¹à¸]");
+ assertEquals(true, isAHit(q, s, analyzer));
+ }
+
+
+ private boolean isAHit(Query q, String content, Analyzer analyzer) throws IOException{
+ Directory ramDir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), ramDir, analyzer);
Document doc = new Document();
FieldType fieldType = new FieldType();
fieldType.setIndexed(true);
fieldType.setTokenized(true);
fieldType.setStored(true);
- Field field = new Field("content","à¹à¸", fieldType);
+ Field field = new Field(FIELD, content, fieldType);
doc.add(field);
writer.addDocument(doc);
writer.close();
DirectoryReader ir = DirectoryReader.open(ramDir);
IndexSearcher is = new IndexSearcher(ir);
- QueryParser qp = new AnalyzingQueryParser(TEST_VERSION_CURRENT, "content", analyzer);
- Query q = qp.parse("[à¹à¸ TO à¹à¸]");
- assertEquals(1, is.search(q, 10).totalHits);
+
+ int hits = is.search(q, 10).totalHits;
ir.close();
ramDir.close();
+ if (hits == 1){
+ return true;
+ } else {
+ return false;
+ }
+
}
}
\ No newline at end of file
Modified: lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java Thu May 30 07:53:18 2013
@@ -28,10 +28,13 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.RegexpQuery;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
@@ -294,7 +297,7 @@ public class TestMultiFieldQueryParser e
mfqp.setDefaultOperator(QueryParser.Operator.AND);
Query q = mfqp.parse("the footest");
IndexReader ir = DirectoryReader.open(ramDir);
- IndexSearcher is = new IndexSearcher(ir);
+ IndexSearcher is = newSearcher(ir);
ScoreDoc[] hits = is.search(q, null, 1000).scoreDocs;
assertEquals(1, hits.length);
ir.close();
@@ -328,5 +331,15 @@ public class TestMultiFieldQueryParser e
return stdAnalyzer.createComponents(fieldName, reader);
}
}
+
+ public void testSimpleRegex() throws ParseException {
+ String[] fields = new String[] {"a", "b"};
+ MultiFieldQueryParser mfqp = new MultiFieldQueryParser(TEST_VERSION_CURRENT, fields, new MockAnalyzer(random()));
+
+ BooleanQuery bq = new BooleanQuery(true);
+ bq.add(new RegexpQuery(new Term("a", "[a-z][123]")), Occur.SHOULD);
+ bq.add(new RegexpQuery(new Term("b", "[a-z][123]")), Occur.SHOULD);
+ assertEquals(bq, mfqp.parse("/[a-z][123]/"));
+ }
}
Modified: lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java Thu May 30 07:53:18 2013
@@ -17,9 +17,17 @@ package org.apache.lucene.queryparser.cl
* limitations under the License.
*/
+import java.io.IOException;
+import java.io.Reader;
+
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.DateTools.Resolution;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.QueryParser.Operator;
@@ -27,6 +35,7 @@ import org.apache.lucene.queryparser.fle
import org.apache.lucene.queryparser.util.QueryParserTestBase;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.MultiPhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
@@ -307,4 +316,178 @@ public class TestQueryParser extends Que
assertEquals(unexpanded, smart.parse("\"dogs\""));
}
+ // TODO: fold these into QueryParserTestBase
+
+ /** adds synonym of "dog" for "dogs". */
+ static class MockSynonymAnalyzer extends Analyzer {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ MockTokenizer tokenizer = new MockTokenizer(reader);
+ return new TokenStreamComponents(tokenizer, new MockSynonymFilter(tokenizer));
+ }
+ }
+
+ /** simple synonyms test */
+ public void testSynonyms() throws Exception {
+ BooleanQuery expected = new BooleanQuery(true);
+ expected.add(new TermQuery(new Term("field", "dogs")), BooleanClause.Occur.SHOULD);
+ expected.add(new TermQuery(new Term("field", "dog")), BooleanClause.Occur.SHOULD);
+ QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new MockSynonymAnalyzer());
+ assertEquals(expected, qp.parse("dogs"));
+ assertEquals(expected, qp.parse("\"dogs\""));
+ qp.setDefaultOperator(Operator.AND);
+ assertEquals(expected, qp.parse("dogs"));
+ assertEquals(expected, qp.parse("\"dogs\""));
+ expected.setBoost(2.0f);
+ assertEquals(expected, qp.parse("dogs^2"));
+ assertEquals(expected, qp.parse("\"dogs\"^2"));
+ }
+
+ /** forms multiphrase query */
+ public void testSynonymsPhrase() throws Exception {
+ MultiPhraseQuery expected = new MultiPhraseQuery();
+ expected.add(new Term("field", "old"));
+ expected.add(new Term[] { new Term("field", "dogs"), new Term("field", "dog") });
+ QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new MockSynonymAnalyzer());
+ assertEquals(expected, qp.parse("\"old dogs\""));
+ qp.setDefaultOperator(Operator.AND);
+ assertEquals(expected, qp.parse("\"old dogs\""));
+ expected.setBoost(2.0f);
+ assertEquals(expected, qp.parse("\"old dogs\"^2"));
+ expected.setSlop(3);
+ assertEquals(expected, qp.parse("\"old dogs\"~3^2"));
+ }
+
+ /**
+ * adds synonym of "å" for "å½".
+ */
+ protected static class MockCJKSynonymFilter extends TokenFilter {
+ CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
+ boolean addSynonym = false;
+
+ public MockCJKSynonymFilter(TokenStream input) {
+ super(input);
+ }
+
+ @Override
+ public final boolean incrementToken() throws IOException {
+ if (addSynonym) { // inject our synonym
+ clearAttributes();
+ termAtt.setEmpty().append("å");
+ posIncAtt.setPositionIncrement(0);
+ addSynonym = false;
+ return true;
+ }
+
+ if (input.incrementToken()) {
+ addSynonym = termAtt.toString().equals("å½");
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+
+ static class MockCJKSynonymAnalyzer extends Analyzer {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new SimpleCJKTokenizer(reader);
+ return new TokenStreamComponents(tokenizer, new MockCJKSynonymFilter(tokenizer));
+ }
+ }
+
+ /** simple CJK synonym test */
+ public void testCJKSynonym() throws Exception {
+ BooleanQuery expected = new BooleanQuery(true);
+ expected.add(new TermQuery(new Term("field", "å½")), BooleanClause.Occur.SHOULD);
+ expected.add(new TermQuery(new Term("field", "å")), BooleanClause.Occur.SHOULD);
+ QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new MockCJKSynonymAnalyzer());
+ assertEquals(expected, qp.parse("å½"));
+ qp.setDefaultOperator(Operator.AND);
+ assertEquals(expected, qp.parse("å½"));
+ expected.setBoost(2.0f);
+ assertEquals(expected, qp.parse("å½^2"));
+ }
+
+ /** synonyms with default OR operator */
+ public void testCJKSynonymsOR() throws Exception {
+ BooleanQuery expected = new BooleanQuery();
+ expected.add(new TermQuery(new Term("field", "ä¸")), BooleanClause.Occur.SHOULD);
+ BooleanQuery inner = new BooleanQuery(true);
+ inner.add(new TermQuery(new Term("field", "å½")), BooleanClause.Occur.SHOULD);
+ inner.add(new TermQuery(new Term("field", "å")), BooleanClause.Occur.SHOULD);
+ expected.add(inner, BooleanClause.Occur.SHOULD);
+ QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new MockCJKSynonymAnalyzer());
+ assertEquals(expected, qp.parse("ä¸å½"));
+ expected.setBoost(2.0f);
+ assertEquals(expected, qp.parse("ä¸å½^2"));
+ }
+
+ /** more complex synonyms with default OR operator */
+ public void testCJKSynonymsOR2() throws Exception {
+ BooleanQuery expected = new BooleanQuery();
+ expected.add(new TermQuery(new Term("field", "ä¸")), BooleanClause.Occur.SHOULD);
+ BooleanQuery inner = new BooleanQuery(true);
+ inner.add(new TermQuery(new Term("field", "å½")), BooleanClause.Occur.SHOULD);
+ inner.add(new TermQuery(new Term("field", "å")), BooleanClause.Occur.SHOULD);
+ expected.add(inner, BooleanClause.Occur.SHOULD);
+ BooleanQuery inner2 = new BooleanQuery(true);
+ inner2.add(new TermQuery(new Term("field", "å½")), BooleanClause.Occur.SHOULD);
+ inner2.add(new TermQuery(new Term("field", "å")), BooleanClause.Occur.SHOULD);
+ expected.add(inner2, BooleanClause.Occur.SHOULD);
+ QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new MockCJKSynonymAnalyzer());
+ assertEquals(expected, qp.parse("ä¸å½å½"));
+ expected.setBoost(2.0f);
+ assertEquals(expected, qp.parse("ä¸å½å½^2"));
+ }
+
+ /** synonyms with default AND operator */
+ public void testCJKSynonymsAND() throws Exception {
+ BooleanQuery expected = new BooleanQuery();
+ expected.add(new TermQuery(new Term("field", "ä¸")), BooleanClause.Occur.MUST);
+ BooleanQuery inner = new BooleanQuery(true);
+ inner.add(new TermQuery(new Term("field", "å½")), BooleanClause.Occur.SHOULD);
+ inner.add(new TermQuery(new Term("field", "å")), BooleanClause.Occur.SHOULD);
+ expected.add(inner, BooleanClause.Occur.MUST);
+ QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new MockCJKSynonymAnalyzer());
+ qp.setDefaultOperator(Operator.AND);
+ assertEquals(expected, qp.parse("ä¸å½"));
+ expected.setBoost(2.0f);
+ assertEquals(expected, qp.parse("ä¸å½^2"));
+ }
+
+ /** more complex synonyms with default AND operator */
+ public void testCJKSynonymsAND2() throws Exception {
+ BooleanQuery expected = new BooleanQuery();
+ expected.add(new TermQuery(new Term("field", "ä¸")), BooleanClause.Occur.MUST);
+ BooleanQuery inner = new BooleanQuery(true);
+ inner.add(new TermQuery(new Term("field", "å½")), BooleanClause.Occur.SHOULD);
+ inner.add(new TermQuery(new Term("field", "å")), BooleanClause.Occur.SHOULD);
+ expected.add(inner, BooleanClause.Occur.MUST);
+ BooleanQuery inner2 = new BooleanQuery(true);
+ inner2.add(new TermQuery(new Term("field", "å½")), BooleanClause.Occur.SHOULD);
+ inner2.add(new TermQuery(new Term("field", "å")), BooleanClause.Occur.SHOULD);
+ expected.add(inner2, BooleanClause.Occur.MUST);
+ QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new MockCJKSynonymAnalyzer());
+ qp.setDefaultOperator(Operator.AND);
+ assertEquals(expected, qp.parse("ä¸å½å½"));
+ expected.setBoost(2.0f);
+ assertEquals(expected, qp.parse("ä¸å½å½^2"));
+ }
+
+ /** forms multiphrase query */
+ public void testCJKSynonymsPhrase() throws Exception {
+ MultiPhraseQuery expected = new MultiPhraseQuery();
+ expected.add(new Term("field", "ä¸"));
+ expected.add(new Term[] { new Term("field", "å½"), new Term("field", "å")});
+ QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new MockCJKSynonymAnalyzer());
+ qp.setDefaultOperator(Operator.AND);
+ assertEquals(expected, qp.parse("\"ä¸å½\""));
+ expected.setBoost(2.0f);
+ assertEquals(expected, qp.parse("\"ä¸å½\"^2"));
+ expected.setSlop(3);
+ assertEquals(expected, qp.parse("\"ä¸å½\"~3^2"));
+ }
+
}
Modified: lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/complexPhrase/TestComplexPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/complexPhrase/TestComplexPhraseQuery.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/complexPhrase/TestComplexPhraseQuery.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/complexPhrase/TestComplexPhraseQuery.java Thu May 30 07:53:18 2013
@@ -127,7 +127,7 @@ public class TestComplexPhraseQuery exte
}
w.close();
reader = DirectoryReader.open(rd);
- searcher = new IndexSearcher(reader);
+ searcher = newSearcher(reader);
}
@Override
Modified: lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java Thu May 30 07:53:18 2013
@@ -546,7 +546,7 @@ public class TestPrecedenceQueryParser e
public void testBoost() throws Exception {
CharacterRunAutomaton stopSet = new CharacterRunAutomaton(BasicAutomata.makeString("on"));
- Analyzer oneStopAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopSet, true);
+ Analyzer oneStopAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopSet);
PrecedenceQueryParser qp = new PrecedenceQueryParser();
qp.setAnalyzer(oneStopAnalyzer);
@@ -561,7 +561,7 @@ public class TestPrecedenceQueryParser e
q = qp.parse("\"on\"^1.0", "field");
assertNotNull(q);
- q = getParser(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true)).parse("the^3",
+ q = getParser(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET)).parse("the^3",
"field");
assertNotNull(q);
}
Modified: lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java Thu May 30 07:53:18 2013
@@ -333,7 +333,7 @@ public class TestMultiFieldQPHelper exte
mfqp.setDefaultOperator(StandardQueryConfigHandler.Operator.AND);
Query q = mfqp.parse("the footest", null);
IndexReader ir = DirectoryReader.open(ramDir);
- IndexSearcher is = new IndexSearcher(ir);
+ IndexSearcher is = newSearcher(ir);
ScoreDoc[] hits = is.search(q, null, 1000).scoreDocs;
assertEquals(1, hits.length);
ir.close();
Modified: lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java Thu May 30 07:53:18 2013
@@ -946,7 +946,7 @@ public class TestQPHelper extends Lucene
public void testBoost() throws Exception {
CharacterRunAutomaton stopSet = new CharacterRunAutomaton(BasicAutomata.makeString("on"));
- Analyzer oneStopAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopSet, true);
+ Analyzer oneStopAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopSet);
StandardQueryParser qp = new StandardQueryParser();
qp.setAnalyzer(oneStopAnalyzer);
@@ -962,7 +962,7 @@ public class TestQPHelper extends Lucene
assertNotNull(q);
StandardQueryParser qp2 = new StandardQueryParser();
- qp2.setAnalyzer(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true));
+ qp2.setAnalyzer(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET));
q = qp2.parse("the^3", "field");
// "the" is a stop word so the result is an empty query:
@@ -1179,7 +1179,7 @@ public class TestQPHelper extends Lucene
public void testStopwords() throws Exception {
StandardQueryParser qp = new StandardQueryParser();
CharacterRunAutomaton stopSet = new CharacterRunAutomaton(new RegExp("the|foo").toAutomaton());
- qp.setAnalyzer(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopSet, true));
+ qp.setAnalyzer(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopSet));
Query result = qp.parse("a:the OR a:foo", "a");
assertNotNull("result is null and it shouldn't be", result);
@@ -1203,7 +1203,7 @@ public class TestQPHelper extends Lucene
public void testPositionIncrement() throws Exception {
StandardQueryParser qp = new StandardQueryParser();
qp.setAnalyzer(
- new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true));
+ new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET));
qp.setEnablePositionIncrements(true);
Modified: lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java Thu May 30 07:53:18 2013
@@ -236,7 +236,7 @@ public abstract class QueryParserTestBas
}
//individual CJK chars as terms, like StandardAnalyzer
- private class SimpleCJKTokenizer extends Tokenizer {
+ protected static class SimpleCJKTokenizer extends Tokenizer {
private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
public SimpleCJKTokenizer(Reader input) {
@@ -244,7 +244,7 @@ public abstract class QueryParserTestBas
}
@Override
- public boolean incrementToken() throws IOException {
+ public final boolean incrementToken() throws IOException {
int ch = input.read();
if (ch < 0)
return false;
@@ -852,7 +852,7 @@ public abstract class QueryParserTestBas
public void testBoost()
throws Exception {
CharacterRunAutomaton stopWords = new CharacterRunAutomaton(BasicAutomata.makeString("on"));
- Analyzer oneStopAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopWords, true);
+ Analyzer oneStopAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopWords);
CommonQueryParserConfiguration qp = getParserConfig(oneStopAnalyzer);
Query q = getQuery("on^1.0",qp);
assertNotNull(q);
@@ -865,7 +865,7 @@ public abstract class QueryParserTestBas
q = getQuery("\"on\"^1.0",qp);
assertNotNull(q);
- Analyzer a2 = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true);
+ Analyzer a2 = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
CommonQueryParserConfiguration qp2 = getParserConfig(a2);
q = getQuery("the^3", qp2);
// "the" is a stop word so the result is an empty query:
@@ -1007,7 +1007,7 @@ public abstract class QueryParserTestBas
public void testStopwords() throws Exception {
CharacterRunAutomaton stopSet = new CharacterRunAutomaton(new RegExp("the|foo").toAutomaton());
- CommonQueryParserConfiguration qp = getParserConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopSet, true));
+ CommonQueryParserConfiguration qp = getParserConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopSet));
Query result = getQuery("field:the OR field:foo",qp);
assertNotNull("result is null and it shouldn't be", result);
assertTrue("result is not a BooleanQuery", result instanceof BooleanQuery);
@@ -1023,7 +1023,7 @@ public abstract class QueryParserTestBas
}
public void testPositionIncrement() throws Exception {
- CommonQueryParserConfiguration qp = getParserConfig( new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true));
+ CommonQueryParserConfiguration qp = getParserConfig( new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET));
qp.setEnablePositionIncrements(true);
String qtxt = "\"the words in poisitions pos02578 are stopped in this phrasequery\"";
// 0 2 5 7 8
@@ -1070,7 +1070,7 @@ public abstract class QueryParserTestBas
// "match"
public void testPositionIncrements() throws Exception {
Directory dir = newDirectory();
- Analyzer a = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true);
+ Analyzer a = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, a));
Document doc = new Document();
doc.add(newTextField("field", "the wizard of ozzy", Field.Store.NO));
@@ -1088,7 +1088,7 @@ public abstract class QueryParserTestBas
/**
* adds synonym of "dog" for "dogs".
*/
- private class MockSynonymFilter extends TokenFilter {
+ protected static class MockSynonymFilter extends TokenFilter {
CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
boolean addSynonym = false;
@@ -1185,7 +1185,7 @@ public abstract class QueryParserTestBas
}
public void testPhraseQueryToString() throws Exception {
- Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true);
+ Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
CommonQueryParserConfiguration qp = getParserConfig(analyzer);
qp.setEnablePositionIncrements(true);
PhraseQuery q = (PhraseQuery)getQuery("\"this hi this is a test is\"", qp);
@@ -1235,26 +1235,13 @@ public abstract class QueryParserTestBas
CharacterRunAutomaton stopStopList =
new CharacterRunAutomaton(new RegExp("[sS][tT][oO][pP]").toAutomaton());
- CommonQueryParserConfiguration qp = getParserConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false, stopStopList, false));
-
- PhraseQuery phraseQuery = new PhraseQuery();
- phraseQuery.add(new Term("field", "1"));
- phraseQuery.add(new Term("field", "2"));
-
- assertEquals(phraseQuery, getQuery("\"1 2\"",qp));
- assertEquals(phraseQuery, getQuery("\"1 stop 2\"",qp));
-
- qp.setEnablePositionIncrements(true);
- assertEquals(phraseQuery, getQuery("\"1 stop 2\"",qp));
-
- qp.setEnablePositionIncrements(false);
- assertEquals(phraseQuery, getQuery("\"1 stop 2\"",qp));
+ CommonQueryParserConfiguration qp = getParserConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false, stopStopList));
qp = getParserConfig(
- new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false, stopStopList, true));
+ new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false, stopStopList));
qp.setEnablePositionIncrements(true);
- phraseQuery = new PhraseQuery();
+ PhraseQuery phraseQuery = new PhraseQuery();
phraseQuery.add(new Term("field", "1"));
phraseQuery.add(new Term("field", "2"), 2);
assertEquals(phraseQuery, getQuery("\"1 stop 2\"",qp));
Modified: lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java Thu May 30 07:53:18 2013
@@ -58,7 +58,7 @@ public class TestParser extends LuceneTe
@BeforeClass
public static void beforeClass() throws Exception {
// TODO: rewrite test (this needs to set QueryParser.enablePositionIncrements, too, for work with CURRENT):
- Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET, false);
+ Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
//initialize the parser
builder = new CorePlusExtensionsParser("contents", analyzer);
Modified: lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestQueryTemplateManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestQueryTemplateManager.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestQueryTemplateManager.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestQueryTemplateManager.java Thu May 30 07:53:18 2013
@@ -152,7 +152,7 @@ public class TestQueryTemplateManager ex
w.forceMerge(1);
w.close();
reader = DirectoryReader.open(dir);
- searcher = new IndexSearcher(reader);
+ searcher = newSearcher(reader);
//initialize the parser
builder = new CorePlusExtensionsParser("artist", analyzer);
Modified: lucene/dev/branches/security/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java (original)
+++ lucene/dev/branches/security/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java Thu May 30 07:53:18 2013
@@ -72,7 +72,7 @@ public class FuzzyLikeThisQuery extends
@Override
public int hashCode() {
final int prime = 31;
- int result = 1;
+ int result = super.hashCode();
result = prime * result + ((analyzer == null) ? 0 : analyzer.hashCode());
result = prime * result
+ ((fieldVals == null) ? 0 : fieldVals.hashCode());
@@ -89,6 +89,9 @@ public class FuzzyLikeThisQuery extends
return false;
if (getClass() != obj.getClass())
return false;
+ if (!super.equals(obj)) {
+ return false;
+ }
FuzzyLikeThisQuery other = (FuzzyLikeThisQuery) obj;
if (analyzer == null) {
if (other.analyzer != null)
Modified: lucene/dev/branches/security/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java (original)
+++ lucene/dev/branches/security/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java Thu May 30 07:53:18 2013
@@ -69,7 +69,7 @@ public class TestSpanRegexQuery extends
writer.close();
IndexReader reader = DirectoryReader.open(directory);
- IndexSearcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
SpanQuery srq = new SpanMultiTermQueryWrapper<RegexQuery>(new RegexQuery(new Term("field", "aut.*")));
SpanFirstQuery sfq = new SpanFirstQuery(srq, 1);
// SpanNearQuery query = new SpanNearQuery(new SpanQuery[] {srq, stq}, 6,
Modified: lucene/dev/branches/security/lucene/site/changes/changes2html.pl
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/site/changes/changes2html.pl?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/site/changes/changes2html.pl (original)
+++ lucene/dev/branches/security/lucene/site/changes/changes2html.pl Thu May 30 07:53:18 2013
@@ -78,9 +78,9 @@ for (my $line_num = 0 ; $line_num <= $#l
($reldate, $relinfo) = get_release_date($release, $relinfo);
$sections = [];
push @releases, [ $release, $reldate, $relinfo, $sections ];
- ($first_relid = lc($release)) =~ s/\s+/_/g
+ ($first_relid = 'v'.lc($release)) =~ s/\s+/_/g
if ($#releases == 0 or ($#releases == 1 and not ($releases[0][0])));
- ($second_relid = lc($release)) =~ s/\s+/_/g
+ ($second_relid = 'v'.lc($release)) =~ s/\s+/_/g
if ( ($#releases == 1 and $releases[0][0])
or ($#releases == 2 and not $releases[0][0]));
$items = undef;
@@ -123,6 +123,11 @@ for (my $line_num = 0 ; $line_num <= $#l
# Make a fake release to hold pre-release sections
push @releases, [ undef, undef, undef, $sections ];
}
+ for my $section (@$sections) {
+ if ($heading eq $section->[0]) {
+ die "Section '$heading' appears more than once under release '$releases[-1][0]'";
+ }
+ }
push @$sections, [ $heading, $items ];
$in_major_component_versions_section
= ($heading =~ /Versions of Major Components/i);
@@ -416,7 +421,7 @@ for my $rel (@releases) {
my $relid = '';
if ($release) { # Pre-release sections have no release ID
++$relcnt;
- ($relid = lc($release)) =~ s/\s+/_/g;
+ ($relid = 'v'.lc($release)) =~ s/\s+/_/g;
print "<$header>";
print "<a id=\"$relid\" href=\"javascript:toggleList('$relid')\">"
unless ($release =~ /^20\d\d/);
Modified: lucene/dev/branches/security/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeFilter.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeFilter.java (original)
+++ lucene/dev/branches/security/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeFilter.java Thu May 30 07:53:18 2013
@@ -27,14 +27,14 @@ import org.apache.lucene.search.DocIdSet
import org.apache.lucene.search.Filter;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.FixedBitSet;
import java.io.IOException;
/**
* Base class for Lucene Filters on SpatialPrefixTree fields.
*
- * @lucene.internal
+ * @lucene.experimental
*/
public abstract class AbstractPrefixTreeFilter extends Filter {
@@ -93,13 +93,13 @@ public abstract class AbstractPrefixTree
this.termsEnum = terms.iterator(null);
}
- protected void collectDocs(OpenBitSet bitSet) throws IOException {
+ protected void collectDocs(FixedBitSet bitSet) throws IOException {
//WARN: keep this specialization in sync
assert termsEnum != null;
docsEnum = termsEnum.docs(acceptDocs, docsEnum, DocsEnum.FLAG_NONE);
int docid;
while ((docid = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- bitSet.fastSet(docid);
+ bitSet.set(docid);
}
}