You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/10/15 12:25:42 UTC
svn commit: r1708778 [2/8] - in /lucene/dev/branches/lucene6825: ./ lucene/
lucene/analysis/ lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/
lucene/analysis/common/src/test/org/apache/lucene/analysis/core/ l...
Modified: lucene/dev/branches/lucene6825/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikFilterFactory.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikFilterFactory.java (original)
+++ lucene/dev/branches/lucene6825/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikFilterFactory.java Thu Oct 15 10:25:39 2015
@@ -17,6 +17,8 @@ package org.apache.lucene.analysis.morfo
* limitations under the License.
*/
+import java.io.IOException;
+import java.io.InputStream;
import java.io.StringReader;
import java.util.Collections;
import java.util.HashMap;
@@ -31,51 +33,59 @@ import org.apache.lucene.analysis.util.R
* Test for {@link MorfologikFilterFactory}.
*/
public class TestMorfologikFilterFactory extends BaseTokenStreamTestCase {
- final ResourceLoader loader = new ClasspathResourceLoader(getClass());
+ private static class ForbidResourcesLoader implements ResourceLoader {
+ @Override
+ public InputStream openResource(String resource) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <T> Class<? extends T> findClass(String cname, Class<T> expectedType) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <T> T newInstance(String cname, Class<T> expectedType) {
+ throw new UnsupportedOperationException();
+ }
+ }
public void testDefaultDictionary() throws Exception {
StringReader reader = new StringReader("rowery bilety");
MorfologikFilterFactory factory = new MorfologikFilterFactory(Collections.<String,String>emptyMap());
- factory.inform(loader);
+ factory.inform(new ForbidResourcesLoader());
TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
assertTokenStreamContents(stream, new String[] {"rower", "bilet"});
}
-
- public void testResourceDictionary() throws Exception {
- StringReader reader = new StringReader("rowery bilety");
- Map<String,String> params = new HashMap<>();
- params.put(MorfologikFilterFactory.DICTIONARY_RESOURCE_ATTRIBUTE, MorfologikFilterFactory.DEFAULT_DICTIONARY_RESOURCE);
- MorfologikFilterFactory factory = new MorfologikFilterFactory(params);
- factory.inform(loader);
- TokenStream stream = whitespaceMockTokenizer(reader);
- stream = factory.create(stream);
- assertTokenStreamContents(stream, new String[] {"rower", "bilet"});
- }
-
- public void testResourceLoaderDictionary1() throws Exception {
- StringReader reader = new StringReader("rowery bilety");
+
+ public void testExplicitDictionary() throws Exception {
+ final ResourceLoader loader = new ClasspathResourceLoader(TestMorfologikFilterFactory.class);
+
+ StringReader reader = new StringReader("inflected1 inflected2");
Map<String,String> params = new HashMap<>();
- params.put(MorfologikFilterFactory.DICTIONARY_FSA_FILE_ATTRIBUTE, "/morfologik/dictionaries/pl.dict");
+ params.put(MorfologikFilterFactory.DICTIONARY_ATTRIBUTE, "custom-dictionary.dict");
MorfologikFilterFactory factory = new MorfologikFilterFactory(params);
factory.inform(loader);
TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
- assertTokenStreamContents(stream, new String[] {"rower", "bilet"});
+ assertTokenStreamContents(stream, new String[] {"lemma1", "lemma2"});
}
-
- public void testResourceLoaderDictionary2() throws Exception {
- StringReader reader = new StringReader("rowery bilety");
- Map<String,String> params = new HashMap<>();
- params.put(MorfologikFilterFactory.DICTIONARY_FSA_FILE_ATTRIBUTE, "/morfologik/dictionaries/pl.dict");
- params.put(MorfologikFilterFactory.DICTIONARY_FEATURES_FILE_ATTRIBUTE, "/morfologik/dictionaries/pl.info");
- MorfologikFilterFactory factory = new MorfologikFilterFactory(params);
- factory.inform(loader);
- TokenStream stream = whitespaceMockTokenizer(reader);
- stream = factory.create(stream);
- assertTokenStreamContents(stream, new String[] {"rower", "bilet"});
+
+ public void testMissingDictionary() throws Exception {
+ final ResourceLoader loader = new ClasspathResourceLoader(TestMorfologikFilterFactory.class);
+
+ try {
+ Map<String,String> params = new HashMap<>();
+ params.put(MorfologikFilterFactory.DICTIONARY_ATTRIBUTE, "missing-dictionary-resource.dict");
+ MorfologikFilterFactory factory = new MorfologikFilterFactory(params);
+ factory.inform(loader);
+ fail();
+ } catch (IOException e) {
+ assertTrue(e.getMessage().contains("Resource not found"));
+ }
}
-
+
/** Test that bogus arguments result in exception */
public void testBogusArguments() throws Exception {
try {
@@ -87,40 +97,4 @@ public class TestMorfologikFilterFactory
assertTrue(expected.getMessage().contains("Unknown parameters"));
}
}
-
- public void testIncompatibleArgs1() throws Exception {
- try {
- HashMap<String,String> params = new HashMap<String,String>();
- params.put(MorfologikFilterFactory.DICTIONARY_RESOURCE_ATTRIBUTE, MorfologikFilterFactory.DEFAULT_DICTIONARY_RESOURCE);
- params.put(MorfologikFilterFactory.DICTIONARY_FSA_FILE_ATTRIBUTE, "/morfologik/dictionaries/pl.dict");
- new MorfologikFilterFactory(params);
- fail();
- } catch (IllegalArgumentException expected) {
- assertTrue(expected.getMessage().contains("at the same time"));
- }
- }
-
- public void testIncompatibleArgs2() throws Exception {
- try {
- HashMap<String,String> params = new HashMap<String,String>();
- params.put(MorfologikFilterFactory.DICTIONARY_RESOURCE_ATTRIBUTE, MorfologikFilterFactory.DEFAULT_DICTIONARY_RESOURCE);
- params.put(MorfologikFilterFactory.DICTIONARY_FSA_FILE_ATTRIBUTE, "/morfologik/dictionaries/pl.dict");
- params.put(MorfologikFilterFactory.DICTIONARY_FEATURES_FILE_ATTRIBUTE, "/morfologik/dictionaries/pl.info");
- new MorfologikFilterFactory(params);
- fail();
- } catch (IllegalArgumentException expected) {
- assertTrue(expected.getMessage().contains("at the same time"));
- }
- }
-
- public void testMissingArgs1() throws Exception {
- try {
- HashMap<String,String> params = new HashMap<String,String>();
- params.put(MorfologikFilterFactory.DICTIONARY_FEATURES_FILE_ATTRIBUTE, "/morfologik/dictionaries/pl.info");
- new MorfologikFilterFactory(params);
- fail();
- } catch (IllegalArgumentException expected) {
- assertTrue(expected.getMessage().contains("Missing"));
- }
- }
}
Modified: lucene/dev/branches/lucene6825/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java (original)
+++ lucene/dev/branches/lucene6825/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java Thu Oct 15 10:25:39 2015
@@ -140,6 +140,9 @@ public class SimpleTextCompoundFormat ex
@Override
public IndexOutput createOutput(String name, IOContext context) { throw new UnsupportedOperationException(); }
+
+ @Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) { throw new UnsupportedOperationException(); }
@Override
public void sync(Collection<String> names) { throw new UnsupportedOperationException(); }
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java Thu Oct 15 10:25:39 2015
@@ -29,12 +29,12 @@ import org.apache.lucene.store.IndexOutp
import org.apache.lucene.store.Lock;
import org.apache.lucene.util.IOUtils;
+import java.io.FileNotFoundException;
+import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
-import java.io.FileNotFoundException;
-import java.io.IOException;
/**
* Class for accessing a compound stream.
@@ -172,6 +172,11 @@ final class Lucene50CompoundReader exten
public IndexOutput createOutput(String name, IOContext context) throws IOException {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
+ throw new UnsupportedOperationException();
+ }
@Override
public void sync(Collection<String> names) {
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Thu Oct 15 10:25:39 2015
@@ -32,8 +32,8 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
-import java.util.Map.Entry;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
@@ -57,10 +57,10 @@ import org.apache.lucene.store.IOContext
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.store.LockValidatingDirectoryWrapper;
import org.apache.lucene.store.MergeInfo;
import org.apache.lucene.store.RateLimitedIndexOutput;
import org.apache.lucene.store.TrackingDirectoryWrapper;
-import org.apache.lucene.store.LockValidatingDirectoryWrapper;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -2615,7 +2615,7 @@ public class IndexWriter implements Clos
SegmentCommitInfo infoPerCommit = new SegmentCommitInfo(info, 0, -1L, -1L, -1L);
info.setFiles(new HashSet<>(trackingDir.getCreatedFiles()));
- trackingDir.getCreatedFiles().clear();
+ trackingDir.clearCreatedFiles();
setDiagnostics(info, SOURCE_ADDINDEXES_READERS);
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java Thu Oct 15 10:25:39 2015
@@ -60,7 +60,13 @@ public class MappedMultiFields extends F
@Override
public TermsEnum iterator() throws IOException {
- return new MappedMultiTermsEnum(field, mergeState, (MultiTermsEnum) in.iterator());
+ TermsEnum iterator = in.iterator();
+ if (iterator == TermsEnum.EMPTY) {
+ // LUCENE-6826
+ return TermsEnum.EMPTY;
+ } else {
+ return new MappedMultiTermsEnum(field, mergeState, (MultiTermsEnum) iterator);
+ }
}
@Override
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/index/Terms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/index/Terms.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/index/Terms.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/index/Terms.java Thu Oct 15 10:25:39 2015
@@ -37,10 +37,7 @@ public abstract class Terms {
}
/** Returns an iterator that will step through all
- * terms. This method will not return null. If you have
- * a previous TermsEnum, for example from a different
- * field, you can pass it for possible reuse if the
- * implementation can do so. */
+ * terms. This method will not return null. */
public abstract TermsEnum iterator() throws IOException;
/** Returns a TermsEnum that iterates over all terms and
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java Thu Oct 15 10:25:39 2015
@@ -20,9 +20,13 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import org.apache.lucene.index.IndexReader;
@@ -109,12 +113,21 @@ public class BooleanQuery extends Query
return this;
}
+ /**
+ * Add a new clause to this {@link Builder}. Note that the order in which
+ * clauses are added does not have any impact on matching documents or query
+ * performance.
+ * @throws TooManyClauses if the new number of clauses exceeds the maximum clause number
+ */
public Builder add(BooleanClause clause) {
add(clause.getQuery(), clause.getOccur());
return this;
}
/**
+ * Add a new clause to this {@link Builder}. Note that the order in which
+ * clauses are added does not have any impact on matching documents or query
+ * performance.
* @throws TooManyClauses if the new number of clauses exceeds the maximum clause number
*/
public Builder add(Query query, Occur occur) {
@@ -135,13 +148,24 @@ public class BooleanQuery extends Query
private final boolean disableCoord;
private final int minimumNumberShouldMatch;
- private final List<BooleanClause> clauses;
+ private final List<BooleanClause> clauses; // used for toString() and getClauses()
+ private final Map<Occur, Collection<Query>> clauseSets; // used for equals/hashcode
private BooleanQuery(boolean disableCoord, int minimumNumberShouldMatch,
BooleanClause[] clauses) {
this.disableCoord = disableCoord;
this.minimumNumberShouldMatch = minimumNumberShouldMatch;
this.clauses = Collections.unmodifiableList(Arrays.asList(clauses));
+ clauseSets = new EnumMap<>(Occur.class);
+ // duplicates matter for SHOULD and MUST
+ clauseSets.put(Occur.SHOULD, new Multiset<>());
+ clauseSets.put(Occur.MUST, new Multiset<>());
+ // but not for FILTER and MUST_NOT
+ clauseSets.put(Occur.FILTER, new HashSet<>());
+ clauseSets.put(Occur.MUST_NOT, new HashSet<>());
+ for (BooleanClause clause : clauses) {
+ clauseSets.get(clause.getOccur()).add(clause.getQuery());
+ }
}
/**
@@ -271,6 +295,19 @@ public class BooleanQuery extends Query
return buffer.toString();
}
+ /**
+ * Compares the specified object with this boolean query for equality.
+ * Returns true if and only if the provided object<ul>
+ * <li>is also a {@link BooleanQuery},</li>
+ * <li>has the same value of {@link #isCoordDisabled()}</li>
+ * <li>has the same value of {@link #getMinimumNumberShouldMatch()}</li>
+ * <li>has the same {@link Occur#SHOULD} clauses, regardless of the order</li>
+ * <li>has the same {@link Occur#MUST} clauses, regardless of the order</li>
+ * <li>has the same set of {@link Occur#FILTER} clauses, regardless of the
+ * order and regardless of duplicates</li>
+ * <li>has the same set of {@link Occur#MUST_NOT} clauses, regardless of
+ * the order and regardless of duplicates</li></ul>
+ */
@Override
public boolean equals(Object o) {
if (super.equals(o) == false) {
@@ -279,12 +316,29 @@ public class BooleanQuery extends Query
BooleanQuery that = (BooleanQuery)o;
return this.getMinimumNumberShouldMatch() == that.getMinimumNumberShouldMatch()
&& this.disableCoord == that.disableCoord
- && clauses.equals(that.clauses);
+ && clauseSets.equals(that.clauseSets);
+ }
+
+ private int computeHashCode() {
+ int hashCode = 31 * super.hashCode() + Objects.hash(disableCoord, minimumNumberShouldMatch, clauseSets);
+ if (hashCode == 0) {
+ hashCode = 1;
+ }
+ return hashCode;
}
+ // cached hash code is ok since boolean queries are immutable
+ private int hashCode;
+
@Override
public int hashCode() {
- return 31 * super.hashCode() + Objects.hash(disableCoord, minimumNumberShouldMatch, clauses);
+ if (hashCode == 0) {
+ // no need for synchronization, in the worst case we would just compute the hash several times
+ hashCode = computeHashCode();
+ assert hashCode != 0;
+ }
+ assert hashCode == computeHashCode();
+ return hashCode;
}
}
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/BoostQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/BoostQuery.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/BoostQuery.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/BoostQuery.java Thu Oct 15 10:25:39 2015
@@ -18,9 +18,6 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
import java.util.Objects;
import java.util.Set;
@@ -36,23 +33,6 @@ import org.apache.lucene.index.Term;
*/
public final class BoostQuery extends Query {
- /** By default we enclose the wrapped query within parenthesis, but this is
- * not required for all queries, so we use a whitelist of queries that don't
- * need parenthesis to have a better toString(). */
- private static final Set<Class<? extends Query>> NO_PARENS_REQUIRED_QUERIES = Collections.unmodifiableSet(
- new HashSet<>(Arrays.asList(
- TermQuery.class,
- PhraseQuery.class,
- MultiPhraseQuery.class,
- ConstantScoreQuery.class,
- TermRangeQuery.class,
- NumericRangeQuery.class,
- PrefixQuery.class,
- FuzzyQuery.class,
- WildcardQuery.class,
- RegexpQuery.class
- )));
-
private final Query query;
private final float boost;
@@ -122,15 +102,10 @@ public final class BoostQuery extends Qu
@Override
public String toString(String field) {
- boolean needsParens = NO_PARENS_REQUIRED_QUERIES.contains(query.getClass()) == false;
StringBuilder builder = new StringBuilder();
- if (needsParens) {
- builder.append("(");
- }
+ builder.append("(");
builder.append(query.toString(field));
- if (needsParens) {
- builder.append(")");
- }
+ builder.append(")");
builder.append("^");
builder.append(boost);
return builder.toString();
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSetIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSetIterator.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSetIterator.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSetIterator.java Thu Oct 15 10:25:39 2015
@@ -22,8 +22,7 @@ import java.io.IOException;
/**
* Abstract decorator class of a DocIdSetIterator
* implementation that provides on-demand filter/validation
- * mechanism on an underlying DocIdSetIterator. See {@link
- * FilteredDocIdSet}.
+ * mechanism on an underlying DocIdSetIterator.
*/
public abstract class FilteredDocIdSetIterator extends DocIdSetIterator {
protected DocIdSetIterator _innerIter;
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/Query.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/Query.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/Query.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/Query.java Thu Oct 15 10:25:39 2015
@@ -81,12 +81,8 @@ public abstract class Query {
@Override
public boolean equals(Object obj) {
- if (this == obj)
- return true;
if (obj == null)
return false;
- if (getClass() != obj.getClass())
- return false;
- return true;
+ return getClass() == obj.getClass();
}
}
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/package-info.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/package-info.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/package-info.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/package-info.java Thu Oct 15 10:25:39 2015
@@ -500,8 +500,6 @@
* Weight object is an internal representation of the Query that allows the Query
* to be reused by the IndexSearcher.</li>
* <li>The IndexSearcher that initiated the call.</li>
- * <li>A {@link org.apache.lucene.search.Filter Filter} for limiting the result set.
- * Note, the Filter may be null.</li>
* <li>A {@link org.apache.lucene.search.Sort Sort} object for specifying how to sort
* the results if the standard score-based sort method is not desired.</li>
* </ol>
@@ -509,8 +507,7 @@
* we call one of the search methods of the IndexSearcher, passing in the
* {@link org.apache.lucene.search.Weight Weight} object created by
* {@link org.apache.lucene.search.IndexSearcher#createNormalizedWeight(org.apache.lucene.search.Query,boolean)
- * IndexSearcher.createNormalizedWeight(Query,boolean)},
- * {@link org.apache.lucene.search.Filter Filter} and the number of results we want.
+ * IndexSearcher.createNormalizedWeight(Query,boolean)} and the number of results we want.
* This method returns a {@link org.apache.lucene.search.TopDocs TopDocs} object,
* which is an internal collection of search results. The IndexSearcher creates
* a {@link org.apache.lucene.search.TopScoreDocCollector TopScoreDocCollector} and
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/spans/SpanBoostQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/spans/SpanBoostQuery.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/spans/SpanBoostQuery.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/search/spans/SpanBoostQuery.java Thu Oct 15 10:25:39 2015
@@ -18,9 +18,6 @@ package org.apache.lucene.search.spans;
*/
import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@@ -41,21 +38,6 @@ import org.apache.lucene.search.Scorer;
*/
public final class SpanBoostQuery extends SpanQuery {
- /** By default we enclose the wrapped query within parenthesis, but this is
- * not required for all queries, so we use a whitelist of queries that don't
- * need parenthesis to have a better toString(). */
- private static final Set<Class<? extends SpanQuery>> NO_PARENS_REQUIRED_QUERIES = Collections.unmodifiableSet(
- new HashSet<>(Arrays.asList(
- SpanTermQuery.class,
- SpanNearQuery.class,
- SpanOrQuery.class,
- SpanFirstQuery.class,
- SpanContainingQuery.class,
- SpanContainQuery.class,
- SpanNotQuery.class,
- SpanWithinQuery.class
- )));
-
private final SpanQuery query;
private final float boost;
@@ -119,16 +101,10 @@ public final class SpanBoostQuery extend
@Override
public String toString(String field) {
- boolean needsParens = NO_PARENS_REQUIRED_QUERIES.contains(query.getClass()) == false;
StringBuilder builder = new StringBuilder();
- if (needsParens) {
- builder.append("(");
- }
+ builder.append("(");
builder.append(query.toString(field));
- if (needsParens) {
- builder.append(")");
- }
- builder.append("^");
+ builder.append(")^");
builder.append(boost);
return builder.toString();
}
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java Thu Oct 15 10:25:39 2015
@@ -18,6 +18,7 @@ package org.apache.lucene.store;
*/
import java.io.IOException;
+import java.util.Random;
/**
* Base implementation for a concrete {@link Directory} that uses a {@link LockFactory} for locking.
@@ -31,6 +32,22 @@ public abstract class BaseDirectory exte
* this Directory instance). */
protected final LockFactory lockFactory;
+ /** Subclasses can use this to generate temp file name candidates */
+ protected static final Random tempFileRandom;
+
+ static {
+ String prop = System.getProperty("tests.seed");
+ int seed;
+ if (prop != null) {
+ // So if there is a test failure that relied on temp file names,
+ //we remain reproducible based on the test seed:
+ seed = prop.hashCode();
+ } else {
+ seed = (int) System.currentTimeMillis();
+ }
+ tempFileRandom = new Random(seed);
+ }
+
/** Sole constructor. */
protected BaseDirectory(LockFactory lockFactory) {
super();
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/Directory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/Directory.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/Directory.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/Directory.java Thu Oct 15 10:25:39 2015
@@ -17,9 +17,9 @@ package org.apache.lucene.store;
* limitations under the License.
*/
+import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.Closeable;
import java.nio.file.NoSuchFileException;
import java.util.Collection; // for javadocs
@@ -70,8 +70,12 @@ public abstract class Directory implemen
/** Creates a new, empty file in the directory with the given name.
Returns a stream writing this file. */
- public abstract IndexOutput createOutput(String name, IOContext context)
- throws IOException;
+ public abstract IndexOutput createOutput(String name, IOContext context) throws IOException;
+
+ /** Creates a new, empty file for writing in the directory, with a
+ * temporary file name derived from prefix and suffix. Use
+ * {@link IndexOutput#getName} to see what name was used. */
+ public abstract IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException;
/**
* Ensure that any writes to these files are moved to
@@ -120,8 +124,7 @@ public abstract class Directory implemen
/** Closes the store. */
@Override
- public abstract void close()
- throws IOException;
+ public abstract void close() throws IOException;
@Override
public String toString() {
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java Thu Oct 15 10:25:39 2015
@@ -19,11 +19,14 @@ package org.apache.lucene.store;
import java.io.FilterOutputStream;
import java.io.IOException;
+import java.nio.channels.ClosedChannelException; // javadoc @link
import java.nio.file.DirectoryStream;
+import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
+import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
-import java.nio.channels.ClosedChannelException; // javadoc @link
+import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -220,11 +223,23 @@ public abstract class FSDirectory extend
@Override
public IndexOutput createOutput(String name, IOContext context) throws IOException {
ensureOpen();
-
ensureCanWrite(name);
return new FSIndexOutput(name);
}
+ @Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
+ ensureOpen();
+ while (true) {
+ String name = prefix + tempFileRandom.nextInt(Integer.MAX_VALUE) + "." + suffix;
+ try {
+ return new FSIndexOutput(name, StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
+ } catch (FileAlreadyExistsException faee) {
+ // Retry with next random name
+ }
+ }
+ }
+
protected void ensureCanWrite(String name) throws IOException {
Files.deleteIfExists(directory.resolve(name)); // delete existing, if any
}
@@ -273,7 +288,11 @@ public abstract class FSDirectory extend
static final int CHUNK_SIZE = 8192;
public FSIndexOutput(String name) throws IOException {
- super("FSIndexOutput(path=\"" + directory.resolve(name) + "\")", new FilterOutputStream(Files.newOutputStream(directory.resolve(name))) {
+ this(name, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
+ }
+
+ FSIndexOutput(String name, OpenOption... options) throws IOException {
+ super("FSIndexOutput(path=\"" + directory.resolve(name) + "\")", name, new FilterOutputStream(Files.newOutputStream(directory.resolve(name), options)) {
// This implementation ensures, that we never write more than CHUNK_SIZE bytes:
@Override
public void write(byte[] b, int offset, int length) throws IOException {
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java Thu Oct 15 10:25:39 2015
@@ -22,13 +22,12 @@ import java.nio.file.AtomicMoveNotSuppor
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import java.util.HashSet;
import org.apache.lucene.util.IOUtils;
-
/**
* Expert: A Directory instance that switches files between
* two other Directory instances.
@@ -156,6 +155,11 @@ public class FileSwitchDirectory extends
}
@Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
+ return getDirectory("."+suffix).createTempOutput(prefix, suffix, context);
+ }
+
+ @Override
public void sync(Collection<String> names) throws IOException {
List<String> primaryNames = new ArrayList<>();
List<String> secondaryNames = new ArrayList<>();
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java Thu Oct 15 10:25:39 2015
@@ -74,6 +74,11 @@ public class FilterDirectory extends Dir
}
@Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
+ return in.createTempOutput(prefix, suffix, context);
+ }
+
+ @Override
public void sync(Collection<String> names) throws IOException {
in.sync(names);
}
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java Thu Oct 15 10:25:39 2015
@@ -31,15 +31,27 @@ import java.io.IOException;
*/
public abstract class IndexOutput extends DataOutput implements Closeable {
+ /** Full description of this output, e.g. which class such as {@code FSIndexOutput}, and the full path to the file */
private final String resourceDescription;
+ /** Just the name part from {@code resourceDescription} */
+ private final String name;
+
/** Sole constructor. resourceDescription should be non-null, opaque string
* describing this resource; it's returned from {@link #toString}. */
- protected IndexOutput(String resourceDescription) {
+ protected IndexOutput(String resourceDescription, String name) {
if (resourceDescription == null) {
throw new IllegalArgumentException("resourceDescription must not be null");
}
this.resourceDescription = resourceDescription;
+ this.name = name;
+ }
+
+ /** Returns the name used to create this {@code IndexOutput}. This is especially useful when using
+ * {@link Directory#createTempOutput}. */
+ // TODO: can we somehow use this as the default resource description or something?
+ public String getName() {
+ return name;
}
/** Closes this stream to further operations. */
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/OutputStreamIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/OutputStreamIndexOutput.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/OutputStreamIndexOutput.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/OutputStreamIndexOutput.java Thu Oct 15 10:25:39 2015
@@ -37,8 +37,8 @@ public class OutputStreamIndexOutput ext
* @param bufferSize the buffer size in bytes used to buffer writes internally.
* @throws IllegalArgumentException if the given buffer size is less or equal to <tt>0</tt>
*/
- public OutputStreamIndexOutput(String resourceDescription, OutputStream out, int bufferSize) {
- super(resourceDescription);
+ public OutputStreamIndexOutput(String resourceDescription, String name, OutputStream out, int bufferSize) {
+ super(resourceDescription, name);
this.os = new BufferedOutputStream(new CheckedOutputStream(out, crc), bufferSize);
}
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java Thu Oct 15 10:25:39 2015
@@ -17,8 +17,8 @@ package org.apache.lucene.store;
* limitations under the License.
*/
-import java.io.IOException;
import java.io.FileNotFoundException;
+import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collection;
@@ -26,12 +26,12 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
-
/**
* A memory-resident {@link Directory} implementation. Locking
* implementation is by default the {@link SingleInstanceLockFactory}.
@@ -111,10 +111,7 @@ public class RAMDirectory extends BaseDi
// and do not synchronize or anything stronger. it's great for testing!
// NOTE: fileMap.keySet().toArray(new String[0]) is broken in non Sun JDKs,
// and the code below is resilient to map changes during the array population.
- Set<String> fileNames = fileMap.keySet();
- List<String> names = new ArrayList<>(fileNames.size());
- for (String name : fileNames) names.add(name);
- return names.toArray(new String[names.size()]);
+ return fileMap.keySet().toArray(new String[fileMap.size()]);
}
public final boolean fileNameExists(String name) {
@@ -150,9 +147,6 @@ public class RAMDirectory extends BaseDi
return Accountables.namedAccountables("file", fileMap);
}
- /** Removes an existing file in the directory.
- * @throws IOException if the file does not exist
- */
@Override
public void deleteFile(String name) throws IOException {
ensureOpen();
@@ -165,7 +159,6 @@ public class RAMDirectory extends BaseDi
}
}
- /** Creates a new, empty file in the directory with the given name. Returns a stream writing this file. */
@Override
public IndexOutput createOutput(String name, IOContext context) throws IOException {
ensureOpen();
@@ -179,6 +172,22 @@ public class RAMDirectory extends BaseDi
return new RAMOutputStream(name, file, true);
}
+ @Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
+ ensureOpen();
+
+ // Make the file first...
+ RAMFile file = newRAMFile();
+
+ // ... then try to find a unique name for it:
+ while (true) {
+ String name = prefix + tempFileRandom.nextInt(Integer.MAX_VALUE) + "." + suffix;
+ if (fileMap.putIfAbsent(name, file) == null) {
+ return new RAMOutputStream(name, file, true);
+ }
+ }
+ }
+
/**
* Returns a new {@link RAMFile} for storing data. This method can be
* overridden to return different {@link RAMFile} impls, that e.g. override
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java Thu Oct 15 10:25:39 2015
@@ -57,7 +57,7 @@ public class RAMOutputStream extends Ind
/** Creates this, with specified name. */
public RAMOutputStream(String name, RAMFile f, boolean checksum) {
- super("RAMOutputStream(name=\"" + name + "\")");
+ super("RAMOutputStream(name=\"" + name + "\")", name);
file = f;
// make sure that we switch to the
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java Thu Oct 15 10:25:39 2015
@@ -38,7 +38,7 @@ public final class RateLimitedIndexOutpu
private long currentMinPauseCheckBytes;
public RateLimitedIndexOutput(final RateLimiter rateLimiter, final IndexOutput delegate) {
- super("RateLimitedIndexOutput(" + delegate + ")");
+ super("RateLimitedIndexOutput(" + delegate + ")", delegate.getName());
this.delegate = delegate;
this.rateLimiter = rateLimiter;
this.currentMinPauseCheckBytes = rateLimiter.getMinPauseCheckBytes();
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java Thu Oct 15 10:25:39 2015
@@ -46,6 +46,14 @@ public final class TrackingDirectoryWrap
}
@Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context)
+ throws IOException {
+ IndexOutput tempOutput = in.createTempOutput(prefix, suffix, context);
+ createdFileNames.add(tempOutput.getName());
+ return tempOutput;
+ }
+
+ @Override
public void copyFrom(Directory from, String src, String dest, IOContext context) throws IOException {
in.copyFrom(from, src, dest, context);
createdFileNames.add(dest);
@@ -60,10 +68,12 @@ public final class TrackingDirectoryWrap
}
}
- // maybe clone before returning.... all callers are
- // cloning anyway....
+ /** NOTE: returns a copy of the created files. */
public Set<String> getCreatedFiles() {
- return createdFileNames;
+ return new HashSet<>(createdFileNames);
}
+ public void clearCreatedFiles() {
+ createdFileNames.clear();
+ }
}
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/util/IOUtils.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/util/IOUtils.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/util/IOUtils.java Thu Oct 15 10:25:39 2015
@@ -190,7 +190,7 @@ public final class IOUtils {
* <p>
* Note that the files should not be null.
*/
- public static void deleteFilesIgnoringExceptions(Directory dir, String... files) {
+ public static void deleteFilesIgnoringExceptions(Directory dir, Collection<String> files) {
for (String name : files) {
try {
dir.deleteFile(name);
@@ -199,6 +199,42 @@ public final class IOUtils {
}
}
}
+
+ public static void deleteFilesIgnoringExceptions(Directory dir, String... files) {
+ deleteFilesIgnoringExceptions(dir, Arrays.asList(files));
+ }
+
+ /**
+ * Deletes all given file names. Some of the
+ * file names may be null; they are
+ * ignored. After everything is deleted, the method either
+ * throws the first exception it hit while deleting, or
+ * completes normally if there were no exceptions.
+ *
+ * @param dir Directory to delete files from
+ * @param files file names to delete
+ */
+ public static void deleteFiles(Directory dir, Collection<String> files) throws IOException {
+ Throwable th = null;
+ for (String name : files) {
+ if (name != null) {
+ try {
+ dir.deleteFile(name);
+ } catch (Throwable t) {
+ addSuppressed(th, t);
+ if (th == null) {
+ th = t;
+ }
+ }
+ }
+ }
+
+ reThrow(th);
+ }
+
+ public static void deleteFiles(Directory dir, String... files) throws IOException {
+ deleteFiles(dir, Arrays.asList(files));
+ }
/**
* Deletes all given files, suppressing all thrown IOExceptions.
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java Thu Oct 15 10:25:39 2015
@@ -17,24 +17,20 @@ package org.apache.lucene.util;
* limitations under the License.
*/
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
import java.io.Closeable;
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.TrackingDirectoryWrapper;
+
/**
* On-disk sorting of byte arrays. Each byte array (entry) is a composed of the following
* fields:
@@ -43,16 +39,12 @@ import java.util.Locale;
* <li>exactly the above count of bytes for the sequence to be sorted.
* </ul>
*
- * @see #sort(Path, Path)
+ * @see #sort(String)
* @lucene.experimental
* @lucene.internal
*/
public final class OfflineSorter {
- // nocommit cutover to Directory API
-
- private static Path DEFAULT_TEMP_DIR;
-
/** Convenience constant for megabytes */
public final static long MB = 1024 * 1024;
/** Convenience constant for gigabytes */
@@ -74,6 +66,10 @@ public final class OfflineSorter {
*/
public final static int MAX_TEMPFILES = 128;
+ private final Directory dir;
+
+ private final String tempFileNamePrefix;
+
/**
* A bit more descriptive unit for constructors.
*
@@ -144,7 +140,7 @@ public final class OfflineSorter {
/** number of partition merges */
public int mergeRounds;
/** number of lines of data read */
- public int lines;
+ public int lineCount;
/** time spent merging sorted partitions (in milliseconds) */
public long mergeTime;
/** time spent sorting data (in milliseconds) */
@@ -164,17 +160,16 @@ public final class OfflineSorter {
return String.format(Locale.ROOT,
"time=%.2f sec. total (%.2f reading, %.2f sorting, %.2f merging), lines=%d, temp files=%d, merges=%d, soft ram limit=%.2f MB",
totalTime / 1000.0d, readTime / 1000.0d, sortTime / 1000.0d, mergeTime / 1000.0d,
- lines, tempMergeFiles, mergeRounds,
+ lineCount, tempMergeFiles, mergeRounds,
(double) bufferSize / MB);
}
}
private final BufferSize ramBufferSize;
- private final Path tempDirectory;
private final Counter bufferBytesUsed = Counter.newCounter();
private final BytesRefArray buffer = new BytesRefArray(bufferBytesUsed);
- private SortInfo sortInfo;
+ SortInfo sortInfo;
private int maxTempFiles;
private final Comparator<BytesRef> comparator;
@@ -184,27 +179,25 @@ public final class OfflineSorter {
/**
* Defaults constructor.
*
- * @see #getDefaultTempDir()
* @see BufferSize#automatic()
*/
- public OfflineSorter() throws IOException {
- this(DEFAULT_COMPARATOR, BufferSize.automatic(), getDefaultTempDir(), MAX_TEMPFILES);
+ public OfflineSorter(Directory dir, String tempFileNamePrefix) throws IOException {
+ this(dir, tempFileNamePrefix, DEFAULT_COMPARATOR, BufferSize.automatic(), MAX_TEMPFILES);
}
/**
* Defaults constructor with a custom comparator.
*
- * @see #getDefaultTempDir()
* @see BufferSize#automatic()
*/
- public OfflineSorter(Comparator<BytesRef> comparator) throws IOException {
- this(comparator, BufferSize.automatic(), getDefaultTempDir(), MAX_TEMPFILES);
+ public OfflineSorter(Directory dir, String tempFileNamePrefix, Comparator<BytesRef> comparator) throws IOException {
+ this(dir, tempFileNamePrefix, comparator, BufferSize.automatic(), MAX_TEMPFILES);
}
/**
* All-details constructor.
*/
- public OfflineSorter(Comparator<BytesRef> comparator, BufferSize ramBufferSize, Path tempDirectory, int maxTempfiles) {
+ public OfflineSorter(Directory dir, String tempFileNamePrefix, Comparator<BytesRef> comparator, BufferSize ramBufferSize, int maxTempfiles) {
if (ramBufferSize.bytes < ABSOLUTE_MIN_SORT_BUFFER_SIZE) {
throw new IllegalArgumentException(MIN_BUFFER_SIZE_MSG + ": " + ramBufferSize.bytes);
}
@@ -214,160 +207,129 @@ public final class OfflineSorter {
}
this.ramBufferSize = ramBufferSize;
- this.tempDirectory = tempDirectory;
this.maxTempFiles = maxTempfiles;
this.comparator = comparator;
+ this.dir = dir;
+ this.tempFileNamePrefix = tempFileNamePrefix;
+ }
+
+ /** Returns the {@link Directory} we use to create temp files. */
+ public Directory getDirectory() {
+ return dir;
+ }
+
+ /** Returns the temp file name prefix passed to {@link Directory#createTempOutput} to generate temporary files. */
+ public String getTempFileNamePrefix() {
+ return tempFileNamePrefix;
}
/**
- * Sort input to output, explicit hint for the buffer size. The amount of allocated
- * memory may deviate from the hint (may be smaller or larger).
+ * Sort input to a new temp file, returning its name.
*/
- public SortInfo sort(Path input, Path output) throws IOException {
+ public String sort(String inputFileName) throws IOException {
+
sortInfo = new SortInfo();
sortInfo.totalTime = System.currentTimeMillis();
- // NOTE: don't remove output here: its existence (often created by the caller
- // up above using Files.createTempFile) prevents another concurrent caller
- // of this API (from a different thread) from incorrectly re-using this file name
-
- ArrayList<Path> merges = new ArrayList<>();
- boolean success3 = false;
- try {
- ByteSequencesReader is = new ByteSequencesReader(input);
- boolean success = false;
- try {
- int lines = 0;
- while ((lines = readPartition(is)) > 0) {
- merges.add(sortPartition(lines));
- sortInfo.tempMergeFiles++;
- sortInfo.lines += lines;
-
- // Handle intermediate merges.
- if (merges.size() == maxTempFiles) {
- Path intermediate = Files.createTempFile(tempDirectory, "sort", "intermediate");
- boolean success2 = false;
- try {
- mergePartitions(merges, intermediate);
- success2 = true;
- } finally {
- if (success2) {
- IOUtils.deleteFilesIfExist(merges);
- } else {
- IOUtils.deleteFilesIgnoringExceptions(merges);
- }
- merges.clear();
- merges.add(intermediate);
- }
- sortInfo.tempMergeFiles++;
- }
- }
- success = true;
- } finally {
- if (success) {
- IOUtils.close(is);
- } else {
- IOUtils.closeWhileHandlingException(is);
+ List<String> segments = new ArrayList<>();
+
+ // So we can remove any partially written temp files on exception:
+ TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(dir);
+
+ boolean success = false;
+ try (ByteSequencesReader is = new ByteSequencesReader(dir.openInput(inputFileName, IOContext.READONCE))) {
+
+ int lineCount;
+ while ((lineCount = readPartition(is)) > 0) {
+ segments.add(sortPartition(trackingDir));
+ sortInfo.tempMergeFiles++;
+ sortInfo.lineCount += lineCount;
+
+ // Handle intermediate merges.
+ if (segments.size() == maxTempFiles) {
+ mergePartitions(trackingDir, segments);
}
}
- // One partition, try to rename or copy if unsuccessful.
- if (merges.size() == 1) {
- Files.move(merges.get(0), output, StandardCopyOption.REPLACE_EXISTING);
- } else {
- // otherwise merge the partitions with a priority queue.
- mergePartitions(merges, output);
+ // Merge the partitions to the output file with a priority queue.
+ if (segments.size() > 1) {
+ mergePartitions(trackingDir, segments);
}
- success3 = true;
- } finally {
- if (success3) {
- IOUtils.deleteFilesIfExist(merges);
+
+ String result;
+ if (segments.isEmpty()) {
+ try (IndexOutput out = trackingDir.createTempOutput(tempFileNamePrefix, "sort", IOContext.DEFAULT)) {
+ result = out.getName();
+ }
} else {
- IOUtils.deleteFilesIgnoringExceptions(merges);
- IOUtils.deleteFilesIgnoringExceptions(output);
+ result = segments.get(0);
}
- }
- sortInfo.totalTime = (System.currentTimeMillis() - sortInfo.totalTime);
- return sortInfo;
- }
+ // We should be explicitly removing all intermediate files ourselves unless there is an exception:
+ assert trackingDir.getCreatedFiles().size() == 1 && trackingDir.getCreatedFiles().contains(result);
- /** Used by test framework */
- static void setDefaultTempDir(Path tempDir) {
- DEFAULT_TEMP_DIR = tempDir;
- }
+ sortInfo.totalTime = (System.currentTimeMillis() - sortInfo.totalTime);
+ success = true;
- /**
- * Returns the default temporary directory. By default, java.io.tmpdir. If not accessible
- * or not available, an IOException is thrown
- */
- public synchronized static Path getDefaultTempDir() throws IOException {
- if (DEFAULT_TEMP_DIR == null) {
- // Lazy init
- String tempDirPath = System.getProperty("java.io.tmpdir");
- if (tempDirPath == null) {
- throw new IOException("Java has no temporary folder property (java.io.tmpdir)?");
- }
- Path tempDirectory = Paths.get(tempDirPath);
- if (Files.isWritable(tempDirectory) == false) {
- throw new IOException("Java's temporary folder not present or writeable?: "
- + tempDirectory.toAbsolutePath());
+ return result;
+
+ } finally {
+ if (success == false) {
+ IOUtils.deleteFilesIgnoringExceptions(trackingDir, trackingDir.getCreatedFiles());
}
- DEFAULT_TEMP_DIR = tempDirectory;
}
-
- return DEFAULT_TEMP_DIR;
}
/** Sort a single partition in-memory. */
- protected Path sortPartition(int len) throws IOException {
+ protected String sortPartition(TrackingDirectoryWrapper trackingDir) throws IOException {
BytesRefArray data = this.buffer;
- Path tempFile = Files.createTempFile(tempDirectory, "sort", "partition");
- long start = System.currentTimeMillis();
- sortInfo.sortTime += (System.currentTimeMillis() - start);
-
- final ByteSequencesWriter out = new ByteSequencesWriter(tempFile);
- BytesRef spare;
- try {
+ try (IndexOutput tempFile = trackingDir.createTempOutput(tempFileNamePrefix, "sort", IOContext.DEFAULT)) {
+ ByteSequencesWriter out = new ByteSequencesWriter(tempFile);
+ BytesRef spare;
+
+ long start = System.currentTimeMillis();
BytesRefIterator iter = buffer.iterator(comparator);
- while((spare = iter.next()) != null) {
+ sortInfo.sortTime += (System.currentTimeMillis() - start);
+
+ while ((spare = iter.next()) != null) {
assert spare.length <= Short.MAX_VALUE;
out.write(spare);
}
- out.close();
-
// Clean up the buffer for the next partition.
data.clear();
- return tempFile;
- } finally {
- IOUtils.close(out);
+
+ return tempFile.getName();
}
}
- /** Merge a list of sorted temporary files (partitions) into an output file */
- void mergePartitions(List<Path> merges, Path outputFile) throws IOException {
+ /** Merge a list of sorted temporary files (partitions) into an output file. Note that this closes the
+ * incoming {@link IndexOutput}. */
+ void mergePartitions(Directory trackingDir, List<String> segments) throws IOException {
long start = System.currentTimeMillis();
- ByteSequencesWriter out = new ByteSequencesWriter(outputFile);
-
- PriorityQueue<FileAndTop> queue = new PriorityQueue<FileAndTop>(merges.size()) {
+ PriorityQueue<FileAndTop> queue = new PriorityQueue<FileAndTop>(segments.size()) {
@Override
protected boolean lessThan(FileAndTop a, FileAndTop b) {
return comparator.compare(a.current.get(), b.current.get()) < 0;
}
};
- ByteSequencesReader [] streams = new ByteSequencesReader [merges.size()];
- try {
+ ByteSequencesReader[] streams = new ByteSequencesReader[segments.size()];
+
+ String newSegmentName = null;
+
+ try (IndexOutput out = trackingDir.createTempOutput(tempFileNamePrefix, "sort", IOContext.DEFAULT)) {
+ newSegmentName = out.getName();
+ ByteSequencesWriter writer = new ByteSequencesWriter(out);
+
// Open streams and read the top for each file
- for (int i = 0; i < merges.size(); i++) {
- streams[i] = new ByteSequencesReader(merges.get(i));
- byte line[] = streams[i].read();
- if (line != null) {
- queue.insertWithOverflow(new FileAndTop(i, line));
- }
+ for (int i = 0; i < segments.size(); i++) {
+ streams[i] = new ByteSequencesReader(dir.openInput(segments.get(i), IOContext.READONCE));
+ byte[] line = streams[i].read();
+ assert line != null;
+ queue.insertWithOverflow(new FileAndTop(i, line));
}
// Unix utility sort() uses ordered array of files to pick the next line from, updating
@@ -376,7 +338,7 @@ public final class OfflineSorter {
// so it shouldn't make much of a difference (didn't check).
FileAndTop top;
while ((top = queue.top()) != null) {
- out.write(top.current.bytes(), 0, top.current.length());
+ writer.write(top.current.bytes(), 0, top.current.length());
if (!streams[top.fd].read(top.current)) {
queue.pop();
} else {
@@ -387,14 +349,15 @@ public final class OfflineSorter {
sortInfo.mergeTime += System.currentTimeMillis() - start;
sortInfo.mergeRounds++;
} finally {
- // The logic below is: if an exception occurs in closing out, it has a priority over exceptions
- // happening in closing streams.
- try {
- IOUtils.close(streams);
- } finally {
- IOUtils.close(out);
- }
+ IOUtils.close(streams);
}
+
+ IOUtils.deleteFiles(trackingDir, segments);
+
+ segments.clear();
+ segments.add(newSegmentName);
+
+ sortInfo.tempMergeFiles++;
}
/** Read in a single partition of data */
@@ -430,18 +393,11 @@ public final class OfflineSorter {
* Complementary to {@link ByteSequencesReader}.
*/
public static class ByteSequencesWriter implements Closeable {
- private final DataOutput os;
-
- /** Constructs a ByteSequencesWriter to the provided Path */
- public ByteSequencesWriter(Path path) throws IOException {
- this(new DataOutputStream(
- new BufferedOutputStream(
- Files.newOutputStream(path))));
- }
+ private final IndexOutput out;
/** Constructs a ByteSequencesWriter to the provided DataOutput */
- public ByteSequencesWriter(DataOutput os) {
- this.os = os;
+ public ByteSequencesWriter(IndexOutput out) {
+ this.out = out;
}
/**
@@ -457,7 +413,7 @@ public final class OfflineSorter {
* Writes a byte array.
* @see #write(byte[], int, int)
*/
- public void write(byte [] bytes) throws IOException {
+ public void write(byte[] bytes) throws IOException {
write(bytes, 0, bytes.length);
}
@@ -467,25 +423,23 @@ public final class OfflineSorter {
* The length is written as a <code>short</code>, followed
* by the bytes.
*/
- public void write(byte [] bytes, int off, int len) throws IOException {
+ public void write(byte[] bytes, int off, int len) throws IOException {
assert bytes != null;
assert off >= 0 && off + len <= bytes.length;
assert len >= 0;
if (len > Short.MAX_VALUE) {
throw new IllegalArgumentException("len must be <= " + Short.MAX_VALUE + "; got " + len);
}
- os.writeShort(len);
- os.write(bytes, off, len);
+ out.writeShort((short) len);
+ out.writeBytes(bytes, off, len);
}
/**
- * Closes the provided {@link DataOutput} if it is {@link Closeable}.
+ * Closes the provided {@link IndexOutput}.
*/
@Override
public void close() throws IOException {
- if (os instanceof Closeable) {
- ((Closeable) os).close();
- }
+ out.close();
}
}
@@ -494,18 +448,11 @@ public final class OfflineSorter {
* Complementary to {@link ByteSequencesWriter}.
*/
public static class ByteSequencesReader implements Closeable {
- private final DataInput is;
+ private final IndexInput in;
- /** Constructs a ByteSequencesReader from the provided Path */
- public ByteSequencesReader(Path path) throws IOException {
- this(new DataInputStream(
- new BufferedInputStream(
- Files.newInputStream(path))));
- }
-
- /** Constructs a ByteSequencesReader from the provided DataInput */
- public ByteSequencesReader(DataInput is) {
- this.is = is;
+ /** Constructs a ByteSequencesReader from the provided IndexInput */
+ public ByteSequencesReader(IndexInput in) {
+ this.in = in;
}
/**
@@ -519,14 +466,14 @@ public final class OfflineSorter {
public boolean read(BytesRefBuilder ref) throws IOException {
short length;
try {
- length = is.readShort();
+ length = in.readShort();
} catch (EOFException e) {
return false;
}
ref.grow(length);
ref.setLength(length);
- is.readFully(ref.bytes(), 0, length);
+ in.readBytes(ref.bytes(), 0, length);
return true;
}
@@ -542,25 +489,23 @@ public final class OfflineSorter {
public byte[] read() throws IOException {
short length;
try {
- length = is.readShort();
+ length = in.readShort();
} catch (EOFException e) {
return null;
}
assert length >= 0 : "Sanity: sequence length < 0: " + length;
- byte [] result = new byte [length];
- is.readFully(result);
+ byte[] result = new byte[length];
+ in.readBytes(result, 0, length);
return result;
}
/**
- * Closes the provided {@link DataInput} if it is {@link Closeable}.
+ * Closes the provided {@link IndexInput}.
*/
@Override
public void close() throws IOException {
- if (is instanceof Closeable) {
- ((Closeable) is).close();
- }
+ in.close();
}
}
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java Thu Oct 15 10:25:39 2015
@@ -33,7 +33,15 @@ import org.apache.lucene.util.UnicodeUti
* @see #build(Collection)
* @see Automata#makeStringUnion(Collection)
*/
-final class DaciukMihovAutomatonBuilder {
+public final class DaciukMihovAutomatonBuilder {
+
+ /**
+ * The default constructor is private. Use static methods directly.
+ */
+ private DaciukMihovAutomatonBuilder() {
+ super();
+ }
+
/**
* DFSA state with <code>char</code> labels on transitions.
*/
Modified: lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java Thu Oct 15 10:25:39 2015
@@ -103,8 +103,6 @@ public final class BKDWriter {
final byte[] scratch1;
final byte[] scratch2;
- // nocommit grep for lat / lon and fix!
-
private OfflineSorter.ByteSequencesWriter offlinePointWriter;
private HeapPointWriter heapPointWriter;
@@ -163,6 +161,8 @@ public final class BKDWriter {
}
}
+ // nocommit use our own trackingdir to track created temp files for deletion on exc?
+
/** If the current segment has too many points then we switchover to temp files / offline sort. */
private void switchToOffline() throws IOException {
System.out.println("W: switchToOffline");
Modified: lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java Thu Oct 15 10:25:39 2015
@@ -283,7 +283,7 @@ public class TestCodecUtil extends Lucen
final IndexOutput output = new RAMOutputStream(file, false);
AtomicLong fakeChecksum = new AtomicLong();
// wrap the index input where we control the checksum for mocking
- IndexOutput fakeOutput = new IndexOutput("fake") {
+ IndexOutput fakeOutput = new IndexOutput("fake", "fake") {
@Override
public void close() throws IOException {
output.close();
Modified: lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java Thu Oct 15 10:25:39 2015
@@ -24,7 +24,6 @@ import org.apache.lucene.index.FieldInve
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.similarities.Similarity;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.PriorityQueue;
/**
@@ -128,34 +127,6 @@ final class JustCompileSearch {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
- }
-
- static final class JustCompileFilter extends Filter {
- // Filter is just an abstract class with no abstract methods. However it is
- // still added here in case someone will add abstract methods in the future.
-
- @Override
- public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) {
- return null;
- }
-
- @Override
- public String toString(String field) {
- return "JustCompileFilter";
- }
- }
-
- static final class JustCompileFilteredDocIdSet extends FilteredDocIdSet {
-
- public JustCompileFilteredDocIdSet(DocIdSet innerSet) {
- super(innerSet);
- }
-
- @Override
- protected boolean match(int docid) {
- throw new UnsupportedOperationException(UNSUPPORTED_MSG);
- }
-
}
static final class JustCompileFilteredDocIdSetIterator extends FilteredDocIdSetIterator {
Modified: lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/TestBlendedTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/TestBlendedTermQuery.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/TestBlendedTermQuery.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/TestBlendedTermQuery.java Thu Oct 15 10:25:39 2015
@@ -82,7 +82,7 @@ public class TestBlendedTermQuery extend
assertEquals("Blended(foo:bar)", new BlendedTermQuery.Builder().add(t1).build().toString());
Term t2 = new Term("foo", "baz");
assertEquals("Blended(foo:bar foo:baz)", new BlendedTermQuery.Builder().add(t1).add(t2).build().toString());
- assertEquals("Blended(foo:bar^4.0 foo:baz^3.0)", new BlendedTermQuery.Builder().add(t1, 4).add(t2, 3).build().toString());
+ assertEquals("Blended((foo:bar)^4.0 (foo:baz)^3.0)", new BlendedTermQuery.Builder().add(t1, 4).add(t2, 3).build().toString());
}
public void testBlendedScores() throws IOException {
Modified: lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java Thu Oct 15 10:25:39 2015
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -52,6 +53,8 @@ import org.apache.lucene.util.LuceneTest
import org.apache.lucene.util.NamedThreadFactory;
import org.apache.lucene.util.TestUtil;
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+
public class TestBooleanQuery extends LuceneTestCase {
public void testEquality() throws Exception {
@@ -74,6 +77,117 @@ public class TestBooleanQuery extends Lu
assertEquals(bq1.build(), bq2.build());
}
+ public void testEqualityDoesNotDependOnOrder() {
+ TermQuery[] queries = new TermQuery[] {
+ new TermQuery(new Term("foo", "bar")),
+ new TermQuery(new Term("foo", "baz"))
+ };
+ for (int iter = 0; iter < 10; ++iter) {
+ List<BooleanClause> clauses = new ArrayList<>();
+ final int numClauses = random().nextInt(20);
+ for (int i = 0; i < numClauses; ++i) {
+ Query query = RandomPicks.randomFrom(random(), queries);
+ if (random().nextBoolean()) {
+ query = new BoostQuery(query, random().nextFloat());
+ }
+ Occur occur = RandomPicks.randomFrom(random(), Occur.values());
+ clauses.add(new BooleanClause(query, occur));
+ }
+
+ final boolean disableCoord = random().nextBoolean();
+ final int minShouldMatch = random().nextInt(5);
+ BooleanQuery.Builder bq1Builder = new BooleanQuery.Builder();
+ bq1Builder.setDisableCoord(disableCoord);
+ bq1Builder.setMinimumNumberShouldMatch(minShouldMatch);
+ for (BooleanClause clause : clauses) {
+ bq1Builder.add(clause);
+ }
+ final BooleanQuery bq1 = bq1Builder.build();
+
+ Collections.shuffle(clauses, random());
+ BooleanQuery.Builder bq2Builder = new BooleanQuery.Builder();
+ bq2Builder.setDisableCoord(disableCoord);
+ bq2Builder.setMinimumNumberShouldMatch(minShouldMatch);
+ for (BooleanClause clause : clauses) {
+ bq2Builder.add(clause);
+ }
+ final BooleanQuery bq2 = bq2Builder.build();
+
+ QueryUtils.checkEqual(bq1, bq2);
+ }
+ }
+
+ public void testEqualityOnDuplicateShouldClauses() {
+ BooleanQuery bq1 = new BooleanQuery.Builder()
+ .setDisableCoord(random().nextBoolean())
+ .setMinimumNumberShouldMatch(random().nextInt(2))
+ .add(new TermQuery(new Term("foo", "bar")), Occur.SHOULD)
+ .build();
+ BooleanQuery bq2 = new BooleanQuery.Builder()
+ .setDisableCoord(bq1.isCoordDisabled())
+ .setMinimumNumberShouldMatch(bq1.getMinimumNumberShouldMatch())
+ .add(new TermQuery(new Term("foo", "bar")), Occur.SHOULD)
+ .add(new TermQuery(new Term("foo", "bar")), Occur.SHOULD)
+ .build();
+ QueryUtils.checkUnequal(bq1, bq2);
+ }
+
+ public void testEqualityOnDuplicateMustClauses() {
+ BooleanQuery bq1 = new BooleanQuery.Builder()
+ .setDisableCoord(random().nextBoolean())
+ .setMinimumNumberShouldMatch(random().nextInt(2))
+ .add(new TermQuery(new Term("foo", "bar")), Occur.MUST)
+ .build();
+ BooleanQuery bq2 = new BooleanQuery.Builder()
+ .setDisableCoord(bq1.isCoordDisabled())
+ .setMinimumNumberShouldMatch(bq1.getMinimumNumberShouldMatch())
+ .add(new TermQuery(new Term("foo", "bar")), Occur.MUST)
+ .add(new TermQuery(new Term("foo", "bar")), Occur.MUST)
+ .build();
+ QueryUtils.checkUnequal(bq1, bq2);
+ }
+
+ public void testEqualityOnDuplicateFilterClauses() {
+ BooleanQuery bq1 = new BooleanQuery.Builder()
+ .setDisableCoord(random().nextBoolean())
+ .setMinimumNumberShouldMatch(random().nextInt(2))
+ .add(new TermQuery(new Term("foo", "bar")), Occur.FILTER)
+ .build();
+ BooleanQuery bq2 = new BooleanQuery.Builder()
+ .setDisableCoord(bq1.isCoordDisabled())
+ .setMinimumNumberShouldMatch(bq1.getMinimumNumberShouldMatch())
+ .add(new TermQuery(new Term("foo", "bar")), Occur.FILTER)
+ .add(new TermQuery(new Term("foo", "bar")), Occur.FILTER)
+ .build();
+ QueryUtils.checkEqual(bq1, bq2);
+ }
+
+ public void testEqualityOnDuplicateMustNotClauses() {
+ BooleanQuery bq1 = new BooleanQuery.Builder()
+ .setDisableCoord(random().nextBoolean())
+ .setMinimumNumberShouldMatch(random().nextInt(2))
+ .add(new MatchAllDocsQuery(), Occur.MUST)
+ .add(new TermQuery(new Term("foo", "bar")), Occur.FILTER)
+ .build();
+ BooleanQuery bq2 = new BooleanQuery.Builder()
+ .setDisableCoord(bq1.isCoordDisabled())
+ .setMinimumNumberShouldMatch(bq1.getMinimumNumberShouldMatch())
+ .add(new MatchAllDocsQuery(), Occur.MUST)
+ .add(new TermQuery(new Term("foo", "bar")), Occur.FILTER)
+ .add(new TermQuery(new Term("foo", "bar")), Occur.FILTER)
+ .build();
+ QueryUtils.checkEqual(bq1, bq2);
+ }
+
+ public void testHashCodeIsStable() {
+ BooleanQuery bq = new BooleanQuery.Builder()
+ .add(new TermQuery(new Term("foo", TestUtil.randomSimpleString(random()))), Occur.SHOULD)
+ .add(new TermQuery(new Term("foo", TestUtil.randomSimpleString(random()))), Occur.SHOULD)
+ .build();
+ final int hashCode = bq.hashCode();
+ assertEquals(hashCode, bq.hashCode());
+ }
+
public void testException() {
try {
BooleanQuery.setMaxClauseCount(0);
Modified: lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/TestBoostQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/TestBoostQuery.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/TestBoostQuery.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/TestBoostQuery.java Thu Oct 15 10:25:39 2015
@@ -43,7 +43,7 @@ public class TestBoostQuery extends Luce
}
public void testToString() {
- assertEquals("foo:bar^2.0", new BoostQuery(new TermQuery(new Term("foo", "bar")), 2).toString());
+ assertEquals("(foo:bar)^2.0", new BoostQuery(new TermQuery(new Term("foo", "bar")), 2).toString());
BooleanQuery bq = new BooleanQuery.Builder()
.add(new TermQuery(new Term("foo", "bar")), Occur.SHOULD)
.add(new TermQuery(new Term("foo", "baz")), Occur.SHOULD)
Modified: lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperQuery.java?rev=1708778&r1=1708777&r2=1708778&view=diff
==============================================================================
--- lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperQuery.java (original)
+++ lucene/dev/branches/lucene6825/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperQuery.java Thu Oct 15 10:25:39 2015
@@ -18,6 +18,7 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -39,7 +40,7 @@ public class TestCachingWrapperQuery ext
DirectoryReader ir;
IndexSearcher is;
RandomIndexWriter iw;
-
+
@Override
public void setUp() throws Exception {
super.setUp();
@@ -60,7 +61,7 @@ public class TestCachingWrapperQuery ext
ir = iw.getReader();
is = newSearcher(ir);
}
-
+
@Override
public void tearDown() throws Exception {
iw.close();
@@ -80,14 +81,14 @@ public class TestCachingWrapperQuery ext
assertEquals(hits3.totalHits, hits4.totalHits);
CheckHits.checkEqual(f1, hits3.scoreDocs, hits4.scoreDocs);
}
-
+
/** test null iterator */
public void testEmpty() throws Exception {
BooleanQuery.Builder expected = new BooleanQuery.Builder();
Query cached = new CachingWrapperQuery(expected.build(), MAYBE_CACHE_POLICY);
assertQueryEquals(expected.build(), cached);
}
-
+
/** test iterator returns NO_MORE_DOCS */
public void testEmpty2() throws Exception {
BooleanQuery.Builder expected = new BooleanQuery.Builder();
@@ -96,7 +97,7 @@ public class TestCachingWrapperQuery ext
Query cached = new CachingWrapperQuery(expected.build(), MAYBE_CACHE_POLICY);
assertQueryEquals(expected.build(), cached);
}
-
+
/** test iterator returns single document */
public void testSingle() throws Exception {
for (int i = 0; i < 10; i++) {
@@ -106,7 +107,7 @@ public class TestCachingWrapperQuery ext
assertQueryEquals(expected, cached);
}
}
-
+
/** test sparse filters (match single documents) */
public void testSparse() throws Exception {
for (int i = 0; i < 10; i++) {
@@ -118,15 +119,43 @@ public class TestCachingWrapperQuery ext
assertQueryEquals(expected, cached);
}
}
-
+
/** test dense filters (match entire index) */
public void testDense() throws Exception {
- Query query = new MatchAllDocsQuery();
- Filter expected = new QueryWrapperFilter(query);
+ Query expected = new MatchAllDocsQuery();
Query cached = new CachingWrapperQuery(expected, MAYBE_CACHE_POLICY);
assertQueryEquals(expected, cached);
}
+ private static class MockQuery extends Query {
+
+ private final AtomicBoolean wasCalled = new AtomicBoolean();
+
+ public boolean wasCalled() {
+ return wasCalled.get();
+ }
+
+ public void clear() {
+ wasCalled.set(false);
+ }
+
+ @Override
+ public String toString(String field) {
+ return "Mock";
+ }
+
+ @Override
+ public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
+ return new ConstantScoreWeight(this) {
+ @Override
+ public Scorer scorer(LeafReaderContext context) throws IOException {
+ wasCalled.set(true);
+ return new ConstantScoreScorer(this, score(), DocIdSetIterator.all(context.reader().maxDoc()));
+ }
+ };
+ }
+ }
+
public void testCachingWorks() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
@@ -135,7 +164,7 @@ public class TestCachingWrapperQuery ext
IndexReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));
IndexSearcher searcher = newSearcher(reader);
LeafReaderContext context = (LeafReaderContext) reader.getContext();
- MockFilter filter = new MockFilter();
+ MockQuery filter = new MockQuery();
CachingWrapperQuery cacher = new CachingWrapperQuery(filter, QueryCachingPolicy.ALWAYS_CACHE);
// first time, nested filter is called
@@ -257,7 +286,7 @@ public class TestCachingWrapperQuery ext
reader = refreshReader(reader);
searcher = newSearcher(reader, false);
-
+
docs = searcher.search(new ConstantScoreQuery(query), 1);
assertEquals("[query + filter] Should find 2 hits...", 2, docs.totalHits);
assertTrue(query.missCount > missCount);