You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/12/21 12:40:14 UTC
svn commit: r1721134 [8/48] - in /lucene/dev/branches/lucene6835: ./
dev-tools/ dev-tools/eclipse/dot.settings/ dev-tools/idea/.idea/libraries/
dev-tools/maven/ dev-tools/maven/lucene/analysis/morfologik/
dev-tools/scripts/ lucene/ lucene/analysis/ luc...
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java Mon Dec 21 11:39:57 2015
@@ -17,6 +17,11 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
@@ -25,12 +30,6 @@ import org.apache.lucene.search.DocIdSet
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TwoPhaseIterator;
-import org.apache.lucene.util.ToStringUtils;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
/** Removes matches which overlap with another SpanQuery or which are
* within x tokens before or y tokens after another SpanQuery.
@@ -127,7 +126,7 @@ public final class SpanNotQuery extends
Spans excludeSpans = excludeWeight.getSpans(context, requiredPostings);
if (excludeSpans == null) {
- return includeSpans;
+ return new ScoringWrapperSpans(includeSpans, getSimScorer(context));
}
TwoPhaseIterator excludeTwoPhase = excludeSpans.asTwoPhaseIterator();
@@ -195,11 +194,10 @@ public final class SpanNotQuery extends
SpanQuery rewrittenInclude = (SpanQuery) include.rewrite(reader);
SpanQuery rewrittenExclude = (SpanQuery) exclude.rewrite(reader);
if (rewrittenInclude != include || rewrittenExclude != exclude) {
- return new SpanNotQuery(rewrittenInclude, rewrittenExclude);
+ return new SpanNotQuery(rewrittenInclude, rewrittenExclude, pre, post);
}
return super.rewrite(reader);
}
-
/** Returns true iff <code>o</code> is equal to this. */
@Override
public boolean equals(Object o) {
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java Mon Dec 21 11:39:57 2015
@@ -17,6 +17,13 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
@@ -28,13 +35,6 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TwoPhaseIterator;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
/** Matches the union of its clauses.
*/
@@ -166,12 +166,12 @@ public final class SpanOrQuery extends S
if (subSpans.size() == 0) {
return null;
} else if (subSpans.size() == 1) {
- return subSpans.get(0);
+ return new ScoringWrapperSpans(subSpans.get(0), getSimScorer(context));
}
- DisiPriorityQueue<Spans> byDocQueue = new DisiPriorityQueue<>(subSpans.size());
+ DisiPriorityQueue byDocQueue = new DisiPriorityQueue(subSpans.size());
for (Spans spans : subSpans) {
- byDocQueue.add(new DisiWrapper<>(spans));
+ byDocQueue.add(new DisiWrapper(spans));
}
SpanPositionQueue byPositionQueue = new SpanPositionQueue(subSpans.size()); // when empty use -1
@@ -182,7 +182,7 @@ public final class SpanOrQuery extends S
@Override
public int nextDoc() throws IOException {
topPositionSpans = null;
- DisiWrapper<Spans> topDocSpans = byDocQueue.top();
+ DisiWrapper topDocSpans = byDocQueue.top();
int currentDoc = topDocSpans.doc;
do {
topDocSpans.doc = topDocSpans.iterator.nextDoc();
@@ -194,7 +194,7 @@ public final class SpanOrQuery extends S
@Override
public int advance(int target) throws IOException {
topPositionSpans = null;
- DisiWrapper<Spans> topDocSpans = byDocQueue.top();
+ DisiWrapper topDocSpans = byDocQueue.top();
do {
topDocSpans.doc = topDocSpans.iterator.advance(target);
topDocSpans = byDocQueue.updateTop();
@@ -204,36 +204,68 @@ public final class SpanOrQuery extends S
@Override
public int docID() {
- DisiWrapper<Spans> topDocSpans = byDocQueue.top();
+ DisiWrapper topDocSpans = byDocQueue.top();
return topDocSpans.doc;
}
@Override
public TwoPhaseIterator asTwoPhaseIterator() {
- boolean hasApproximation = false;
- for (DisiWrapper<Spans> w : byDocQueue) {
+ float sumMatchCost = 0; // See also DisjunctionScorer.asTwoPhaseIterator()
+ long sumApproxCost = 0;
+
+ for (DisiWrapper w : byDocQueue) {
if (w.twoPhaseView != null) {
- hasApproximation = true;
- break;
+ long costWeight = (w.cost <= 1) ? 1 : w.cost;
+ sumMatchCost += w.twoPhaseView.matchCost() * costWeight;
+ sumApproxCost += costWeight;
}
}
- if (!hasApproximation) { // none of the sub spans supports approximations
+ if (sumApproxCost == 0) { // no sub spans supports approximations
+ computePositionsCost();
return null;
}
- return new TwoPhaseIterator(new DisjunctionDISIApproximation<Spans>(byDocQueue)) {
+ final float matchCost = sumMatchCost / sumApproxCost;
+
+ return new TwoPhaseIterator(new DisjunctionDISIApproximation(byDocQueue)) {
@Override
public boolean matches() throws IOException {
return twoPhaseCurrentDocMatches();
}
+
+ @Override
+ public float matchCost() {
+ return matchCost;
+ }
};
}
+ float positionsCost = -1;
+
+ void computePositionsCost() {
+ float sumPositionsCost = 0;
+ long sumCost = 0;
+ for (DisiWrapper w : byDocQueue) {
+ long costWeight = (w.cost <= 1) ? 1 : w.cost;
+ sumPositionsCost += w.spans.positionsCost() * costWeight;
+ sumCost += costWeight;
+ }
+ positionsCost = sumPositionsCost / sumCost;
+ }
+
+ @Override
+ public float positionsCost() {
+ // This may be called when asTwoPhaseIterator returned null,
+ // which happens when none of the sub spans supports approximations.
+ assert positionsCost > 0;
+ return positionsCost;
+ }
+
int lastDocTwoPhaseMatched = -1;
boolean twoPhaseCurrentDocMatches() throws IOException {
- DisiWrapper<Spans> listAtCurrentDoc = byDocQueue.topList();
+ DisiWrapper listAtCurrentDoc = byDocQueue.topList();
// remove the head of the list as long as it does not match
final int currentDoc = listAtCurrentDoc.doc;
while (listAtCurrentDoc.twoPhaseView != null) {
@@ -257,9 +289,9 @@ public final class SpanOrQuery extends S
void fillPositionQueue() throws IOException { // called at first nextStartPosition
assert byPositionQueue.size() == 0;
// add all matching Spans at current doc to byPositionQueue
- DisiWrapper<Spans> listAtCurrentDoc = byDocQueue.topList();
+ DisiWrapper listAtCurrentDoc = byDocQueue.topList();
while (listAtCurrentDoc != null) {
- Spans spansAtDoc = listAtCurrentDoc.iterator;
+ Spans spansAtDoc = listAtCurrentDoc.spans;
if (lastDocTwoPhaseMatched == listAtCurrentDoc.doc) { // matched by DisjunctionDisiApproximation
if (listAtCurrentDoc.twoPhaseView != null) { // matched by approximation
if (listAtCurrentDoc.lastApproxNonMatchDoc == listAtCurrentDoc.doc) { // matches() returned false
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java Mon Dec 21 11:39:57 2015
@@ -17,6 +17,11 @@ package org.apache.lucene.search.spans;
*/
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
@@ -25,11 +30,6 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.Query;
import org.apache.lucene.search.spans.FilterSpans.AcceptStatus;
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-
/**
* Base class for filtering a SpanQuery based on the position of a match.
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java Mon Dec 21 11:39:57 2015
@@ -17,6 +17,12 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
@@ -27,13 +33,6 @@ import org.apache.lucene.index.TermState
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.util.ToStringUtils;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
/** Matches spans containing a term.
* This should not be used for terms that are indexed at position Integer.MAX_VALUE.
@@ -118,10 +117,40 @@ public class SpanTermQuery extends SpanQ
termsEnum.seekExact(term.bytes(), state);
final PostingsEnum postings = termsEnum.postings(null, requiredPostings.getRequiredPostings());
- return new TermSpans(postings, term);
+ float positionsCost = termPositionsCost(termsEnum) * PHRASE_TO_SPAN_TERM_POSITIONS_COST;
+ return new TermSpans(getSimScorer(context), postings, term, positionsCost);
}
}
+ /** A guess of
+ * the relative cost of dealing with the term positions
+ * when using a SpanNearQuery instead of a PhraseQuery.
+ */
+ private static final float PHRASE_TO_SPAN_TERM_POSITIONS_COST = 4.0f;
+
+ private static final int TERM_POSNS_SEEK_OPS_PER_DOC = 128;
+
+ private static final int TERM_OPS_PER_POS = 7;
+
+ /** Returns an expected cost in simple operations
+ * of processing the occurrences of a term
+ * in a document that contains the term.
+ * <br>This may be inaccurate when {@link TermsEnum#totalTermFreq()} is not available.
+ * @param termsEnum The term is the term at which this TermsEnum is positioned.
+ * <p>
+ * This is a copy of org.apache.lucene.search.PhraseQuery.termPositionsCost().
+ * <br>
+ * TODO: keep only a single copy of this method and the constants used in it
+ * when SpanTermQuery moves to the o.a.l.search package.
+ */
+ static float termPositionsCost(TermsEnum termsEnum) throws IOException {
+ int docFreq = termsEnum.docFreq();
+ assert docFreq > 0;
+ long totalTermFreq = termsEnum.totalTermFreq(); // -1 when not available
+ float expOccurrencesInMatchingDoc = (totalTermFreq < docFreq) ? 1 : (totalTermFreq / (float) docFreq);
+ return TERM_POSNS_SEEK_OPS_PER_DOC + expOccurrencesInMatchingDoc * TERM_OPS_PER_POS;
+ }
+
@Override
public String toString(String field) {
StringBuilder buffer = new StringBuilder();
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java Mon Dec 21 11:39:57 2015
@@ -24,11 +24,9 @@ import org.apache.lucene.index.LeafReade
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
-import org.apache.lucene.index.Terms;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.similarities.Similarity;
@@ -131,28 +129,30 @@ public abstract class SpanWeight extends
}
@Override
- public Scorer scorer(LeafReaderContext context) throws IOException {
- if (field == null) {
+ public SpanScorer scorer(LeafReaderContext context) throws IOException {
+ final Spans spans = getSpans(context, Postings.POSITIONS);
+ if (spans == null) {
return null;
}
- Terms terms = context.reader().terms(field);
- if (terms != null && terms.hasPositions() == false) {
- throw new IllegalStateException("field \"" + field + "\" was indexed without position data; cannot run SpanQuery (query=" + parentQuery + ")");
- }
- Spans spans = getSpans(context, Postings.POSITIONS);
- Similarity.SimScorer simScorer = getSimScorer(context);
- return (spans == null) ? null : new SpanScorer(spans, this, simScorer);
+ final Similarity.SimScorer docScorer = getSimScorer(context);
+ return new SpanScorer(this, spans, docScorer);
}
+ /**
+ * Return a SimScorer for this context
+ * @param context the LeafReaderContext
+ * @return a SimWeight
+ * @throws IOException on error
+ */
public Similarity.SimScorer getSimScorer(LeafReaderContext context) throws IOException {
return simWeight == null ? null : similarity.simScorer(simWeight, context);
}
@Override
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
- SpanScorer scorer = (SpanScorer) scorer(context);
+ SpanScorer scorer = scorer(context);
if (scorer != null) {
- int newDoc = scorer.advance(doc);
+ int newDoc = scorer.iterator().advance(doc);
if (newDoc == doc) {
float freq = scorer.sloppyFreq();
SimScorer docScorer = similarity.simScorer(simWeight, context);
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanWithinQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanWithinQuery.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanWithinQuery.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanWithinQuery.java Mon Dec 21 11:39:57 2015
@@ -17,15 +17,15 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Map;
+
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
import org.apache.lucene.search.IndexSearcher;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Map;
-
/** Keep matches that are contained within another Spans. */
public final class SpanWithinQuery extends SpanContainQuery {
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java Mon Dec 21 11:39:57 2015
@@ -16,12 +16,13 @@ package org.apache.lucene.search.spans;
*/
+import java.io.IOException;
+import java.util.Objects;
+
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DocIdSetIterator;
-
-import java.io.IOException;
-import java.util.Objects;
+import org.apache.lucene.search.similarities.Similarity;
/**
* Expert:
@@ -36,12 +37,16 @@ public class TermSpans extends Spans {
protected int count;
protected int position;
protected boolean readPayload;
+ private final float positionsCost;
- public TermSpans(PostingsEnum postings, Term term) {
+ public TermSpans(Similarity.SimScorer scorer,
+ PostingsEnum postings, Term term, float positionsCost) {
this.postings = Objects.requireNonNull(postings);
this.term = Objects.requireNonNull(term);
this.doc = -1;
this.position = -1;
+ assert positionsCost > 0; // otherwise the TermSpans should not be created.
+ this.positionsCost = positionsCost;
}
@Override
@@ -117,6 +122,11 @@ public class TermSpans extends Spans {
}
@Override
+ public float positionsCost() {
+ return positionsCost;
+ }
+
+ @Override
public String toString() {
return "spans(" + term.toString() + ")@" +
(doc == -1 ? "START" : (doc == NO_MORE_DOCS) ? "ENDDOC"
@@ -126,5 +136,4 @@ public class TermSpans extends Spans {
public PostingsEnum getPostings() {
return postings;
}
-
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java Mon Dec 21 11:39:57 2015
@@ -18,7 +18,6 @@ package org.apache.lucene.store;
*/
import java.io.IOException;
-import java.util.Random;
/**
* Base implementation for a concrete {@link Directory} that uses a {@link LockFactory} for locking.
@@ -32,22 +31,6 @@ public abstract class BaseDirectory exte
* this Directory instance). */
protected final LockFactory lockFactory;
- /** Subclasses can use this to generate temp file name candidates */
- protected static final Random tempFileRandom;
-
- static {
- String prop = System.getProperty("tests.seed");
- int seed;
- if (prop != null) {
- // So if there is a test failure that relied on temp file names,
- //we remain reproducible based on the test seed:
- seed = prop.hashCode();
- } else {
- seed = (int) System.currentTimeMillis();
- }
- tempFileRandom = new Random(seed);
- }
-
/** Sole constructor. */
protected BaseDirectory(LockFactory lockFactory) {
super();
@@ -64,13 +47,13 @@ public abstract class BaseDirectory exte
@Override
protected final void ensureOpen() throws AlreadyClosedException {
- if (!isOpen)
+ if (!isOpen) {
throw new AlreadyClosedException("this Directory is closed");
+ }
}
@Override
public String toString() {
return super.toString() + " lockFactory=" + lockFactory;
}
-
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/Directory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/Directory.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/Directory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/Directory.java Mon Dec 21 11:39:57 2015
@@ -72,7 +72,8 @@ public abstract class Directory implemen
public abstract IndexOutput createOutput(String name, IOContext context) throws IOException;
/** Creates a new, empty file for writing in the directory, with a
- * temporary file name derived from prefix and suffix. Use
+ * temporary file name including prefix and suffix, ending with the
+ * reserved extension <code>.tmp</code>. Use
* {@link IndexOutput#getName} to see what name was used. */
public abstract IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException;
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java Mon Dec 21 11:39:57 2015
@@ -37,6 +37,7 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.util.Constants;
@@ -125,11 +126,15 @@ public abstract class FSDirectory extend
protected final Path directory; // The underlying filesystem directory
+
/** Files we previously tried to delete, but hit exception (on Windows) last time we tried.
* These files are in "pending delete" state, where we refuse to openInput or createOutput
* them, nor include them in .listAll. */
protected final Set<String> pendingDeletes = Collections.newSetFromMap(new ConcurrentHashMap<String,Boolean>());
+ /** Used to generate temp file names in {@link #createTempOutput}. */
+ private final AtomicLong nextTempFileCounter = new AtomicLong();
+
/** Create a new FSDirectory for the named location (ctor for subclasses).
* The directory is created at the named location if it does not yet exist.
*
@@ -251,9 +256,10 @@ public abstract class FSDirectory extend
public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
ensureOpen();
while (true) {
- String name = prefix + tempFileRandom.nextInt(Integer.MAX_VALUE) + "." + suffix;
try {
- return new FSIndexOutput(name, StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
+ String name = IndexFileNames.segmentFileName(prefix, suffix + "_" + Long.toString(nextTempFileCounter.getAndIncrement(), Character.MAX_RADIX), "tmp");
+ return new FSIndexOutput(name,
+ StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
} catch (FileAlreadyExistsException faee) {
// Retry with next random name
}
@@ -273,10 +279,7 @@ public abstract class FSDirectory extend
protected void ensureCanRead(String name) throws IOException {
deletePendingFiles();
if (pendingDeletes.contains(name)) {
- throw new IOException("file \"" + name + "\" is pending delete and cannot be overwritten");
- }
- if (pendingDeletes.contains(name)) {
- throw new IOException("file \"" + name + "\" is pending delete and cannot be opened");
+ throw new NoSuchFileException("file \"" + name + "\" is pending delete and cannot be overwritten");
}
}
@@ -341,9 +344,9 @@ public abstract class FSDirectory extend
// it should only happen on filesystems that can do this, so really we should
// move this logic to WindowsDirectory or something
- // nocommit can/should we do if (Constants.WINDOWS) here, else throw the exc?
+ // TODO: can/should we do if (Constants.WINDOWS) here, else throw the exc?
// but what about a Linux box with a CIFS mount?
- System.out.println("FS.deleteFile failed (" + ioe + "): will retry later");
+ //System.out.println("FS.deleteFile failed (" + ioe + "): will retry later");
pendingDeletes.add(name);
return false;
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java Mon Dec 21 11:39:57 2015
@@ -26,9 +26,9 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
@@ -52,6 +52,9 @@ public class RAMDirectory extends BaseDi
protected final Map<String,RAMFile> fileMap = new ConcurrentHashMap<>();
protected final AtomicLong sizeInBytes = new AtomicLong();
+ /** Used to generate temp file names in {@link #createTempOutput}. */
+ private final AtomicLong nextTempFileCounter = new AtomicLong();
+
/** Constructs an empty {@link Directory}. */
public RAMDirectory() {
this(new SingleInstanceLockFactory());
@@ -111,7 +114,13 @@ public class RAMDirectory extends BaseDi
// and do not synchronize or anything stronger. it's great for testing!
// NOTE: fileMap.keySet().toArray(new String[0]) is broken in non Sun JDKs,
// and the code below is resilient to map changes during the array population.
- return fileMap.keySet().toArray(new String[fileMap.size()]);
+ // NOTE: don't replace this with return names.toArray(new String[names.size()]);
+ // or some files could be null at the end of the array if files are being deleted
+ // concurrently
+ Set<String> fileNames = fileMap.keySet();
+ List<String> names = new ArrayList<>(fileNames.size());
+ for (String name : fileNames) names.add(name);
+ return names.toArray(new String[names.size()]);
}
public final boolean fileNameExists(String name) {
@@ -184,7 +193,7 @@ public class RAMDirectory extends BaseDi
// ... then try to find a unique name for it:
while (true) {
- String name = prefix + tempFileRandom.nextInt(Integer.MAX_VALUE) + "." + suffix;
+ String name = IndexFileNames.segmentFileName(prefix, suffix + "_" + Long.toString(nextTempFileCounter.getAndIncrement(), Character.MAX_RADIX), "tmp");
if (fileMap.putIfAbsent(name, file) == null) {
return new RAMOutputStream(name, file, true);
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java Mon Dec 21 11:39:57 2015
@@ -24,6 +24,7 @@ import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.NoSuchElementException;
+import java.util.Objects;
import java.util.Set;
import org.apache.lucene.analysis.TokenStream; // for javadocs
@@ -80,9 +81,7 @@ public class AttributeSource {
* An AttributeSource that uses the same attributes as the supplied one.
*/
public AttributeSource(AttributeSource input) {
- if (input == null) {
- throw new IllegalArgumentException("input AttributeSource must not be null");
- }
+ Objects.requireNonNull(input, "input AttributeSource must not be null");
this.attributes = input.attributes;
this.attributeImpls = input.attributeImpls;
this.currentState = input.currentState;
@@ -96,7 +95,7 @@ public class AttributeSource {
this.attributes = new LinkedHashMap<>();
this.attributeImpls = new LinkedHashMap<>();
this.currentState = new State[1];
- this.factory = factory;
+ this.factory = Objects.requireNonNull(factory, "AttributeFactory must not be null");
}
/**
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java Mon Dec 21 11:39:57 2015
@@ -186,6 +186,7 @@ public final class ByteBlockPool {
}
}
}
+
/**
* Advances the pool to its next buffer. This method should be called once
* after the constructor to initialize the pool. In contrast to the
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/IOUtils.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/IOUtils.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/IOUtils.java Mon Dec 21 11:39:57 2015
@@ -408,46 +408,21 @@ public final class IOUtils {
* because not all file systems and operating systems allow to fsync on a directory)
*/
public static void fsync(Path fileToSync, boolean isDir) throws IOException {
- IOException exc = null;
-
// If the file is a directory we have to open read-only, for regular files we must open r/w for the fsync to have an effect.
// See http://blog.httrack.com/blog/2013/11/15/everything-you-always-wanted-to-know-about-fsync/
try (final FileChannel file = FileChannel.open(fileToSync, isDir ? StandardOpenOption.READ : StandardOpenOption.WRITE)) {
- for (int retry = 0; retry < 5; retry++) {
- try {
- file.force(true);
- return;
- } catch (IOException ioe) {
- if (exc == null) {
- exc = ioe;
- }
- try {
- // Pause 5 msec
- Thread.sleep(5L);
- } catch (InterruptedException ie) {
- ThreadInterruptedException ex = new ThreadInterruptedException(ie);
- ex.addSuppressed(exc);
- throw ex;
- }
- }
- }
+ file.force(true);
} catch (IOException ioe) {
- if (exc == null) {
- exc = ioe;
+ if (isDir) {
+ assert (Constants.LINUX || Constants.MAC_OS_X) == false :
+ "On Linux and MacOSX fsyncing a directory should not throw IOException, "+
+ "we just don't want to rely on that in production (undocumented). Got: " + ioe;
+ // Ignore exception if it is a directory
+ return;
}
+ // Throw original exception
+ throw ioe;
}
-
- if (isDir) {
- // TODO: LUCENE-6169 - Fix this assert once Java 9 problems are solved!
- assert (Constants.LINUX || Constants.MAC_OS_X) == false || Constants.JRE_IS_MINIMUM_JAVA9 :
- "On Linux and MacOSX fsyncing a directory should not throw IOException, "+
- "we just don't want to rely on that in production (undocumented). Got: " + exc;
- // Ignore exception if it is a directory
- return;
- }
-
- // Throw original exception
- throw exc;
}
/** If the dir is an {@link FSDirectory} or wraps one via possibly
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/NamedSPILoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/NamedSPILoader.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/NamedSPILoader.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/NamedSPILoader.java Mon Dec 21 11:39:57 2015
@@ -20,6 +20,7 @@ package org.apache.lucene.util;
import java.util.Collections;
import java.util.Iterator;
import java.util.Map;
+import java.util.Objects;
import java.util.LinkedHashMap;
import java.util.Set;
import java.util.ServiceConfigurationError;
@@ -41,6 +42,9 @@ public final class NamedSPILoader<S exte
this.clazz = clazz;
// if clazz' classloader is not a parent of the given one, we scan clazz's classloader, too:
final ClassLoader clazzClassloader = clazz.getClassLoader();
+ if (classloader == null) {
+ classloader = clazzClassloader;
+ }
if (clazzClassloader != null && !SPIClassIterator.isParentClassLoader(clazzClassloader, classloader)) {
reload(clazzClassloader);
}
@@ -59,6 +63,7 @@ public final class NamedSPILoader<S exte
* of new service providers on the given classpath/classloader!</em>
*/
public void reload(ClassLoader classloader) {
+ Objects.requireNonNull(classloader, "classloader");
final LinkedHashMap<String,S> services = new LinkedHashMap<>(this.services);
final SPIClassIterator<S> loader = SPIClassIterator.get(clazz, classloader);
while (loader.hasNext()) {
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java Mon Dec 21 11:39:57 2015
@@ -43,7 +43,7 @@ import org.apache.lucene.store.TrackingD
* @lucene.experimental
* @lucene.internal
*/
-public final class OfflineSorter {
+public class OfflineSorter {
/** Convenience constant for megabytes */
public final static long MB = 1024 * 1024;
@@ -237,7 +237,7 @@ public final class OfflineSorter {
TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(dir);
boolean success = false;
- try (ByteSequencesReader is = new ByteSequencesReader(dir.openInput(inputFileName, IOContext.READONCE))) {
+ try (ByteSequencesReader is = getReader(dir.openInput(inputFileName, IOContext.READONCE))) {
int lineCount;
while ((lineCount = readPartition(is)) > 0) {
@@ -284,8 +284,9 @@ public final class OfflineSorter {
protected String sortPartition(TrackingDirectoryWrapper trackingDir) throws IOException {
BytesRefArray data = this.buffer;
- try (IndexOutput tempFile = trackingDir.createTempOutput(tempFileNamePrefix, "sort", IOContext.DEFAULT)) {
- ByteSequencesWriter out = new ByteSequencesWriter(tempFile);
+ try (IndexOutput tempFile = trackingDir.createTempOutput(tempFileNamePrefix, "sort", IOContext.DEFAULT);
+ ByteSequencesWriter out = getWriter(tempFile);) {
+
BytesRef spare;
long start = System.currentTimeMillis();
@@ -320,16 +321,18 @@ public final class OfflineSorter {
String newSegmentName = null;
- try (IndexOutput out = trackingDir.createTempOutput(tempFileNamePrefix, "sort", IOContext.DEFAULT)) {
- newSegmentName = out.getName();
- ByteSequencesWriter writer = new ByteSequencesWriter(out);
+ try (IndexOutput out = trackingDir.createTempOutput(tempFileNamePrefix, "sort", IOContext.DEFAULT);
+ ByteSequencesWriter writer = getWriter(out);) {
+ newSegmentName = out.getName();
+
// Open streams and read the top for each file
for (int i = 0; i < segments.size(); i++) {
- streams[i] = new ByteSequencesReader(dir.openInput(segments.get(i), IOContext.READONCE));
- byte[] line = streams[i].read();
- assert line != null;
- queue.insertWithOverflow(new FileAndTop(i, line));
+ streams[i] = getReader(dir.openInput(segments.get(i), IOContext.READONCE));
+ BytesRefBuilder bytes = new BytesRefBuilder();
+ boolean result = streams[i].read(bytes);
+ assert result;
+ queue.insertWithOverflow(new FileAndTop(i, bytes));
}
// Unix utility sort() uses ordered array of files to pick the next line from, updating
@@ -363,13 +366,12 @@ public final class OfflineSorter {
/** Read in a single partition of data */
int readPartition(ByteSequencesReader reader) throws IOException {
long start = System.currentTimeMillis();
- final BytesRef scratch = new BytesRef();
- while ((scratch.bytes = reader.read()) != null) {
- scratch.length = scratch.bytes.length;
- buffer.append(scratch);
+ final BytesRefBuilder scratch = new BytesRefBuilder();
+ while (reader.read(scratch)) {
+ buffer.append(scratch.get());
// Account for the created objects.
// (buffer slots do not account to buffer size.)
- if (ramBufferSize.bytes < bufferBytesUsed.get()) {
+ if (bufferBytesUsed.get() > ramBufferSize.bytes) {
break;
}
}
@@ -381,19 +383,28 @@ public final class OfflineSorter {
final int fd;
final BytesRefBuilder current;
- FileAndTop(int fd, byte[] firstLine) {
+ FileAndTop(int fd, BytesRefBuilder firstLine) {
this.fd = fd;
- this.current = new BytesRefBuilder();
- this.current.copyBytes(firstLine, 0, firstLine.length);
+ this.current = firstLine;
}
}
+ /** Subclasses can override to change how byte sequences are written to disk. */
+ protected ByteSequencesWriter getWriter(IndexOutput out) throws IOException {
+ return new ByteSequencesWriter(out);
+ }
+
+ /** Subclasses can override to change how byte sequences are read from disk. */
+ protected ByteSequencesReader getReader(IndexInput in) throws IOException {
+ return new ByteSequencesReader(in);
+ }
+
/**
* Utility class to emit length-prefixed byte[] entries to an output stream for sorting.
* Complementary to {@link ByteSequencesReader}.
*/
public static class ByteSequencesWriter implements Closeable {
- private final IndexOutput out;
+ protected final IndexOutput out;
/** Constructs a ByteSequencesWriter to the provided DataOutput */
public ByteSequencesWriter(IndexOutput out) {
@@ -404,7 +415,7 @@ public final class OfflineSorter {
* Writes a BytesRef.
* @see #write(byte[], int, int)
*/
- public void write(BytesRef ref) throws IOException {
+ public final void write(BytesRef ref) throws IOException {
assert ref != null;
write(ref.bytes, ref.offset, ref.length);
}
@@ -413,7 +424,7 @@ public final class OfflineSorter {
* Writes a byte array.
* @see #write(byte[], int, int)
*/
- public void write(byte[] bytes) throws IOException {
+ public final void write(byte[] bytes) throws IOException {
write(bytes, 0, bytes.length);
}
@@ -448,7 +459,7 @@ public final class OfflineSorter {
* Complementary to {@link ByteSequencesWriter}.
*/
public static class ByteSequencesReader implements Closeable {
- private final IndexInput in;
+ protected final IndexInput in;
/** Constructs a ByteSequencesReader from the provided IndexInput */
public ByteSequencesReader(IndexInput in) {
@@ -478,29 +489,6 @@ public final class OfflineSorter {
}
/**
- * Reads the next entry and returns it if successful.
- *
- * @see #read(BytesRefBuilder)
- *
- * @return Returns <code>null</code> if EOF occurred before the next entry
- * could be read.
- * @throws EOFException if the file ends before the full sequence is read.
- */
- public byte[] read() throws IOException {
- short length;
- try {
- length = in.readShort();
- } catch (EOFException e) {
- return null;
- }
-
- assert length >= 0 : "Sanity: sequence length < 0: " + length;
- byte[] result = new byte[length];
- in.readBytes(result, 0, length);
- return result;
- }
-
- /**
* Closes the provided {@link IndexInput}.
*/
@Override
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/PriorityQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/PriorityQueue.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/PriorityQueue.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/PriorityQueue.java Mon Dec 21 11:39:57 2015
@@ -1,5 +1,8 @@
package org.apache.lucene.util;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -27,10 +30,12 @@ package org.apache.lucene.util;
* <code>maxSize+1</code> if instantiated via the
* {@link #PriorityQueue(int,boolean)} constructor with <code>prepopulate</code>
* set to <code>true</code>.
- *
+ *
+ * <b>NOTE</b>: Iteration order is not specified.
+ *
* @lucene.internal
*/
-public abstract class PriorityQueue<T> {
+public abstract class PriorityQueue<T> implements Iterable<T> {
private int size = 0;
private final int maxSize;
private final T[] heap;
@@ -58,7 +63,7 @@ public abstract class PriorityQueue<T> {
@SuppressWarnings("unchecked") final T[] h = (T[]) new Object[heapSize];
this.heap = h;
this.maxSize = maxSize;
-
+
if (prepopulate) {
// If sentinel objects are supported, populate the queue with them
T sentinel = getSentinelObject();
@@ -80,41 +85,41 @@ public abstract class PriorityQueue<T> {
/**
* This method can be overridden by extending classes to return a sentinel
- * object which will be used by the {@link PriorityQueue#PriorityQueue(int,boolean)}
+ * object which will be used by the {@link PriorityQueue#PriorityQueue(int,boolean)}
* constructor to fill the queue, so that the code which uses that queue can always
* assume it's full and only change the top without attempting to insert any new
* object.<br>
- *
+ *
* Those sentinel values should always compare worse than any non-sentinel
* value (i.e., {@link #lessThan} should always favor the
* non-sentinel values).<br>
- *
+ *
* By default, this method returns null, which means the queue will not be
* filled with sentinel values. Otherwise, the value returned will be used to
* pre-populate the queue. Adds sentinel values to the queue.<br>
- *
+ *
* If this method is extended to return a non-null value, then the following
* usage pattern is recommended:
- *
+ *
* <pre class="prettyprint">
* // extends getSentinelObject() to return a non-null value.
* PriorityQueue<MyObject> pq = new MyQueue<MyObject>(numHits);
* // save the 'top' element, which is guaranteed to not be null.
* MyObject pqTop = pq.top();
* <...>
- * // now in order to add a new element, which is 'better' than top (after
+ * // now in order to add a new element, which is 'better' than top (after
* // you've verified it is better), it is as simple as:
* pqTop.change().
* pqTop = pq.updateTop();
* </pre>
- *
+ *
* <b>NOTE:</b> if this method returns a non-null value, it will be called by
- * the {@link PriorityQueue#PriorityQueue(int,boolean)} constructor
+ * the {@link PriorityQueue#PriorityQueue(int,boolean)} constructor
* {@link #size()} times, relying on a new object to be returned and will not
* check if it's null again. Therefore you should ensure any call to this
* method creates a new instance and behaves consistently, e.g., it cannot
* return null if it previously returned non-null.
- *
+ *
* @return the sentinel object to use to pre-populate the queue, or null if
* sentinel objects are not supported.
*/
@@ -126,7 +131,7 @@ public abstract class PriorityQueue<T> {
* Adds an Object to a PriorityQueue in log(size) time. If one tries to add
* more objects than maxSize from initialize an
* {@link ArrayIndexOutOfBoundsException} is thrown.
- *
+ *
* @return the new 'top' element in the queue.
*/
public final T add(T element) {
@@ -182,24 +187,24 @@ public abstract class PriorityQueue<T> {
return null;
}
}
-
+
/**
* Should be called when the Object at top changes values. Still log(n) worst
* case, but it's at least twice as fast to
- *
+ *
* <pre class="prettyprint">
* pq.top().change();
* pq.updateTop();
* </pre>
- *
+ *
* instead of
- *
+ *
* <pre class="prettyprint">
* o = pq.pop();
* o.change();
* pq.push(o);
* </pre>
- *
+ *
* @return the new 'top' element.
*/
public final T updateTop() {
@@ -263,7 +268,7 @@ public abstract class PriorityQueue<T> {
heap[i] = node; // install saved node
return i != origPos;
}
-
+
private final void downHeap(int i) {
T node = heap[i]; // save top node
int j = i << 1; // find smaller child
@@ -289,4 +294,26 @@ public abstract class PriorityQueue<T> {
protected final Object[] getHeapArray() {
return (Object[]) heap;
}
+
+ @Override
+ public Iterator<T> iterator() {
+ return new Iterator<T>() {
+
+ int i = 1;
+
+ @Override
+ public boolean hasNext() {
+ return i <= size;
+ }
+
+ @Override
+ public T next() {
+ if (hasNext() == false) {
+ throw new NoSuchElementException();
+ }
+ return heap[i++];
+ }
+
+ };
+ }
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java Mon Dec 21 11:39:57 2015
@@ -274,7 +274,7 @@ public class QueryBuilder {
throw new AssertionError();
}
- return newTermQuery(new Term(field, BytesRef.deepCopyOf(termAtt.getBytesRef())));
+ return newTermQuery(new Term(field, termAtt.getBytesRef()));
}
/**
@@ -286,7 +286,7 @@ public class QueryBuilder {
stream.reset();
List<Term> terms = new ArrayList<>();
while (stream.incrementToken()) {
- terms.add(new Term(field, BytesRef.deepCopyOf(termAtt.getBytesRef())));
+ terms.add(new Term(field, termAtt.getBytesRef()));
}
return newSynonymQuery(terms.toArray(new Term[terms.size()]));
@@ -319,7 +319,7 @@ public class QueryBuilder {
add(q, currentQuery, operator);
currentQuery.clear();
}
- currentQuery.add(new Term(field, BytesRef.deepCopyOf(termAtt.getBytesRef())));
+ currentQuery.add(new Term(field, termAtt.getBytesRef()));
}
add(q, currentQuery, operator);
@@ -376,7 +376,7 @@ public class QueryBuilder {
multiTerms.clear();
}
position += positionIncrement;
- multiTerms.add(new Term(field, BytesRef.deepCopyOf(termAtt.getBytesRef())));
+ multiTerms.add(new Term(field, termAtt.getBytesRef()));
}
if (enablePositionIncrements) {
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java Mon Dec 21 11:39:57 2015
@@ -21,6 +21,8 @@ import java.lang.reflect.Array;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.util.IdentityHashMap;
@@ -290,7 +292,13 @@ public final class RamUsageEstimator {
// Walk type hierarchy
for (;clazz != null; clazz = clazz.getSuperclass()) {
- final Field[] fields = clazz.getDeclaredFields();
+ final Class<?> target = clazz;
+ final Field[] fields = AccessController.doPrivileged(new PrivilegedAction<Field[]>() {
+ @Override
+ public Field[] run() {
+ return target.getDeclaredFields();
+ }
+ });
for (Field f : fields) {
if (!Modifier.isStatic(f.getModifiers())) {
size = adjustForField(size, f);
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/SPIClassIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/SPIClassIterator.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/SPIClassIterator.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/SPIClassIterator.java Mon Dec 21 11:39:57 2015
@@ -29,6 +29,7 @@ import java.util.Enumeration;
import java.util.Iterator;
import java.util.Locale;
import java.util.NoSuchElementException;
+import java.util.Objects;
import java.util.ServiceConfigurationError;
/**
@@ -47,35 +48,51 @@ public final class SPIClassIterator<S> i
private final Enumeration<URL> profilesEnum;
private Iterator<String> linesIterator;
+ /** Creates a new SPI iterator to lookup services of type {@code clazz} using the context classloader. */
public static <S> SPIClassIterator<S> get(Class<S> clazz) {
- return new SPIClassIterator<>(clazz, Thread.currentThread().getContextClassLoader());
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+ if (cl == null) {
+ cl = clazz.getClassLoader();
+ }
+ return new SPIClassIterator<>(clazz, cl);
}
+ /** Creates a new SPI iterator to lookup services of type {@code clazz} using the given classloader. */
public static <S> SPIClassIterator<S> get(Class<S> clazz, ClassLoader loader) {
return new SPIClassIterator<>(clazz, loader);
}
- /** Utility method to check if some class loader is a (grand-)parent of or the same as another one.
- * This means the child will be able to load all classes from the parent, too. */
- public static boolean isParentClassLoader(final ClassLoader parent, ClassLoader child) {
- while (child != null) {
- if (child == parent) {
- return true;
+ /**
+ * Utility method to check if some class loader is a (grand-)parent of or the same as another one.
+ * This means the child will be able to load all classes from the parent, too.
+ * <p>
+ * If caller's codesource doesn't have enough permissions to do the check, {@code false} is returned
+ * (this is fine, because if we get a {@code SecurityException} it is for sure no parent).
+ */
+ public static boolean isParentClassLoader(final ClassLoader parent, final ClassLoader child) {
+ try {
+ ClassLoader cl = child;
+ while (cl != null) {
+ if (cl == parent) {
+ return true;
+ }
+ cl = cl.getParent();
}
- child = child.getParent();
+ return false;
+ } catch (SecurityException se) {
+ return false;
}
- return false;
}
private SPIClassIterator(Class<S> clazz, ClassLoader loader) {
- this.clazz = clazz;
+ this.clazz = Objects.requireNonNull(clazz, "clazz");
+ this.loader = Objects.requireNonNull(loader, "loader");
try {
final String fullName = META_INF_SERVICES + clazz.getName();
- this.profilesEnum = (loader == null) ? ClassLoader.getSystemResources(fullName) : loader.getResources(fullName);
+ this.profilesEnum = loader.getResources(fullName);
} catch (IOException ioe) {
throw new ServiceConfigurationError("Error loading SPI profiles for type " + clazz.getName() + " from classpath", ioe);
}
- this.loader = (loader == null) ? ClassLoader.getSystemClassLoader() : loader;
this.linesIterator = Collections.<String>emptySet().iterator();
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/SloppyMath.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/SloppyMath.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/SloppyMath.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/SloppyMath.java Mon Dec 21 11:39:57 2015
@@ -33,8 +33,10 @@ package org.apache.lucene.util;
public class SloppyMath {
/**
- * Returns the distance in kilometers between two points
- * specified in decimal degrees (latitude/longitude).
+ * Returns the Haversine distance in kilometers between two points
+ * specified in decimal degrees (latitude/longitude). This works correctly
+ * even if the dateline is between the two points.
+ *
* @param lat1 Latitude of the first point.
* @param lon1 Longitude of the first point.
* @param lat2 Latitude of the second point.
@@ -85,7 +87,41 @@ public class SloppyMath {
double indexSin = sinTab[index];
return indexCos + delta * (-indexSin + delta * (-indexCos * ONE_DIV_F2 + delta * (indexSin * ONE_DIV_F3 + delta * indexCos * ONE_DIV_F4)));
}
-
+
+ /**
+ * Returns the trigonometric sine of an angle converted as a cos operation.
+ * <p>
+ * Error is around 1E-15.
+ * <p>
+ * Special cases:
+ * <ul>
+ * <li>If the argument is {@code NaN} or an infinity, then the result is {@code NaN}.
+ * </ul>
+ * @param a an angle, in radians.
+ * @return the sine of the argument.
+ * @see Math#cos(double)
+ */
+ public static double sin(double a) {
+ return cos(a - PIO2);
+ }
+
+ /**
+ * Returns the trigonometric tangent of an angle converted in terms of a sin/cos operation.
+ * <p>
+ * Error is around 1E-15.
+ * <p>
+ * Special cases:
+ * <ul>
+ * <li>If the argument is {@code NaN} or an infinity, then the result is {@code NaN}.
+ * </ul>
+ * @param a an angle, in radians.
+ * @return the tangent of the argument.
+ * @see Math#sin(double) aand Math#cos(double)
+ */
+ public static double tan(double a) {
+ return sin(a) / cos(a);
+ }
+
/**
* Returns the arc sine of a value.
* <p>
@@ -144,13 +180,15 @@ public class SloppyMath {
}
// haversin
- private static final double TO_RADIANS = Math.PI / 180D;
+ static final double TO_RADIANS = Math.PI / 180D;
+ static final double TO_DEGREES = 180D / Math.PI;
// cos/asin
private static final double ONE_DIV_F2 = 1/2.0;
private static final double ONE_DIV_F3 = 1/6.0;
private static final double ONE_DIV_F4 = 1/24.0;
-
+
+ static final double PIO2 = Math.PI / 2D;
private static final double PIO2_HI = Double.longBitsToDouble(0x3FF921FB54400000L); // 1.57079632673412561417e+00 first 33 bits of pi/2
private static final double PIO2_LO = Double.longBitsToDouble(0x3DD0B4611A626331L); // 6.07710050650619224932e-11 pi/2 - PIO2_HI
private static final double TWOPI_HI = 4*PIO2_HI;
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/StringHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/StringHelper.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/StringHelper.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/StringHelper.java Mon Dec 21 11:39:57 2015
@@ -380,4 +380,18 @@ public abstract class StringHelper {
return new BytesRef(bytes);
}
+
+ /** Compares a fixed length slice of two byte arrays interpreted as
+ * big-endian unsigned values. Returns positive int if a > b,
+ * negative int if a < b and 0 if a == b */
+ public static int compare(int count, byte[] a, int aOffset, byte[] b, int bOffset) {
+ for(int i=0;i<count;i++) {
+ int cmp = (a[aOffset+i]&0xff) - (b[bOffset+i]&0xff);
+ if (cmp != 0) {
+ return cmp;
+ }
+ }
+
+ return 0;
+ }
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/Version.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/Version.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/Version.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/Version.java Mon Dec 21 11:39:57 2015
@@ -81,6 +81,13 @@ public final class Version {
@Deprecated
public static final Version LUCENE_5_4_0 = new Version(5, 4, 0);
+ /**
+ * Match settings and bugs in Lucene's 5.5.0 release.
+ * @deprecated Use latest
+ */
+ @Deprecated
+ public static final Version LUCENE_5_5_0 = new Version(5, 5, 0);
+
/** Match settings and bugs in Lucene's 6.0 release.
* <p>
* Use this to get the latest & greatest settings, bug
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java?rev=1721134&r1=1721126&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java Mon Dec 21 11:39:57 2015
@@ -22,6 +22,7 @@ import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Comparator;
import java.util.List;
@@ -39,8 +40,8 @@ import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntroSorter;
import org.apache.lucene.util.LongBitSet;
import org.apache.lucene.util.NumericUtils;
-import org.apache.lucene.util.OfflineSorter;
import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter;
+import org.apache.lucene.util.OfflineSorter;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.StringHelper;
@@ -790,7 +791,7 @@ public class BKDWriter implements Closea
//System.out.println("sort time: " + ((t1-t0)/1000000.0) + " msec");
if (tempInput != null) {
- tempDir.deleteFile(tempInput.getName());
+ tempDir.deleteFiles(Collections.singleton(tempInput.getName()));
tempInput = null;
} else {
assert heapPointWriter != null;
@@ -881,7 +882,7 @@ public class BKDWriter implements Closea
try {
tempInput.close();
} finally {
- tempDir.deleteFile(tempInput.getName());
+ tempDir.deleteFiles(Collections.singleton(tempInput.getName()));
tempInput = null;
}
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java?rev=1721134&r1=1721126&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java Mon Dec 21 11:39:57 2015
@@ -18,7 +18,7 @@ package org.apache.lucene.util.bkd;
*/
import java.io.IOException;
-
+import java.util.Collections;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -75,7 +75,7 @@ final class OfflinePointWriter implement
@Override
public void destroy() throws IOException {
- tempDir.deleteFile(out.getName());
+ tempDir.deleteFiles(Collections.singleton(out.getName()));
}
@Override
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/packed/DirectReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/packed/DirectReader.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/packed/DirectReader.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/packed/DirectReader.java Mon Dec 21 11:39:57 2015
@@ -43,37 +43,47 @@ public class DirectReader {
* {@code bitsPerValue} for each value
*/
public static LongValues getInstance(RandomAccessInput slice, int bitsPerValue) {
+ return getInstance(slice, bitsPerValue, 0);
+ }
+
+ /**
+ * Retrieves an instance from the specified {@code offset} of the given slice
+ * decoding {@code bitsPerValue} for each value
+ */
+ public static LongValues getInstance(RandomAccessInput slice, int bitsPerValue, long offset) {
switch (bitsPerValue) {
- case 1: return new DirectPackedReader1(slice);
- case 2: return new DirectPackedReader2(slice);
- case 4: return new DirectPackedReader4(slice);
- case 8: return new DirectPackedReader8(slice);
- case 12: return new DirectPackedReader12(slice);
- case 16: return new DirectPackedReader16(slice);
- case 20: return new DirectPackedReader20(slice);
- case 24: return new DirectPackedReader24(slice);
- case 28: return new DirectPackedReader28(slice);
- case 32: return new DirectPackedReader32(slice);
- case 40: return new DirectPackedReader40(slice);
- case 48: return new DirectPackedReader48(slice);
- case 56: return new DirectPackedReader56(slice);
- case 64: return new DirectPackedReader64(slice);
+ case 1: return new DirectPackedReader1(slice, offset);
+ case 2: return new DirectPackedReader2(slice, offset);
+ case 4: return new DirectPackedReader4(slice, offset);
+ case 8: return new DirectPackedReader8(slice, offset);
+ case 12: return new DirectPackedReader12(slice, offset);
+ case 16: return new DirectPackedReader16(slice, offset);
+ case 20: return new DirectPackedReader20(slice, offset);
+ case 24: return new DirectPackedReader24(slice, offset);
+ case 28: return new DirectPackedReader28(slice, offset);
+ case 32: return new DirectPackedReader32(slice, offset);
+ case 40: return new DirectPackedReader40(slice, offset);
+ case 48: return new DirectPackedReader48(slice, offset);
+ case 56: return new DirectPackedReader56(slice, offset);
+ case 64: return new DirectPackedReader64(slice, offset);
default: throw new IllegalArgumentException("unsupported bitsPerValue: " + bitsPerValue);
}
}
static final class DirectPackedReader1 extends LongValues {
final RandomAccessInput in;
+ final long offset;
- DirectPackedReader1(RandomAccessInput in) {
+ DirectPackedReader1(RandomAccessInput in, long offset) {
this.in = in;
+ this.offset = offset;
}
@Override
public long get(long index) {
try {
int shift = 7 - (int) (index & 7);
- return (in.readByte(index >>> 3) >>> shift) & 0x1;
+ return (in.readByte(offset + (index >>> 3)) >>> shift) & 0x1;
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -82,16 +92,18 @@ public class DirectReader {
static final class DirectPackedReader2 extends LongValues {
final RandomAccessInput in;
+ final long offset;
- DirectPackedReader2(RandomAccessInput in) {
+ DirectPackedReader2(RandomAccessInput in, long offset) {
this.in = in;
+ this.offset = offset;
}
@Override
public long get(long index) {
try {
int shift = (3 - (int)(index & 3)) << 1;
- return (in.readByte(index >>> 2) >>> shift) & 0x3;
+ return (in.readByte(offset + (index >>> 2)) >>> shift) & 0x3;
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -100,16 +112,18 @@ public class DirectReader {
static final class DirectPackedReader4 extends LongValues {
final RandomAccessInput in;
+ final long offset;
- DirectPackedReader4(RandomAccessInput in) {
+ DirectPackedReader4(RandomAccessInput in, long offset) {
this.in = in;
+ this.offset = offset;
}
@Override
public long get(long index) {
try {
int shift = (int) ((index + 1) & 1) << 2;
- return (in.readByte(index >>> 1) >>> shift) & 0xF;
+ return (in.readByte(offset + (index >>> 1)) >>> shift) & 0xF;
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -118,15 +132,17 @@ public class DirectReader {
static final class DirectPackedReader8 extends LongValues {
final RandomAccessInput in;
+ final long offset;
- DirectPackedReader8(RandomAccessInput in) {
+ DirectPackedReader8(RandomAccessInput in, long offset) {
this.in = in;
+ this.offset = offset;
}
@Override
public long get(long index) {
try {
- return in.readByte(index) & 0xFF;
+ return in.readByte(offset + index) & 0xFF;
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -135,9 +151,11 @@ public class DirectReader {
static final class DirectPackedReader12 extends LongValues {
final RandomAccessInput in;
+ final long offset;
- DirectPackedReader12(RandomAccessInput in) {
+ DirectPackedReader12(RandomAccessInput in, long offset) {
this.in = in;
+ this.offset = offset;
}
@Override
@@ -145,7 +163,7 @@ public class DirectReader {
try {
long offset = (index * 12) >>> 3;
int shift = (int) ((index + 1) & 1) << 2;
- return (in.readShort(offset) >>> shift) & 0xFFF;
+ return (in.readShort(this.offset + offset) >>> shift) & 0xFFF;
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -154,15 +172,17 @@ public class DirectReader {
static final class DirectPackedReader16 extends LongValues {
final RandomAccessInput in;
+ final long offset;
- DirectPackedReader16(RandomAccessInput in) {
+ DirectPackedReader16(RandomAccessInput in, long offset) {
this.in = in;
+ this.offset = offset;
}
@Override
public long get(long index) {
try {
- return in.readShort(index << 1) & 0xFFFF;
+ return in.readShort(offset + (index << 1)) & 0xFFFF;
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -171,9 +191,11 @@ public class DirectReader {
static final class DirectPackedReader20 extends LongValues {
final RandomAccessInput in;
+ final long offset;
- DirectPackedReader20(RandomAccessInput in) {
+ DirectPackedReader20(RandomAccessInput in, long offset) {
this.in = in;
+ this.offset = offset;
}
@Override
@@ -181,7 +203,7 @@ public class DirectReader {
try {
long offset = (index * 20) >>> 3;
// TODO: clean this up...
- int v = in.readInt(offset) >>> 8;
+ int v = in.readInt(this.offset + offset) >>> 8;
int shift = (int) ((index + 1) & 1) << 2;
return (v >>> shift) & 0xFFFFF;
} catch (IOException e) {
@@ -192,15 +214,17 @@ public class DirectReader {
static final class DirectPackedReader24 extends LongValues {
final RandomAccessInput in;
+ final long offset;
- DirectPackedReader24(RandomAccessInput in) {
+ DirectPackedReader24(RandomAccessInput in, long offset) {
this.in = in;
+ this.offset = offset;
}
@Override
public long get(long index) {
try {
- return in.readInt(index * 3) >>> 8;
+ return in.readInt(offset + index * 3) >>> 8;
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -209,9 +233,11 @@ public class DirectReader {
static final class DirectPackedReader28 extends LongValues {
final RandomAccessInput in;
+ final long offset;
- DirectPackedReader28(RandomAccessInput in) {
+ DirectPackedReader28(RandomAccessInput in, long offset) {
this.in = in;
+ this.offset = offset;
}
@Override
@@ -219,7 +245,7 @@ public class DirectReader {
try {
long offset = (index * 28) >>> 3;
int shift = (int) ((index + 1) & 1) << 2;
- return (in.readInt(offset) >>> shift) & 0xFFFFFFFL;
+ return (in.readInt(this.offset + offset) >>> shift) & 0xFFFFFFFL;
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -228,15 +254,17 @@ public class DirectReader {
static final class DirectPackedReader32 extends LongValues {
final RandomAccessInput in;
+ final long offset;
- DirectPackedReader32(RandomAccessInput in) {
+ DirectPackedReader32(RandomAccessInput in, long offset) {
this.in = in;
+ this.offset = offset;
}
@Override
public long get(long index) {
try {
- return in.readInt(index << 2) & 0xFFFFFFFFL;
+ return in.readInt(this.offset + (index << 2)) & 0xFFFFFFFFL;
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -245,15 +273,17 @@ public class DirectReader {
static final class DirectPackedReader40 extends LongValues {
final RandomAccessInput in;
+ final long offset;
- DirectPackedReader40(RandomAccessInput in) {
+ DirectPackedReader40(RandomAccessInput in, long offset) {
this.in = in;
+ this.offset = offset;
}
@Override
public long get(long index) {
try {
- return in.readLong(index * 5) >>> 24;
+ return in.readLong(this.offset + index * 5) >>> 24;
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -262,15 +292,17 @@ public class DirectReader {
static final class DirectPackedReader48 extends LongValues {
final RandomAccessInput in;
+ final long offset;
- DirectPackedReader48(RandomAccessInput in) {
+ DirectPackedReader48(RandomAccessInput in, long offset) {
this.in = in;
+ this.offset = offset;
}
@Override
public long get(long index) {
try {
- return in.readLong(index * 6) >>> 16;
+ return in.readLong(this.offset + index * 6) >>> 16;
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -279,15 +311,17 @@ public class DirectReader {
static final class DirectPackedReader56 extends LongValues {
final RandomAccessInput in;
+ final long offset;
- DirectPackedReader56(RandomAccessInput in) {
+ DirectPackedReader56(RandomAccessInput in, long offset) {
this.in = in;
+ this.offset = offset;
}
@Override
public long get(long index) {
try {
- return in.readLong(index * 7) >>> 8;
+ return in.readLong(this.offset + index * 7) >>> 8;
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -296,15 +330,17 @@ public class DirectReader {
static final class DirectPackedReader64 extends LongValues {
final RandomAccessInput in;
+ final long offset;
- DirectPackedReader64(RandomAccessInput in) {
+ DirectPackedReader64(RandomAccessInput in, long offset) {
this.in = in;
+ this.offset = offset;
}
@Override
public long get(long index) {
try {
- return in.readLong(index << 3);
+ return in.readLong(offset + (index << 3));
} catch (IOException e) {
throw new RuntimeException(e);
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec Mon Dec 21 11:39:57 2015
@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.codecs.lucene53.Lucene53Codec
+org.apache.lucene.codecs.lucene60.Lucene60Codec
Modified: lucene/dev/branches/lucene6835/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat Mon Dec 21 11:39:57 2015
@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.codecs.lucene50.Lucene50DocValuesFormat
+org.apache.lucene.codecs.lucene54.Lucene54DocValuesFormat
Modified: lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/TestSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/TestSearch.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/TestSearch.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/TestSearch.java Mon Dec 21 11:39:57 2015
@@ -17,18 +17,18 @@ package org.apache.lucene;
* limitations under the License.
*/
+import java.io.PrintWriter;
+import java.io.StringWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.store.*;
-import org.apache.lucene.document.*;
import org.apache.lucene.analysis.*;
+import org.apache.lucene.document.*;
import org.apache.lucene.index.*;
import org.apache.lucene.search.*;
+import org.apache.lucene.store.*;
+import org.apache.lucene.util.LuceneTestCase;
/** JUnit adaptation of an older test case SearchTest. */
public class TestSearch extends LuceneTestCase {
@@ -125,7 +125,6 @@ public class TestSearch extends LuceneTe
for (int j = 0; j < docs.length; j++) {
Document d = new Document();
d.add(newTextField("contents", docs[j], Field.Store.YES));
- d.add(new IntField("id", j, Field.Store.NO));
d.add(new NumericDocValuesField("id", j));
writer.addDocument(d);
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/TestSearchForDuplicates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/TestSearchForDuplicates.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/TestSearchForDuplicates.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/TestSearchForDuplicates.java Mon Dec 21 11:39:57 2015
@@ -26,8 +26,8 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.IntField;
import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
@@ -98,7 +98,7 @@ public class TestSearchForDuplicates ext
for (int j = 0; j < MAX_DOCS; j++) {
Document d = new Document();
d.add(newTextField(PRIORITY_FIELD, HIGH_PRIORITY, Field.Store.YES));
- d.add(new IntField(ID_FIELD, j, Field.Store.YES));
+ d.add(new StoredField(ID_FIELD, j));
d.add(new NumericDocValuesField(ID_FIELD, j));
writer.addDocument(d);
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java Mon Dec 21 11:39:57 2015
@@ -17,8 +17,8 @@ package org.apache.lucene.analysis;
* limitations under the License.
*/
-import org.apache.lucene.util.NumericUtils;
-import org.apache.lucene.analysis.NumericTokenStream.NumericTermAttributeImpl;
+import org.apache.lucene.util.LegacyNumericUtils;
+import org.apache.lucene.analysis.LegacyNumericTokenStream.LegacyNumericTermAttributeImpl;
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.analysis.tokenattributes.TestCharTermAttributeImpl;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
@@ -32,21 +32,21 @@ public class TestNumericTokenStream exte
public void testLongStream() throws Exception {
@SuppressWarnings("resource")
- final NumericTokenStream stream=new NumericTokenStream().setLongValue(lvalue);
+ final LegacyNumericTokenStream stream=new LegacyNumericTokenStream().setLongValue(lvalue);
final TermToBytesRefAttribute bytesAtt = stream.getAttribute(TermToBytesRefAttribute.class);
assertNotNull(bytesAtt);
final TypeAttribute typeAtt = stream.getAttribute(TypeAttribute.class);
assertNotNull(typeAtt);
- final NumericTokenStream.NumericTermAttribute numericAtt = stream.getAttribute(NumericTokenStream.NumericTermAttribute.class);
+ final LegacyNumericTokenStream.LegacyNumericTermAttribute numericAtt = stream.getAttribute(LegacyNumericTokenStream.LegacyNumericTermAttribute.class);
assertNotNull(numericAtt);
stream.reset();
assertEquals(64, numericAtt.getValueSize());
- for (int shift=0; shift<64; shift+=NumericUtils.PRECISION_STEP_DEFAULT) {
+ for (int shift=0; shift<64; shift+= LegacyNumericUtils.PRECISION_STEP_DEFAULT) {
assertTrue("New token is available", stream.incrementToken());
assertEquals("Shift value wrong", shift, numericAtt.getShift());
- assertEquals("Term is incorrectly encoded", lvalue & ~((1L << shift) - 1L), NumericUtils.prefixCodedToLong(bytesAtt.getBytesRef()));
+ assertEquals("Term is incorrectly encoded", lvalue & ~((1L << shift) - 1L), LegacyNumericUtils.prefixCodedToLong(bytesAtt.getBytesRef()));
assertEquals("Term raw value is incorrectly encoded", lvalue & ~((1L << shift) - 1L), numericAtt.getRawValue());
- assertEquals("Type incorrect", (shift == 0) ? NumericTokenStream.TOKEN_TYPE_FULL_PREC : NumericTokenStream.TOKEN_TYPE_LOWER_PREC, typeAtt.type());
+ assertEquals("Type incorrect", (shift == 0) ? LegacyNumericTokenStream.TOKEN_TYPE_FULL_PREC : LegacyNumericTokenStream.TOKEN_TYPE_LOWER_PREC, typeAtt.type());
}
assertFalse("More tokens available", stream.incrementToken());
stream.end();
@@ -55,21 +55,21 @@ public class TestNumericTokenStream exte
public void testIntStream() throws Exception {
@SuppressWarnings("resource")
- final NumericTokenStream stream=new NumericTokenStream().setIntValue(ivalue);
+ final LegacyNumericTokenStream stream=new LegacyNumericTokenStream().setIntValue(ivalue);
final TermToBytesRefAttribute bytesAtt = stream.getAttribute(TermToBytesRefAttribute.class);
assertNotNull(bytesAtt);
final TypeAttribute typeAtt = stream.getAttribute(TypeAttribute.class);
assertNotNull(typeAtt);
- final NumericTokenStream.NumericTermAttribute numericAtt = stream.getAttribute(NumericTokenStream.NumericTermAttribute.class);
+ final LegacyNumericTokenStream.LegacyNumericTermAttribute numericAtt = stream.getAttribute(LegacyNumericTokenStream.LegacyNumericTermAttribute.class);
assertNotNull(numericAtt);
stream.reset();
assertEquals(32, numericAtt.getValueSize());
- for (int shift=0; shift<32; shift+=NumericUtils.PRECISION_STEP_DEFAULT) {
+ for (int shift=0; shift<32; shift+= LegacyNumericUtils.PRECISION_STEP_DEFAULT) {
assertTrue("New token is available", stream.incrementToken());
assertEquals("Shift value wrong", shift, numericAtt.getShift());
- assertEquals("Term is incorrectly encoded", ivalue & ~((1 << shift) - 1), NumericUtils.prefixCodedToInt(bytesAtt.getBytesRef()));
+ assertEquals("Term is incorrectly encoded", ivalue & ~((1 << shift) - 1), LegacyNumericUtils.prefixCodedToInt(bytesAtt.getBytesRef()));
assertEquals("Term raw value is incorrectly encoded", ((long) ivalue) & ~((1L << shift) - 1L), numericAtt.getRawValue());
- assertEquals("Type incorrect", (shift == 0) ? NumericTokenStream.TOKEN_TYPE_FULL_PREC : NumericTokenStream.TOKEN_TYPE_LOWER_PREC, typeAtt.type());
+ assertEquals("Type incorrect", (shift == 0) ? LegacyNumericTokenStream.TOKEN_TYPE_FULL_PREC : LegacyNumericTokenStream.TOKEN_TYPE_LOWER_PREC, typeAtt.type());
}
assertFalse("More tokens available", stream.incrementToken());
stream.end();
@@ -77,7 +77,7 @@ public class TestNumericTokenStream exte
}
public void testNotInitialized() throws Exception {
- final NumericTokenStream stream=new NumericTokenStream();
+ final LegacyNumericTokenStream stream=new LegacyNumericTokenStream();
try {
stream.reset();
@@ -100,28 +100,28 @@ public class TestNumericTokenStream exte
public static class TestAttributeImpl extends CharTermAttributeImpl implements TestAttribute {}
public void testCTA() throws Exception {
- final NumericTokenStream stream=new NumericTokenStream();
+ final LegacyNumericTokenStream stream=new LegacyNumericTokenStream();
try {
stream.addAttribute(CharTermAttribute.class);
fail("Succeeded to add CharTermAttribute.");
} catch (IllegalArgumentException iae) {
- assertTrue(iae.getMessage().startsWith("NumericTokenStream does not support"));
+ assertTrue(iae.getMessage().startsWith("LegacyNumericTokenStream does not support"));
}
try {
stream.addAttribute(TestAttribute.class);
fail("Succeeded to add TestAttribute.");
} catch (IllegalArgumentException iae) {
- assertTrue(iae.getMessage().startsWith("NumericTokenStream does not support"));
+ assertTrue(iae.getMessage().startsWith("LegacyNumericTokenStream does not support"));
}
stream.close();
}
public void testAttributeClone() throws Exception {
- NumericTermAttributeImpl att = new NumericTermAttributeImpl();
+ LegacyNumericTermAttributeImpl att = new LegacyNumericTermAttributeImpl();
att.init(1234L, 64, 8, 0); // set some value, to make getBytesRef() work
- NumericTermAttributeImpl copy = TestCharTermAttributeImpl.assertCloneIsEqual(att);
+ LegacyNumericTermAttributeImpl copy = TestCharTermAttributeImpl.assertCloneIsEqual(att);
assertNotSame(att.getBytesRef(), copy.getBytesRef());
- NumericTermAttributeImpl copy2 = TestCharTermAttributeImpl.assertCopyIsEqual(att);
+ LegacyNumericTermAttributeImpl copy2 = TestCharTermAttributeImpl.assertCopyIsEqual(att);
assertNotSame(att.getBytesRef(), copy2.getBytesRef());
}