You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/07/22 13:47:47 UTC
svn commit: r1149576 [2/3] - in /lucene/dev/branches/positions: ./
dev-tools/idea/lucene/contrib/ lucene/ lucene/contrib/
lucene/contrib/misc/src/java/org/apache/lucene/index/
lucene/contrib/queries/src/test/org/apache/lucene/search/regex/
lucene/contr...
Modified: lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/BooleanQuery.java Fri Jul 22 11:47:38 2011
@@ -18,10 +18,14 @@ package org.apache.lucene.search;
*/
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.util.ToStringUtils;
import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.ConjunctionTermScorer.DocsAndFreqs;
+import org.apache.lucene.search.Similarity.ExactDocScorer;
+import org.apache.lucene.search.TermQuery.TermWeight;
import java.io.IOException;
import java.util.*;
@@ -166,17 +170,24 @@ public class BooleanQuery extends Query
protected ArrayList<Weight> weights;
protected int maxCoord; // num optional + num required
private final boolean disableCoord;
+ private final boolean termConjunction;
public BooleanWeight(IndexSearcher searcher, boolean disableCoord)
throws IOException {
this.similarityProvider = searcher.getSimilarityProvider();
this.disableCoord = disableCoord;
weights = new ArrayList<Weight>(clauses.size());
+ boolean termConjunction = clauses.isEmpty() || minNrShouldMatch != 0 ? false : true;
for (int i = 0 ; i < clauses.size(); i++) {
BooleanClause c = clauses.get(i);
- weights.add(c.getQuery().createWeight(searcher));
+ Weight w = c.getQuery().createWeight(searcher);
+ if (!(c.isRequired() && (w instanceof TermWeight))) {
+ termConjunction = false;
+ }
+ weights.add(w);
if (!c.isProhibited()) maxCoord++;
}
+ this.termConjunction = termConjunction;
}
@Override
@@ -290,6 +301,10 @@ public class BooleanQuery extends Query
@Override
public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext)
throws IOException {
+ if (termConjunction) {
+ // specialized scorer for term conjunctions
+ return createConjunctionTermScorer(context, scorerContext);
+ }
List<Scorer> required = new ArrayList<Scorer>();
List<Scorer> prohibited = new ArrayList<Scorer>();
List<Scorer> optional = new ArrayList<Scorer>();
@@ -328,6 +343,28 @@ public class BooleanQuery extends Query
// Return a BooleanScorer2
return new BooleanScorer2(this, disableCoord, minNrShouldMatch, required, prohibited, optional, maxCoord, scorerContext.needsPositions);
}
+
+ private Scorer createConjunctionTermScorer(AtomicReaderContext context, ScorerContext scorerContext)
+ throws IOException {
+ final DocsAndFreqs[] docsAndFreqs = new DocsAndFreqs[weights.size()];
+ for (int i = 0; i < docsAndFreqs.length; i++) {
+ final TermWeight weight = (TermWeight) weights.get(i);
+ final TermsEnum termsEnum = weight.getTermsEnum(context);
+ if (termsEnum == null) {
+ return null;
+ }
+ final ExactDocScorer docScorer = weight.createDocScorer(context);
+ if (scorerContext.needsPositions) {
+ docsAndFreqs[i] = new DocsAndFreqs(termsEnum.docs(
+ context.reader.getLiveDocs(), null), termsEnum.docFreq(), docScorer, weight.createDocsAndPosEnumFactory(context, scorerContext));
+ } else {
+ docsAndFreqs[i] = new DocsAndFreqs(termsEnum.docs(
+ context.reader.getLiveDocs(), null), termsEnum.docFreq(), docScorer, null);
+ }
+ }
+ return new ConjunctionTermScorer(this, disableCoord ? 1.0f : coord(
+ docsAndFreqs.length, docsAndFreqs.length), docsAndFreqs, scorerContext.needsPositions);
+ }
@Override
public boolean scoresDocsOutOfOrder() {
Copied: lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java (from r1149559, lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java?p2=lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java&r1=1149559&r2=1149576&rev=1149576&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java (original)
+++ lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java Fri Jul 22 11:47:38 2011
@@ -19,6 +19,9 @@ package org.apache.lucene.search;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.search.Similarity.ExactDocScorer;
+import org.apache.lucene.search.TermQuery.DocsAndPositionsEnumFactory;
+import org.apache.lucene.search.positions.ConjunctionPositionIterator;
+import org.apache.lucene.search.positions.PositionIntervalIterator;
import org.apache.lucene.util.ArrayUtil;
import java.io.IOException;
import java.util.Comparator;
@@ -29,12 +32,19 @@ final class ConjunctionTermScorer extend
private int lastDoc = -1;
private final DocsAndFreqs[] docsAndFreqs;
private final DocsAndFreqs lead;
+ private final DocsAndFreqs[] orderedDocsAndFreqs;
ConjunctionTermScorer(Weight weight, float coord,
- DocsAndFreqs[] docsAndFreqs) throws IOException {
+ DocsAndFreqs[] docsAndFreqs, boolean needsPositions) throws IOException {
super(weight);
this.coord = coord;
this.docsAndFreqs = docsAndFreqs;
+ if (needsPositions) {
+ orderedDocsAndFreqs = new DocsAndFreqs[docsAndFreqs.length];
+ System.arraycopy(docsAndFreqs, 0, orderedDocsAndFreqs, 0, orderedDocsAndFreqs.length);
+ } else {
+ orderedDocsAndFreqs = null;
+ }
// Sort the array the first time to allow the least frequent DocsEnum to
// lead the matching.
ArrayUtil.mergeSort(docsAndFreqs, new Comparator<DocsAndFreqs>() {
@@ -95,16 +105,32 @@ final class ConjunctionTermScorer extend
return sum * coord;
}
+ @Override
+ public PositionIntervalIterator positions() throws IOException {
+ if (orderedDocsAndFreqs == null) {
+ throw new IllegalStateException("no positions requested for this scorer");
+ }
+ PositionIntervalIterator[] iterators = new PositionIntervalIterator[orderedDocsAndFreqs.length];
+ for (int i = 0; i < iterators.length; i++) {
+ DocsAndPositionsEnumFactory factory = orderedDocsAndFreqs[i].factory;
+ iterators[i] = new TermScorer.TermPositions(this, factory.create(), factory.doPayloads);
+ }
+ // only created if needed for this scorer - no penalty for non-positional queries
+ return new ConjunctionPositionIterator(this, iterators);
+ }
+
static final class DocsAndFreqs {
final DocsEnum docs;
final int freq;
final ExactDocScorer docScorer;
int doc = -1;
+ final DocsAndPositionsEnumFactory factory;
- DocsAndFreqs(DocsEnum docs, int freq, ExactDocScorer docScorer) {
+ DocsAndFreqs(DocsEnum docs, int freq, ExactDocScorer docScorer, TermQuery.DocsAndPositionsEnumFactory factory) {
this.docs = docs;
this.freq = freq;
this.docScorer = docScorer;
+ this.factory = factory;
}
}
}
Modified: lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/TermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/TermQuery.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/TermQuery.java (original)
+++ lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/TermQuery.java Fri Jul 22 11:47:38 2011
@@ -25,10 +25,12 @@ import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Similarity.ExactDocScorer;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ReaderUtil;
@@ -42,7 +44,7 @@ public class TermQuery extends Query {
private int docFreq;
private transient TermContext perReaderTermState;
- private class TermWeight extends Weight {
+ final class TermWeight extends Weight {
private final Similarity similarity;
private final Similarity.Stats stats;
private transient TermContext termStates;
@@ -73,30 +75,48 @@ public class TermQuery extends Query {
@Override
public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
- final String field = term.field();
- final IndexReader reader = context.reader;
assert termStates.topReaderContext == ReaderUtil.getTopLevelContext(context) : "The top-reader used to create Weight (" + termStates.topReaderContext + ") is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context);
- final TermState state = termStates.get(context.ord);
- if (state == null) { // term is not present in that reader
- assert termNotInReader(reader, field, term.bytes()) : "no termstate found but term exists in reader";
+ final TermsEnum termsEnum = getTermsEnum(context);
+ if (termsEnum == null) {
return null;
}
- final DocsEnum docs = reader.termDocsEnum(reader.getLiveDocs(), field,
- term.bytes(), state);
+ // TODO should we reuse the DocsEnum here?
+ final DocsEnum docs = termsEnum.docs(context.reader.getLiveDocs(), null);
+ assert docs != null;
if (scorerContext.needsPositions) {
- final DocsAndPositionsEnum docsAndPos = reader.termPositionsEnum(
- reader.getLiveDocs(), field, term.bytes(), state);
assert docs != null;
- if (docsAndPos == null) {
- throw new IllegalStateException("field \"" + term.field() + "\" was indexed with Field.omitTermFreqAndPositions=true; cannot run Query (term=" + term.text() + ")");
- }
- return new TermScorer(this, docs, new DocsAndPositionsEnumFactory(reader, term, state, scorerContext.needsPayloads), similarity.exactDocScorer(
- stats, field, context));
+ return new TermScorer(this, docs, createDocsAndPosEnumFactory(context, scorerContext),createDocScorer(context));
} else {
assert docs != null;
- return new TermScorer(this, docs, similarity.exactDocScorer(stats,
- field, context));
+ return new TermScorer(this, docs, createDocScorer(context));
+ }
+
+ }
+
+ /**
+ * Creates an {@link ExactDocScorer} for this {@link TermWeight}*/
+ ExactDocScorer createDocScorer(AtomicReaderContext context)
+ throws IOException {
+ return similarity.exactDocScorer(stats, term.field(), context);
+ }
+
+ DocsAndPositionsEnumFactory createDocsAndPosEnumFactory(AtomicReaderContext context, ScorerContext scorerContext) {
+ return new DocsAndPositionsEnumFactory(context.reader, term, termStates.get(context.ord), scorerContext.needsPayloads);
+ }
+ /**
+ * Returns a {@link TermsEnum} positioned at this weights Term or null if
+ * the term does not exist in the given context
+ */
+ TermsEnum getTermsEnum(AtomicReaderContext context) throws IOException {
+ final TermState state = termStates.get(context.ord);
+ if (state == null) { // term is not present in that reader
+ assert termNotInReader(context.reader, term.field(), term.bytes()) : "no termstate found but term exists in reader";
+ return null;
}
+ final TermsEnum termsEnum = context.reader.terms(term.field())
+ .getThreadTermsEnum();
+ termsEnum.seekExact(term.bytes(), state);
+ return termsEnum;
}
private boolean termNotInReader(IndexReader reader, String field, BytesRef bytes) throws IOException {
@@ -223,11 +243,8 @@ public class TermQuery extends Query {
}
DocsAndPositionsEnum create() throws IOException {
- DocsAndPositionsEnum termPositionsEnum = reader.termPositionsEnum(reader.getLiveDocs(), term.field(),
+ return reader.termPositionsEnum(reader.getLiveDocs(), term.field(),
term.bytes(), state);
- return termPositionsEnum;
-// return reader.termPositionsEnum(reader.getLiveDocs(), term.field(),
-// term.bytes(), state);
}
}
}
Modified: lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/TermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/TermScorer.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/TermScorer.java (original)
+++ lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/TermScorer.java Fri Jul 22 11:47:38 2011
@@ -174,17 +174,17 @@ final class TermScorer extends Scorer {
@Override
public PositionIntervalIterator positions() throws IOException {
assert docsAndPosFactory != null;
- return new TermPositions(docsAndPosFactory.create(), docsAndPosFactory.doPayloads);
+ return new TermPositions(this, docsAndPosFactory.create(), docsAndPosFactory.doPayloads);
}
- private final class TermPositions extends PositionIntervalIterator {
+ static final class TermPositions extends PositionIntervalIterator {
private final PositionInterval interval;
int positionsPending;
private final DocsAndPositionsEnum docsAndPos;
private int docID = -1;
- public TermPositions(DocsAndPositionsEnum docsAndPos, boolean doPayloads) {
- super(TermScorer.this);
+ public TermPositions(Scorer scorer, DocsAndPositionsEnum docsAndPos, boolean doPayloads) {
+ super(scorer);
this.docsAndPos = docsAndPos;
this.interval = doPayloads ? new PayloadPosInterval(docsAndPos, this)
: new PositionInterval();
@@ -219,7 +219,7 @@ final class TermScorer extends Scorer {
public int advanceTo(int docId) throws IOException {
int advance = docsAndPos.advance(docId);
if (advance != NO_MORE_DOCS) {
- positionsPending = freq = docsAndPos.freq();
+ positionsPending = docsAndPos.freq();
}
interval.reset();
return docID = docsAndPos.docID();
Modified: lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/TopScoreDocCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/TopScoreDocCollector.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/TopScoreDocCollector.java (original)
+++ lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/TopScoreDocCollector.java Fri Jul 22 11:47:38 2011
@@ -82,8 +82,13 @@ public abstract class TopScoreDocCollect
assert !Float.isNaN(score);
totalHits++;
+ if (score < pqTop.score) {
+ // Doesn't compete w/ bottom entry in queue
+ return;
+ }
doc += docBase;
- if (score < pqTop.score || (score == pqTop.score && doc > pqTop.doc)) {
+ if (score == pqTop.score && doc > pqTop.doc) {
+ // Break tie in score by doc ID:
return;
}
pqTop.doc = doc;
Modified: lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/positions/BooleanPositionIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/positions/BooleanPositionIterator.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/positions/BooleanPositionIterator.java (original)
+++ lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/positions/BooleanPositionIterator.java Fri Jul 22 11:47:38 2011
@@ -39,6 +39,13 @@ abstract class BooleanPositionIterator e
iterators[i] = subScorers[i].positions();
}
}
+
+ public BooleanPositionIterator(Scorer scorer, PositionIntervalIterator[] iterators,
+ IntervalQueue queue) throws IOException {
+ super(scorer);
+ this.queue = queue;
+ this.iterators = iterators;
+ }
@Override
public PositionIntervalIterator[] subs(boolean inOrder) {
Modified: lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/positions/ConjunctionPositionIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/positions/ConjunctionPositionIterator.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/positions/ConjunctionPositionIterator.java (original)
+++ lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/search/positions/ConjunctionPositionIterator.java Fri Jul 22 11:47:38 2011
@@ -37,6 +37,13 @@ public final class ConjunctionPositionIt
this (scorer, subScorers, subScorers.length);
}
+ public ConjunctionPositionIterator(Scorer scorer, PositionIntervalIterator... iterators) throws IOException {
+ super(scorer, iterators, new IntervalQueueAnd(iterators.length));
+ queue = (IntervalQueueAnd) super.queue; // avoid lots of casts?
+ this.nrMustMatch = iterators.length;
+
+ }
+
public ConjunctionPositionIterator(Scorer scorer, Scorer[] subScorers, int nrMustMatch) throws IOException {
super(scorer, subScorers, new IntervalQueueAnd(subScorers.length));
queue = (IntervalQueueAnd) super.queue; // avoid lots of casts?
Modified: lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/util/BitVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/util/BitVector.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/util/BitVector.java (original)
+++ lucene/dev/branches/positions/lucene/src/java/org/apache/lucene/util/BitVector.java Fri Jul 22 11:47:38 2011
@@ -353,6 +353,11 @@ public final class BitVector implements
} else {
readBits(input);
}
+
+ if (version < VERSION_DGAPS_CLEARED) {
+ invertAll();
+ }
+
assert verifyCount();
} finally {
input.close();
Modified: lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java Fri Jul 22 11:47:38 2011
@@ -272,14 +272,18 @@ public class MockDirectoryWrapper extend
}
void maybeThrowIOException() throws IOException {
+ maybeThrowIOException(null);
+ }
+
+ void maybeThrowIOException(String message) throws IOException {
if (randomIOExceptionRate > 0.0) {
int number = Math.abs(randomState.nextInt() % 1000);
if (number < randomIOExceptionRate*1000) {
if (LuceneTestCase.VERBOSE) {
- System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception");
+ System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception" + (message == null ? "" : " (" + message + ")"));
new Throwable().printStackTrace(System.out);
}
- throw new IOException("a random IOException");
+ throw new IOException("a random IOException" + (message == null ? "" : "(" + message + ")"));
}
}
}
Modified: lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java (original)
+++ lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java Fri Jul 22 11:47:38 2011
@@ -147,4 +147,10 @@ public class MockIndexInputWrapper exten
public long readVLong() throws IOException {
return delegate.readVLong();
}
+
+ @Override
+ public String toString() {
+ return "MockIndexInputWrapper(" + delegate + ")";
+ }
}
+
Modified: lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java (original)
+++ lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java Fri Jul 22 11:47:38 2011
@@ -126,7 +126,7 @@ public class MockIndexOutputWrapper exte
// Maybe throw random exception; only do this on first
// write to a new file:
first = false;
- dir.maybeThrowIOException();
+ dir.maybeThrowIOException(name);
}
}
@@ -156,4 +156,9 @@ public class MockIndexOutputWrapper exte
// TODO: we may need to check disk full here as well
dir.maybeThrowDeterministicException();
}
+
+ @Override
+ public String toString() {
+ return "MockIndexOutputWrapper(" + delegate + ")";
+ }
}
Modified: lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java Fri Jul 22 11:47:38 2011
@@ -754,7 +754,7 @@ public abstract class LuceneTestCase ext
* is active and {@link #RANDOM_MULTIPLIER}, but also with some random fudge.
*/
public static int atLeast(Random random, int i) {
- int min = (TEST_NIGHTLY ? 5*i : i) * RANDOM_MULTIPLIER;
+ int min = (TEST_NIGHTLY ? 3*i : i) * RANDOM_MULTIPLIER;
int max = min+(min/2);
return _TestUtil.nextInt(random, min, max);
}
@@ -770,9 +770,9 @@ public abstract class LuceneTestCase ext
* is active and {@link #RANDOM_MULTIPLIER}.
*/
public static boolean rarely(Random random) {
- int p = TEST_NIGHTLY ? 25 : 5;
+ int p = TEST_NIGHTLY ? 10 : 5;
p += (p * Math.log(RANDOM_MULTIPLIER));
- int min = 100 - Math.min(p, 90); // never more than 90
+ int min = 100 - Math.min(p, 50); // never more than 50
return random.nextInt(100) >= min;
}
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/TestExternalCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/TestExternalCodecs.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/TestExternalCodecs.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/TestExternalCodecs.java Fri Jul 22 11:47:38 2011
@@ -92,10 +92,6 @@ public class TestExternalCodecs extends
@Override
public void close() {
}
-
- @Override
- public void loadTermsIndex(int indexDivisor) {
- }
}
static class RAMField extends Terms {
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/Test2BPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/Test2BPostings.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/Test2BPostings.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/Test2BPostings.java Fri Jul 22 11:47:38 2011
@@ -70,14 +70,16 @@ public class Test2BPostings extends Luce
final int numDocs = (Integer.MAX_VALUE / 26) + 1;
for (int i = 0; i < numDocs; i++) {
w.addDocument(doc);
- if (i % 100000 == 0) {
+ if (VERBOSE && i % 100000 == 0) {
System.out.println(i + " of " + numDocs + "...");
}
}
w.optimize();
w.close();
CheckIndex ci = new CheckIndex(dir);
- ci.setInfoStream(System.out);
+ if (VERBOSE) {
+ ci.setInfoStream(System.out);
+ }
ci.checkIndex();
dir.close();
}
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Fri Jul 22 11:47:38 2011
@@ -551,7 +551,9 @@ public class TestIndexWriterDelete exten
if (!success) {
// Must force the close else the writer can have
// open files which cause exc in MockRAMDir.close
-
+ if (VERBOSE) {
+ System.out.println("TEST: now rollback");
+ }
modifier.rollback();
}
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java Fri Jul 22 11:47:38 2011
@@ -90,6 +90,7 @@ public class TestRollingUpdates extends
for (int r = 0; r < 3; r++) {
final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
+ w.setInfoStream(VERBOSE ? System.out : null);
final int numUpdates = atLeast(20);
int numThreads = _TestUtil.nextInt(random, 2, 6);
IndexingThread[] threads = new IndexingThread[numThreads];
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestStressAdvance.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestStressAdvance.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestStressAdvance.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestStressAdvance.java Fri Jul 22 11:47:38 2011
@@ -40,7 +40,7 @@ public class TestStressAdvance extends L
doc.add(f);
final Field idField = newField("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS);
doc.add(idField);
- int num = atLeast(5000);
+ int num = atLeast(4097);
for(int id=0;id<num;id++) {
if (random.nextInt(4) == 3) {
f.setValue("a");
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java Fri Jul 22 11:47:38 2011
@@ -56,7 +56,7 @@ public class TestNumericRangeQuery32 ext
directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random, directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
- .setMaxBufferedDocs(_TestUtil.nextInt(random, 50, 1000))
+ .setMaxBufferedDocs(_TestUtil.nextInt(random, 100, 1000))
.setMergePolicy(newLogMergePolicy()));
NumericField
@@ -337,7 +337,7 @@ public class TestNumericRangeQuery32 ext
private void testRandomTrieAndClassicRangeQuery(int precisionStep) throws Exception {
String field="field"+precisionStep;
int termCountT=0,termCountC=0;
- int num = atLeast(10);
+ int num = _TestUtil.nextInt(random, 10, 20);
for (int i = 0; i < num; i++) {
int lower=(int)(random.nextDouble()*noDocs*distance)+startOffset;
int upper=(int)(random.nextDouble()*noDocs*distance)+startOffset;
@@ -415,7 +415,7 @@ public class TestNumericRangeQuery32 ext
private void testRangeSplit(int precisionStep) throws Exception {
String field="ascfield"+precisionStep;
// 10 random tests
- int num = atLeast(10);
+ int num = _TestUtil.nextInt(random, 10, 20);
for (int i =0; i< num; i++) {
int lower=(int)(random.nextDouble()*noDocs - noDocs/2);
int upper=(int)(random.nextDouble()*noDocs - noDocs/2);
@@ -491,7 +491,7 @@ public class TestNumericRangeQuery32 ext
String field="field"+precisionStep;
// 10 random tests, the index order is ascending,
// so using a reverse sort field should retun descending documents
- int num = atLeast(10);
+ int num = _TestUtil.nextInt(random, 10, 20);
for (int i = 0; i < num; i++) {
int lower=(int)(random.nextDouble()*noDocs*distance)+startOffset;
int upper=(int)(random.nextDouble()*noDocs*distance)+startOffset;
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java Fri Jul 22 11:47:38 2011
@@ -53,7 +53,7 @@ public class TestNumericRangeQuery64 ext
directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random, directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
- .setMaxBufferedDocs(_TestUtil.nextInt(random, 50, 1000))
+ .setMaxBufferedDocs(_TestUtil.nextInt(random, 100, 1000))
.setMergePolicy(newLogMergePolicy()));
NumericField
@@ -354,7 +354,7 @@ public class TestNumericRangeQuery64 ext
private void testRandomTrieAndClassicRangeQuery(int precisionStep) throws Exception {
String field="field"+precisionStep;
int termCountT=0,termCountC=0;
- int num = atLeast(10);
+ int num = _TestUtil.nextInt(random, 10, 20);
for (int i = 0; i < num; i++) {
long lower=(long)(random.nextDouble()*noDocs*distance)+startOffset;
long upper=(long)(random.nextDouble()*noDocs*distance)+startOffset;
@@ -437,7 +437,7 @@ public class TestNumericRangeQuery64 ext
private void testRangeSplit(int precisionStep) throws Exception {
String field="ascfield"+precisionStep;
// 10 random tests
- int num = atLeast(10);
+ int num = _TestUtil.nextInt(random, 10, 20);
for (int i = 0; i < num; i++) {
long lower=(long)(random.nextDouble()*noDocs - noDocs/2);
long upper=(long)(random.nextDouble()*noDocs - noDocs/2);
@@ -523,7 +523,7 @@ public class TestNumericRangeQuery64 ext
String field="field"+precisionStep;
// 10 random tests, the index order is ascending,
// so using a reverse sort field should retun descending documents
- int num = atLeast(10);
+ int num = _TestUtil.nextInt(random, 10, 20);
for (int i = 0; i < num; i++) {
long lower=(long)(random.nextDouble()*noDocs*distance)+startOffset;
long upper=(long)(random.nextDouble()*noDocs*distance)+startOffset;
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java Fri Jul 22 11:47:38 2011
@@ -59,7 +59,7 @@ public class TestPrefixRandom extends Lu
// we generate aweful prefixes: good for testing.
// but for preflex codec, the test can be very slow, so use less iterations.
final String codec = CodecProvider.getDefault().getFieldCodec("field");
- int num = codec.equals("PreFlex") ? 200 * RANDOM_MULTIPLIER : atLeast(2000);
+ int num = codec.equals("PreFlex") ? 200 * RANDOM_MULTIPLIER : atLeast(1000);
for (int i = 0; i < num; i++) {
field.setValue(_TestUtil.randomUnicodeString(random, 10));
writer.addDocument(doc);
@@ -114,7 +114,7 @@ public class TestPrefixRandom extends Lu
/** test a bunch of random prefixes */
public void testPrefixes() throws Exception {
- int num = atLeast(1000);
+ int num = atLeast(100);
for (int i = 0; i < num; i++)
assertSame(_TestUtil.randomUnicodeString(random, 5));
}
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java Fri Jul 22 11:47:38 2011
@@ -830,7 +830,7 @@ public class TestFSTs extends LuceneTest
final IntsRef prefix = ent.getKey();
final CountMinOutput<T> cmo = ent.getValue();
if (VERBOSE) {
- System.out.println(" term=" + inputToString(inputMode, prefix) + " count=" + cmo.count + " isLeaf=" + cmo.isLeaf + " output=" + outputs.outputToString(cmo.output) + " isFinal=" + cmo.isFinal);
+ System.out.println(" term prefix=" + inputToString(inputMode, prefix, false) + " count=" + cmo.count + " isLeaf=" + cmo.isLeaf + " output=" + outputs.outputToString(cmo.output) + " isFinal=" + cmo.isFinal);
}
final boolean keep;
if (prune1 > 0) {
@@ -897,7 +897,7 @@ public class TestFSTs extends LuceneTest
IntsRefFSTEnum.InputOutput<T> current;
while((current = fstEnum.next()) != null) {
if (VERBOSE) {
- System.out.println(" fstEnum.next term=" + inputToString(inputMode, current.input) + " output=" + outputs.outputToString(current.output));
+ System.out.println(" fstEnum.next prefix=" + inputToString(inputMode, current.input, false) + " output=" + outputs.outputToString(current.output));
}
final CountMinOutput cmo = prefixes.get(current.input);
assertNotNull(cmo);
@@ -920,7 +920,7 @@ public class TestFSTs extends LuceneTest
final CountMinOutput<T> cmo = ent.getValue();
final T output = run(fst, ent.getKey(), stopNode);
if (VERBOSE) {
- System.out.println("TEST: verify term=" + inputToString(inputMode, ent.getKey()) + " output=" + outputs.outputToString(cmo.output));
+ System.out.println("TEST: verify prefix=" + inputToString(inputMode, ent.getKey(), false) + " output=" + outputs.outputToString(cmo.output));
}
// if (cmo.isFinal && !cmo.isLeaf) {
if (cmo.isFinal) {
@@ -980,11 +980,17 @@ public class TestFSTs extends LuceneTest
@Nightly
public void testBigSet() throws IOException {
- testRandomWords(_TestUtil.nextInt(random, 50000, 60000), atLeast(1));
+ testRandomWords(_TestUtil.nextInt(random, 50000, 60000), 1);
}
-
+
private static String inputToString(int inputMode, IntsRef term) {
- if (inputMode == 0) {
+ return inputToString(inputMode, term, true);
+ }
+
+ private static String inputToString(int inputMode, IntsRef term, boolean isValidUnicode) {
+ if (!isValidUnicode) {
+ return term.toString();
+ } else if (inputMode == 0) {
// utf8
return toBytesRef(term).utf8ToString() + " " + term;
} else {
@@ -1007,7 +1013,7 @@ public class TestFSTs extends LuceneTest
final int RUN_TIME_MSEC = atLeast(500);
final IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(-1).setRAMBufferSizeMB(64);
final File tempDir = _TestUtil.getTempDir("fstlines");
- final MockDirectoryWrapper dir = new MockDirectoryWrapper(random, FSDirectory.open(tempDir));
+ final MockDirectoryWrapper dir = newFSDirectory(tempDir);
final IndexWriter writer = new IndexWriter(dir, conf);
writer.setInfoStream(VERBOSE ? System.out : null);
final long stopTime = System.currentTimeMillis() + RUN_TIME_MSEC;
@@ -1057,7 +1063,7 @@ public class TestFSTs extends LuceneTest
}
builder.add(term, outputs.get(output));
ord++;
- if (ord % 100000 == 0 && LuceneTestCase.TEST_NIGHTLY) {
+ if (VERBOSE && ord % 100000 == 0 && LuceneTestCase.TEST_NIGHTLY) {
System.out.println(ord + " terms...");
}
}
Modified: lucene/dev/branches/positions/modules/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/modules/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/modules/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java (original)
+++ lucene/dev/branches/positions/modules/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java Fri Jul 22 11:47:38 2011
@@ -70,8 +70,7 @@ import java.util.*;
* <h3>Initial Usage</h3>
* <p/>
* This class has lots of options to try to make it efficient and flexible.
- * See the body of {@link #main main()} below in the source for real code, or
- * if you want pseudo code, the simplest possible usage is as follows. The bold
+ * The simplest possible usage is as follows. The bold
* fragment is specific to this class.
* <p/>
* <pre class="prettyprint">
@@ -574,45 +573,12 @@ public final class MoreLikeThis {
}
/**
- * Return a query that will return docs like the passed file.
- *
- * @return a query that will return docs like the passed file.
- */
- public Query like(File f) throws IOException {
- if (fieldNames == null) {
- // gather list of valid fields from lucene
- Collection<String> fields = ir.getFieldNames(IndexReader.FieldOption.INDEXED);
- fieldNames = fields.toArray(new String[fields.size()]);
- }
-
- return like(new FileReader(f));
- }
-
- /**
- * Return a query that will return docs like the passed URL.
- *
- * @return a query that will return docs like the passed URL.
- */
- public Query like(URL u) throws IOException {
- return like(new InputStreamReader(u.openConnection().getInputStream()));
- }
-
- /**
- * Return a query that will return docs like the passed stream.
- *
- * @return a query that will return docs like the passed stream.
- */
- public Query like(java.io.InputStream is) throws IOException {
- return like(new InputStreamReader(is));
- }
-
- /**
* Return a query that will return docs like the passed Reader.
*
* @return a query that will return docs like the passed Reader.
*/
- public Query like(Reader r) throws IOException {
- return createQuery(retrieveTerms(r));
+ public Query like(Reader r, String fieldName) throws IOException {
+ return createQuery(retrieveTerms(r, fieldName));
}
/**
@@ -727,65 +693,6 @@ public final class MoreLikeThis {
}
/**
- * Test driver.
- * Pass in "-i INDEX" and then either "-fn FILE" or "-url URL".
- */
- public static void main(String[] a) throws Throwable {
- String indexName = "localhost_index";
- String fn = "c:/Program Files/Apache Group/Apache/htdocs/manual/vhosts/index.html.en";
- URL url = null;
- for (int i = 0; i < a.length; i++) {
- if (a[i].equals("-i")) {
- indexName = a[++i];
- } else if (a[i].equals("-f")) {
- fn = a[++i];
- } else if (a[i].equals("-url")) {
- url = new URL(a[++i]);
- }
- }
-
- PrintStream o = System.out;
- FSDirectory dir = FSDirectory.open(new File(indexName));
- IndexReader r = IndexReader.open(dir, true);
- o.println("Open index " + indexName + " which has " + r.numDocs() + " docs");
-
- MoreLikeThis mlt = new MoreLikeThis(r);
-
- o.println("Query generation parameters:");
- o.println(mlt.describeParams());
- o.println();
-
- Query query = null;
- if (url != null) {
- o.println("Parsing URL: " + url);
- query = mlt.like(url);
- } else if (fn != null) {
- o.println("Parsing file: " + fn);
- query = mlt.like(new File(fn));
- }
-
- o.println("q: " + query);
- o.println();
- IndexSearcher searcher = new IndexSearcher(dir, true);
-
- TopDocs hits = searcher.search(query, null, 25);
- int len = hits.totalHits;
- o.println("found: " + len + " documents matching");
- o.println();
- ScoreDoc[] scoreDocs = hits.scoreDocs;
- for (int i = 0; i < Math.min(25, len); i++) {
- Document d = searcher.doc(scoreDocs[i].doc);
- String summary = d.get("summary");
- o.println("score : " + scoreDocs[i].score);
- o.println("url : " + d.get("url"));
- o.println("\ttitle : " + d.get("title"));
- if (summary != null)
- o.println("\tsummary: " + d.get("summary"));
- o.println();
- }
- }
-
- /**
* Find words for a more-like-this query former.
*
* @param docNum the id of the lucene document from which to find terms
@@ -918,19 +825,18 @@ public final class MoreLikeThis {
* For an easier method to call see {@link #retrieveInterestingTerms retrieveInterestingTerms()}.
*
* @param r the reader that has the content of the document
+ * @param fieldName field passed to the analyzer to use when analyzing the content
* @return the most interesting words in the document ordered by score, with the highest scoring, or best entry, first
* @see #retrieveInterestingTerms
*/
- public PriorityQueue<Object[]> retrieveTerms(Reader r) throws IOException {
+ public PriorityQueue<Object[]> retrieveTerms(Reader r, String fieldName) throws IOException {
Map<String, Int> words = new HashMap<String, Int>();
- for (String fieldName : fieldNames) {
- addTermFrequencies(r, words, fieldName);
- }
+ addTermFrequencies(r, words, fieldName);
return createQueue(words);
}
/**
- * @see #retrieveInterestingTerms(java.io.Reader)
+ * @see #retrieveInterestingTerms(java.io.Reader, String)
*/
public String[] retrieveInterestingTerms(int docNum) throws IOException {
ArrayList<Object> al = new ArrayList<Object>(maxQueryTerms);
@@ -948,16 +854,17 @@ public final class MoreLikeThis {
/**
* Convenience routine to make it easy to return the most interesting words in a document.
- * More advanced users will call {@link #retrieveTerms(java.io.Reader) retrieveTerms()} directly.
+ * More advanced users will call {@link #retrieveTerms(Reader, String) retrieveTerms()} directly.
*
* @param r the source document
+ * @param fieldName field passed to analyzer to use when analyzing the content
* @return the most interesting words in the document
- * @see #retrieveTerms(java.io.Reader)
+ * @see #retrieveTerms(java.io.Reader, String)
* @see #setMaxQueryTerms
*/
- public String[] retrieveInterestingTerms(Reader r) throws IOException {
+ public String[] retrieveInterestingTerms(Reader r, String fieldName) throws IOException {
ArrayList<Object> al = new ArrayList<Object>(maxQueryTerms);
- PriorityQueue<Object[]> pq = retrieveTerms(r);
+ PriorityQueue<Object[]> pq = retrieveTerms(r, fieldName);
Object cur;
int lim = maxQueryTerms; // have to be careful, retrieveTerms returns all words but that's probably not useful to our caller...
// we just want to return the top words
Modified: lucene/dev/branches/positions/modules/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/modules/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisQuery.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/modules/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisQuery.java (original)
+++ lucene/dev/branches/positions/modules/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisQuery.java Fri Jul 22 11:47:38 2011
@@ -28,6 +28,7 @@ import org.apache.lucene.search.Query;
import java.io.ByteArrayInputStream;
import java.io.IOException;
+import java.io.StringReader;
import java.util.Set;
/**
@@ -40,6 +41,7 @@ public class MoreLikeThisQuery extends Q
private String likeText;
private String[] moreLikeFields;
private Analyzer analyzer;
+ private String fieldName;
private float percentTermsToMatch = 0.3f;
private int minTermFrequency = 1;
private int maxQueryTerms = 5;
@@ -49,10 +51,11 @@ public class MoreLikeThisQuery extends Q
/**
* @param moreLikeFields
*/
- public MoreLikeThisQuery(String likeText, String[] moreLikeFields, Analyzer analyzer) {
+ public MoreLikeThisQuery(String likeText, String[] moreLikeFields, Analyzer analyzer, String fieldName) {
this.likeText = likeText;
this.moreLikeFields = moreLikeFields;
this.analyzer = analyzer;
+ this.fieldName = fieldName;
}
@Override
@@ -67,7 +70,7 @@ public class MoreLikeThisQuery extends Q
}
mlt.setMaxQueryTerms(maxQueryTerms);
mlt.setStopWords(stopWords);
- BooleanQuery bq = (BooleanQuery) mlt.like(new ByteArrayInputStream(likeText.getBytes()));
+ BooleanQuery bq = (BooleanQuery) mlt.like(new StringReader(likeText), fieldName);
BooleanClause[] clauses = bq.getClauses();
//make at least half the terms match
bq.setMinimumNumberShouldMatch((int) (clauses.length * percentTermsToMatch));
Modified: lucene/dev/branches/positions/modules/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/modules/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/modules/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java (original)
+++ lucene/dev/branches/positions/modules/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java Fri Jul 22 11:47:38 2011
@@ -87,7 +87,7 @@ public class TestMoreLikeThis extends Lu
mlt.setBoostFactor(boostFactor);
BooleanQuery query = (BooleanQuery) mlt.like(new StringReader(
- "lucene release"));
+ "lucene release"), "text");
List<BooleanClause> clauses = query.clauses();
assertEquals("Expected " + originalValues.size() + " clauses.",
@@ -115,7 +115,7 @@ public class TestMoreLikeThis extends Lu
mlt.setFieldNames(new String[] {"text"});
mlt.setBoost(true);
BooleanQuery query = (BooleanQuery) mlt.like(new StringReader(
- "lucene release"));
+ "lucene release"), "text");
List<BooleanClause> clauses = query.clauses();
for (BooleanClause clause : clauses) {
@@ -124,4 +124,15 @@ public class TestMoreLikeThis extends Lu
}
return originalValues;
}
+
+ // LUCENE-3326
+ public void testMultiFields() throws Exception {
+ MoreLikeThis mlt = new MoreLikeThis(reader);
+ mlt.setAnalyzer(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
+ mlt.setMinDocFreq(1);
+ mlt.setMinTermFreq(1);
+ mlt.setMinWordLen(1);
+ mlt.setFieldNames(new String[] {"text", "foobar"});
+ mlt.like(new StringReader("this is a test"), "foobar");
+ }
}
Modified: lucene/dev/branches/positions/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/solr/CHANGES.txt?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/solr/CHANGES.txt (original)
+++ lucene/dev/branches/positions/solr/CHANGES.txt Fri Jul 22 11:47:38 2011
@@ -154,6 +154,10 @@ New Features
for faster reopen times. There is also a new 'soft' autocommit tracker that can be
configured. (Mark Miller, Robert Muir)
+* SOLR-2399: Updated Solr Admin interface. New look and feel with per core administration
+ and many new options. (Stefan Matheis via ryan)
+
+
Optimizations
----------------------
@@ -264,6 +268,9 @@ Other Changes
* SOLR-1825: SolrQuery.addFacetQuery now enables facets automatically, like
addFacetField (Chris Male)
+* SOLR-2663: FieldTypePluginLoader has been refactored out of IndexSchema
+ and made public. (hossman)
+
Documentation
----------------------
@@ -310,6 +317,9 @@ New Features
* LUCENE-2048: Added omitPositions to the schema, so you can omit position
information while still indexing term frequencies. (rmuir)
+* SOLR-2584: add UniqFieldsUpdateProcessor that removes duplicate values in the
+ specified fields. (Elmer Garduno, koji)
+
Optimizations
----------------------
Modified: lucene/dev/branches/positions/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java (original)
+++ lucene/dev/branches/positions/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java Fri Jul 22 11:47:38 2011
@@ -188,7 +188,7 @@ public class TestMailEntityProcessor ext
Boolean commitCalled;
public SolrWriterImpl() {
- super(null, ".", null);
+ super(null, null);
}
@Override
@@ -196,10 +196,6 @@ public class TestMailEntityProcessor ext
return docs.add(doc);
}
- @Override
- public void log(int event, String name, Object row) {
- // Do nothing
- }
@Override
public void doDeleteAll() {
Modified: lucene/dev/branches/positions/solr/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/solr/contrib/dataimporthandler/CHANGES.txt?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/solr/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/dev/branches/positions/solr/contrib/dataimporthandler/CHANGES.txt Fri Jul 22 11:47:38 2011
@@ -19,6 +19,7 @@ Bug Fixes
* SOLR-2644: When using threads=2 the default logging is set too high (Bill Bell via shalin)
* SOLR-2492: DIH does not commit if only deletes are processed (James Dyer via shalin)
* SOLR-2186: DataImportHandler's multi-threaded option throws NPE (Lance Norskog, Frank Wesemann, shalin)
+* SOLR-2655: DIH multi threaded mode does not resolve attributes correctly (Frank Wesemann, shalin)
================== 3.3.0 ==================
Modified: lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java (original)
+++ lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java Fri Jul 22 11:47:38 2011
@@ -100,7 +100,7 @@ public class ContextImpl extends Context
if (entity.dataSrc != null && docBuilder != null && docBuilder.verboseDebug &&
Context.FULL_DUMP.equals(currentProcess())) {
//debug is not yet implemented properly for deltas
- entity.dataSrc = docBuilder.writer.getDebugLogger().wrapDs(entity.dataSrc);
+ entity.dataSrc = docBuilder.getDebugLogger().wrapDs(entity.dataSrc);
}
return entity.dataSrc;
}
Modified: lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java (original)
+++ lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java Fri Jul 22 11:47:38 2011
@@ -113,7 +113,7 @@ public class DataImportHandler extends R
final InputSource is = new InputSource(core.getResourceLoader().openConfig(configLoc));
is.setSystemId(SystemIdResolver.createSystemIdFromResourceName(configLoc));
importer = new DataImporter(is, core,
- dataSources, coreScopeSession);
+ dataSources, coreScopeSession, myName);
}
}
} catch (Throwable e) {
@@ -165,7 +165,7 @@ public class DataImportHandler extends R
try {
processConfiguration((NamedList) initArgs.get("defaults"));
importer = new DataImporter(new InputSource(new StringReader(requestParams.dataConfig)), req.getCore()
- , dataSources, coreScopeSession);
+ , dataSources, coreScopeSession, myName);
} catch (RuntimeException e) {
rsp.add("exception", DebugLogger.getStacktraceString(e));
importer = null;
@@ -280,7 +280,7 @@ public class DataImportHandler extends R
private SolrWriter getSolrWriter(final UpdateRequestProcessor processor,
final SolrResourceLoader loader, final DataImporter.RequestParams requestParams, SolrQueryRequest req) {
- return new SolrWriter(processor, loader.getConfigDir(), myName, req) {
+ return new SolrWriter(processor, req) {
@Override
public boolean upload(SolrInputDocument document) {
Modified: lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java (original)
+++ lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java Fri Jul 22 11:47:38 2011
@@ -39,7 +39,6 @@ import org.apache.commons.io.IOUtils;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
-import java.io.File;
import java.io.StringReader;
import java.text.SimpleDateFormat;
import java.util.*;
@@ -80,26 +79,35 @@ public class DataImporter {
public DocBuilder.Statistics cumulativeStatistics = new DocBuilder.Statistics();
private SolrCore core;
+
+ private DIHPropertiesWriter propWriter;
private ReentrantLock importLock = new ReentrantLock();
private final Map<String , Object> coreScopeSession;
private boolean isDeltaImportSupported = false;
+ private final String handlerName;
/**
* Only for testing purposes
*/
DataImporter() {
coreScopeSession = new ConcurrentHashMap<String, Object>();
+ this.propWriter = new SimplePropertiesWriter();
+ propWriter.init(this);
+ this.handlerName = "dataimport" ;
}
- DataImporter(InputSource dataConfig, SolrCore core, Map<String, Properties> ds, Map<String, Object> session) {
+ DataImporter(InputSource dataConfig, SolrCore core, Map<String, Properties> ds, Map<String, Object> session, String handlerName) {
+ this.handlerName = handlerName;
if (dataConfig == null)
throw new DataImportHandlerException(SEVERE,
"Configuration not found");
this.core = core;
this.schema = core.getSchema();
+ this.propWriter = new SimplePropertiesWriter();
+ propWriter.init(this);
dataSourceProps = ds;
if (session == null)
session = new HashMap<String, Object>();
@@ -120,7 +128,11 @@ public class DataImporter {
}
}
- private void verifyWithSchema(Map<String, DataConfig.Field> fields) {
+ public String getHandlerName() {
+ return handlerName;
+ }
+
+ private void verifyWithSchema(Map<String, DataConfig.Field> fields) {
Map<String, SchemaField> schemaFields = schema.getFields();
for (Map.Entry<String, SchemaField> entry : schemaFields.entrySet()) {
SchemaField sf = entry.getValue();
@@ -353,7 +365,7 @@ public class DataImporter {
setIndexStartTime(new Date());
try {
- docBuilder = new DocBuilder(this, writer, requestParams);
+ docBuilder = new DocBuilder(this, writer, propWriter, requestParams);
checkWritablePersistFile(writer);
docBuilder.execute();
if (!requestParams.debug)
@@ -370,11 +382,11 @@ public class DataImporter {
}
private void checkWritablePersistFile(SolrWriter writer) {
- File persistFile = writer.getPersistFile();
- boolean isWritable = persistFile.exists() ? persistFile.canWrite() : persistFile.getParentFile().canWrite();
- if (isDeltaImportSupported && !isWritable) {
- throw new DataImportHandlerException(SEVERE, persistFile.getAbsolutePath() +
- " is not writable. Delta imports are supported by data config but will not work.");
+// File persistFile = propWriter.getPersistFile();
+// boolean isWritable = persistFile.exists() ? persistFile.canWrite() : persistFile.getParentFile().canWrite();
+ if (isDeltaImportSupported && !propWriter.isWritable()) {
+ throw new DataImportHandlerException(SEVERE,
+ "Properties is not writable. Delta imports are supported by data config but will not work.");
}
}
@@ -384,7 +396,7 @@ public class DataImporter {
try {
setIndexStartTime(new Date());
- docBuilder = new DocBuilder(this, writer, requestParams);
+ docBuilder = new DocBuilder(this, writer, propWriter, requestParams);
checkWritablePersistFile(writer);
docBuilder.execute();
if (!requestParams.debug)
Modified: lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugLogger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugLogger.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugLogger.java (original)
+++ lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DebugLogger.java Fri Jul 22 11:47:38 2011
@@ -45,7 +45,7 @@ class DebugLogger {
private Stack<DebugInfo> debugStack;
NamedList output;
- private final SolrWriter writer;
+// private final SolrWriter writer1;
private static final String LINE = "---------------------------------------------";
@@ -54,8 +54,8 @@ class DebugLogger {
boolean enabled = true;
- public DebugLogger(SolrWriter solrWriter) {
- writer = solrWriter;
+ public DebugLogger() {
+// writer = solrWriter;
output = new NamedList();
debugStack = new Stack<DebugInfo>() {
@@ -67,7 +67,7 @@ class DebugLogger {
return super.pop();
}
};
- debugStack.push(new DebugInfo(null, -1, null));
+ debugStack.push(new DebugInfo(null, DIHLogLevels.NONE, null));
output = debugStack.peek().lst;
}
@@ -75,47 +75,47 @@ class DebugLogger {
return debugStack.isEmpty() ? null : debugStack.peek();
}
- public void log(int event, String name, Object row) {
- if (event == SolrWriter.DISABLE_LOGGING) {
+ public void log(DIHLogLevels event, String name, Object row) {
+ if (event == DIHLogLevels.DISABLE_LOGGING) {
enabled = false;
return;
- } else if (event == SolrWriter.ENABLE_LOGGING) {
+ } else if (event == DIHLogLevels.ENABLE_LOGGING) {
enabled = true;
return;
}
- if (!enabled && event != SolrWriter.START_ENTITY
- && event != SolrWriter.END_ENTITY) {
+ if (!enabled && event != DIHLogLevels.START_ENTITY
+ && event != DIHLogLevels.END_ENTITY) {
return;
}
- if (event == SolrWriter.START_DOC) {
- debugStack.push(new DebugInfo(null, SolrWriter.START_DOC, peekStack()));
- } else if (SolrWriter.START_ENTITY == event) {
+ if (event == DIHLogLevels.START_DOC) {
+ debugStack.push(new DebugInfo(null, DIHLogLevels.START_DOC, peekStack()));
+ } else if (DIHLogLevels.START_ENTITY == event) {
debugStack
- .push(new DebugInfo(name, SolrWriter.START_ENTITY, peekStack()));
- } else if (SolrWriter.ENTITY_OUT == event
- || SolrWriter.PRE_TRANSFORMER_ROW == event) {
- if (debugStack.peek().type == SolrWriter.START_ENTITY
- || debugStack.peek().type == SolrWriter.START_DOC) {
+ .push(new DebugInfo(name, DIHLogLevels.START_ENTITY, peekStack()));
+ } else if (DIHLogLevels.ENTITY_OUT == event
+ || DIHLogLevels.PRE_TRANSFORMER_ROW == event) {
+ if (debugStack.peek().type == DIHLogLevels.START_ENTITY
+ || debugStack.peek().type == DIHLogLevels.START_DOC) {
debugStack.peek().lst.add(null, fmt.format(new Object[]{++debugStack
.peek().rowCount}));
addToNamedList(debugStack.peek().lst, row);
debugStack.peek().lst.add(null, LINE);
}
- } else if (event == SolrWriter.ROW_END) {
+ } else if (event == DIHLogLevels.ROW_END) {
popAllTransformers();
- } else if (SolrWriter.END_ENTITY == event) {
- while (debugStack.pop().type != SolrWriter.START_ENTITY)
+ } else if (DIHLogLevels.END_ENTITY == event) {
+ while (debugStack.pop().type != DIHLogLevels.START_ENTITY)
;
- } else if (SolrWriter.END_DOC == event) {
- while (debugStack.pop().type != SolrWriter.START_DOC)
+ } else if (DIHLogLevels.END_DOC == event) {
+ while (debugStack.pop().type != DIHLogLevels.START_DOC)
;
- } else if (event == SolrWriter.TRANSFORMER_EXCEPTION) {
+ } else if (event == DIHLogLevels.TRANSFORMER_EXCEPTION) {
debugStack.push(new DebugInfo(name, event, peekStack()));
debugStack.peek().lst.add("EXCEPTION",
getStacktraceString((Exception) row));
- } else if (SolrWriter.TRANSFORMED_ROW == event) {
+ } else if (DIHLogLevels.TRANSFORMED_ROW == event) {
debugStack.push(new DebugInfo(name, event, peekStack()));
debugStack.peek().lst.add(null, LINE);
addToNamedList(debugStack.peek().lst, row);
@@ -124,10 +124,10 @@ class DebugLogger {
DataImportHandlerException dataImportHandlerException = (DataImportHandlerException) row;
dataImportHandlerException.debugged = true;
}
- } else if (SolrWriter.ENTITY_META == event) {
+ } else if (DIHLogLevels.ENTITY_META == event) {
popAllTransformers();
debugStack.peek().lst.add(name, row);
- } else if (SolrWriter.ENTITY_EXCEPTION == event) {
+ } else if (DIHLogLevels.ENTITY_EXCEPTION == event) {
if (row instanceof DataImportHandlerException) {
DataImportHandlerException dihe = (DataImportHandlerException) row;
if (dihe.debugged)
@@ -143,8 +143,8 @@ class DebugLogger {
private void popAllTransformers() {
while (true) {
- int type = debugStack.peek().type;
- if (type == SolrWriter.START_DOC || type == SolrWriter.START_ENTITY)
+ DIHLogLevels type = debugStack.peek().type;
+ if (type == DIHLogLevels.START_DOC || type == DIHLogLevels.START_ENTITY)
break;
debugStack.pop();
}
@@ -181,23 +181,23 @@ class DebugLogger {
@Override
public Object getData(String query) {
- writer.log(SolrWriter.ENTITY_META, "query", query);
+ log(DIHLogLevels.ENTITY_META, "query", query);
long start = System.currentTimeMillis();
try {
return ds.getData(query);
} catch (DataImportHandlerException de) {
- writer.log(SolrWriter.ENTITY_EXCEPTION,
+ log(DIHLogLevels.ENTITY_EXCEPTION,
null, de);
throw de;
} catch (Exception e) {
- writer.log(SolrWriter.ENTITY_EXCEPTION,
+ log(DIHLogLevels.ENTITY_EXCEPTION,
null, e);
DataImportHandlerException de = new DataImportHandlerException(
DataImportHandlerException.SEVERE, "", e);
de.debugged = true;
throw de;
} finally {
- writer.log(SolrWriter.ENTITY_META, "time-taken", DocBuilder
+ log(DIHLogLevels.ENTITY_META, "time-taken", DocBuilder
.getTimeElapsedSince(start));
}
}
@@ -208,18 +208,18 @@ class DebugLogger {
return new Transformer() {
@Override
public Object transformRow(Map<String, Object> row, Context context) {
- writer.log(SolrWriter.PRE_TRANSFORMER_ROW, null, row);
+ log(DIHLogLevels.PRE_TRANSFORMER_ROW, null, row);
String tName = getTransformerName(t);
Object result = null;
try {
result = t.transformRow(row, context);
- writer.log(SolrWriter.TRANSFORMED_ROW, tName, result);
+ log(DIHLogLevels.TRANSFORMED_ROW, tName, result);
} catch (DataImportHandlerException de) {
- writer.log(SolrWriter.TRANSFORMER_EXCEPTION, tName, de);
+ log(DIHLogLevels.TRANSFORMER_EXCEPTION, tName, de);
de.debugged = true;
throw de;
} catch (Exception e) {
- writer.log(SolrWriter.TRANSFORMER_EXCEPTION, tName, e);
+ log(DIHLogLevels.TRANSFORMER_EXCEPTION, tName, e);
DataImportHandlerException de = new DataImportHandlerException(DataImportHandlerException.SEVERE, "", e);
de.debugged = true;
throw de;
@@ -258,23 +258,23 @@ class DebugLogger {
NamedList lst;
- int type;
+ DIHLogLevels type;
DebugInfo parent;
- public DebugInfo(String name, int type, DebugInfo parent) {
+ public DebugInfo(String name, DIHLogLevels type, DebugInfo parent) {
this.name = name;
this.type = type;
this.parent = parent;
lst = new NamedList();
if (parent != null) {
String displayName = null;
- if (type == SolrWriter.START_ENTITY) {
+ if (type == DIHLogLevels.START_ENTITY) {
displayName = "entity:" + name;
- } else if (type == SolrWriter.TRANSFORMED_ROW
- || type == SolrWriter.TRANSFORMER_EXCEPTION) {
+ } else if (type == DIHLogLevels.TRANSFORMED_ROW
+ || type == DIHLogLevels.TRANSFORMER_EXCEPTION) {
displayName = "transformer:" + name;
- } else if (type == SolrWriter.START_DOC) {
+ } else if (type == DIHLogLevels.START_DOC) {
this.name = displayName = "document#" + SolrWriter.getDocCount();
}
parent.lst.add(displayName, lst);
Modified: lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java Fri Jul 22 11:47:38 2011
@@ -56,33 +56,67 @@ public class DocBuilder {
public Statistics importStatistics = new Statistics();
- SolrWriter writer;
+ DIHWriter writer;
DataImporter.RequestParams requestParameters;
boolean verboseDebug = false;
- Map<String, Object> session = new ConcurrentHashMap<String, Object>();
+ Map<String, Object> session = new ConcurrentHashMap<String, Object>();
static final ThreadLocal<DocBuilder> INSTANCE = new ThreadLocal<DocBuilder>();
Map<String, Object> functionsNamespace;
private Properties persistedProperties;
+
+ private DIHPropertiesWriter propWriter;
+ private static final String PARAM_WRITER_IMPL = "writerImpl";
+ private static final String DEFAULT_WRITER_NAME = "SolrWriter";
+ private DebugLogger debugLogger;
- public DocBuilder(DataImporter dataImporter, SolrWriter writer, DataImporter.RequestParams reqParams) {
+ @SuppressWarnings("unchecked")
+ public DocBuilder(DataImporter dataImporter, SolrWriter solrWriter, DIHPropertiesWriter propWriter, DataImporter.RequestParams reqParams) {
INSTANCE.set(this);
this.dataImporter = dataImporter;
- this.writer = writer;
+ this.propWriter = propWriter;
DataImporter.QUERY_COUNT.set(importStatistics.queryCount);
requestParameters = reqParams;
verboseDebug = requestParameters.debug && requestParameters.verbose;
functionsNamespace = EvaluatorBag.getFunctionsNamespace(this.dataImporter.getConfig().functions, this);
- persistedProperties = writer.readIndexerProperties();
+ persistedProperties = propWriter.readIndexerProperties();
+
+ String writerClassStr = null;
+ if(reqParams!=null && reqParams.requestParams != null) {
+ writerClassStr = (String) reqParams.requestParams.get(PARAM_WRITER_IMPL);
+ }
+ if(writerClassStr != null && !writerClassStr.equals(DEFAULT_WRITER_NAME) && !writerClassStr.equals(DocBuilder.class.getPackage().getName() + "." + DEFAULT_WRITER_NAME)) {
+ try {
+ Class<DIHWriter> writerClass = loadClass(writerClassStr, dataImporter.getCore());
+ this.writer = writerClass.newInstance();
+ } catch (Exception e) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE, "Unable to load Writer implementation:" + writerClassStr, e);
+ }
+ } else {
+ writer = solrWriter;
+ }
+ ContextImpl ctx = new ContextImpl(null, null, null, null, reqParams.requestParams, null, this);
+ writer.init(ctx);
+ }
+
+
+
+
+ DebugLogger getDebugLogger(){
+ if (debugLogger == null) {
+ debugLogger = new DebugLogger();
+ }
+ return debugLogger;
}
public VariableResolverImpl getVariableResolver() {
try {
VariableResolverImpl resolver = null;
- if(dataImporter != null && dataImporter.getCore() != null){
+ if(dataImporter != null && dataImporter.getCore() != null
+ && dataImporter.getCore().getResourceLoader().getCoreProperties() != null){
resolver = new VariableResolverImpl(dataImporter.getCore().getResourceLoader().getCoreProperties());
} else resolver = new VariableResolverImpl();
Map<String, Object> indexerNamespace = new HashMap<String, Object>();
@@ -135,94 +169,100 @@ public class DocBuilder {
@SuppressWarnings("unchecked")
public void execute() {
- dataImporter.store(DataImporter.STATUS_MSGS, statusMessages);
- document = dataImporter.getConfig().document;
- final AtomicLong startTime = new AtomicLong(System.currentTimeMillis());
- statusMessages.put(TIME_ELAPSED, new Object() {
- @Override
- public String toString() {
- return getTimeElapsedSince(startTime.get());
- }
- });
-
- statusMessages.put(DataImporter.MSG.TOTAL_QUERIES_EXECUTED,
- importStatistics.queryCount);
- statusMessages.put(DataImporter.MSG.TOTAL_ROWS_EXECUTED,
- importStatistics.rowsCount);
- statusMessages.put(DataImporter.MSG.TOTAL_DOC_PROCESSED,
- importStatistics.docCount);
- statusMessages.put(DataImporter.MSG.TOTAL_DOCS_SKIPPED,
- importStatistics.skipDocCount);
-
- List<String> entities = requestParameters.entities;
-
- // Trigger onImportStart
- if (document.onImportStart != null) {
- invokeEventListener(document.onImportStart);
- }
- AtomicBoolean fullCleanDone = new AtomicBoolean(false);
- //we must not do a delete of *:* multiple times if there are multiple root entities to be run
- Properties lastIndexTimeProps = new Properties();
- lastIndexTimeProps.setProperty(LAST_INDEX_KEY,
- DataImporter.DATE_TIME_FORMAT.get().format(dataImporter.getIndexStartTime()));
- for (DataConfig.Entity e : document.entities) {
- if (entities != null && !entities.contains(e.name))
- continue;
- lastIndexTimeProps.setProperty(e.name + "." + LAST_INDEX_KEY,
- DataImporter.DATE_TIME_FORMAT.get().format(new Date()));
- root = e;
- String delQuery = e.allAttributes.get("preImportDeleteQuery");
- if (dataImporter.getStatus() == DataImporter.Status.RUNNING_DELTA_DUMP) {
- cleanByQuery(delQuery, fullCleanDone);
- doDelta();
- delQuery = e.allAttributes.get("postImportDeleteQuery");
- if (delQuery != null) {
- fullCleanDone.set(false);
- cleanByQuery(delQuery, fullCleanDone);
- }
- } else {
- cleanByQuery(delQuery, fullCleanDone);
- doFullDump();
- delQuery = e.allAttributes.get("postImportDeleteQuery");
- if (delQuery != null) {
- fullCleanDone.set(false);
- cleanByQuery(delQuery, fullCleanDone);
- }
- }
- statusMessages.remove(DataImporter.MSG.TOTAL_DOC_PROCESSED);
- }
-
- if (stop.get()) {
- // Dont commit if aborted using command=abort
- statusMessages.put("Aborted", DataImporter.DATE_TIME_FORMAT.get().format(new Date()));
- rollback();
- } else {
- // Do not commit unnecessarily if this is a delta-import and no documents were created or deleted
- if (!requestParameters.clean) {
- if (importStatistics.docCount.get() > 0 || importStatistics.deletedDocCount.get() > 0) {
- finish(lastIndexTimeProps);
- }
- } else {
- // Finished operation normally, commit now
- finish(lastIndexTimeProps);
- }
-
- if (writer != null) {
- writer.finish();
- }
-
- if (document.onImportEnd != null) {
- invokeEventListener(document.onImportEnd);
- }
- }
-
- statusMessages.remove(TIME_ELAPSED);
- statusMessages.put(DataImporter.MSG.TOTAL_DOC_PROCESSED, ""+ importStatistics.docCount.get());
- if(importStatistics.failedDocCount.get() > 0)
- statusMessages.put(DataImporter.MSG.TOTAL_FAILED_DOCS, ""+ importStatistics.failedDocCount.get());
-
- statusMessages.put("Time taken ", getTimeElapsedSince(startTime.get()));
- LOG.info("Time taken = " + getTimeElapsedSince(startTime.get()));
+ try {
+ dataImporter.store(DataImporter.STATUS_MSGS, statusMessages);
+ document = dataImporter.getConfig().document;
+ final AtomicLong startTime = new AtomicLong(System.currentTimeMillis());
+ statusMessages.put(TIME_ELAPSED, new Object() {
+ @Override
+ public String toString() {
+ return getTimeElapsedSince(startTime.get());
+ }
+ });
+
+ statusMessages.put(DataImporter.MSG.TOTAL_QUERIES_EXECUTED,
+ importStatistics.queryCount);
+ statusMessages.put(DataImporter.MSG.TOTAL_ROWS_EXECUTED,
+ importStatistics.rowsCount);
+ statusMessages.put(DataImporter.MSG.TOTAL_DOC_PROCESSED,
+ importStatistics.docCount);
+ statusMessages.put(DataImporter.MSG.TOTAL_DOCS_SKIPPED,
+ importStatistics.skipDocCount);
+
+ List<String> entities = requestParameters.entities;
+
+ // Trigger onImportStart
+ if (document.onImportStart != null) {
+ invokeEventListener(document.onImportStart);
+ }
+ AtomicBoolean fullCleanDone = new AtomicBoolean(false);
+ //we must not do a delete of *:* multiple times if there are multiple root entities to be run
+ Properties lastIndexTimeProps = new Properties();
+ lastIndexTimeProps.setProperty(LAST_INDEX_KEY,
+ DataImporter.DATE_TIME_FORMAT.get().format(dataImporter.getIndexStartTime()));
+ for (DataConfig.Entity e : document.entities) {
+ if (entities != null && !entities.contains(e.name))
+ continue;
+ lastIndexTimeProps.setProperty(e.name + "." + LAST_INDEX_KEY,
+ DataImporter.DATE_TIME_FORMAT.get().format(new Date()));
+ root = e;
+ String delQuery = e.allAttributes.get("preImportDeleteQuery");
+ if (dataImporter.getStatus() == DataImporter.Status.RUNNING_DELTA_DUMP) {
+ cleanByQuery(delQuery, fullCleanDone);
+ doDelta();
+ delQuery = e.allAttributes.get("postImportDeleteQuery");
+ if (delQuery != null) {
+ fullCleanDone.set(false);
+ cleanByQuery(delQuery, fullCleanDone);
+ }
+ } else {
+ cleanByQuery(delQuery, fullCleanDone);
+ doFullDump();
+ delQuery = e.allAttributes.get("postImportDeleteQuery");
+ if (delQuery != null) {
+ fullCleanDone.set(false);
+ cleanByQuery(delQuery, fullCleanDone);
+ }
+ }
+ statusMessages.remove(DataImporter.MSG.TOTAL_DOC_PROCESSED);
+ }
+
+ if (stop.get()) {
+ // Dont commit if aborted using command=abort
+ statusMessages.put("Aborted", DataImporter.DATE_TIME_FORMAT.get().format(new Date()));
+ rollback();
+ } else {
+ // Do not commit unnecessarily if this is a delta-import and no documents were created or deleted
+ if (!requestParameters.clean) {
+ if (importStatistics.docCount.get() > 0 || importStatistics.deletedDocCount.get() > 0) {
+ finish(lastIndexTimeProps);
+ }
+ } else {
+ // Finished operation normally, commit now
+ finish(lastIndexTimeProps);
+ }
+
+ if (document.onImportEnd != null) {
+ invokeEventListener(document.onImportEnd);
+ }
+ }
+
+ statusMessages.remove(TIME_ELAPSED);
+ statusMessages.put(DataImporter.MSG.TOTAL_DOC_PROCESSED, ""+ importStatistics.docCount.get());
+ if(importStatistics.failedDocCount.get() > 0)
+ statusMessages.put(DataImporter.MSG.TOTAL_FAILED_DOCS, ""+ importStatistics.failedDocCount.get());
+
+ statusMessages.put("Time taken ", getTimeElapsedSince(startTime.get()));
+ LOG.info("Time taken = " + getTimeElapsedSince(startTime.get()));
+ } catch(Exception e)
+ {
+ throw new RuntimeException(e);
+ } finally
+ {
+ if (writer != null) {
+ writer.close();
+ }
+ }
}
@SuppressWarnings("unchecked")
@@ -238,7 +278,7 @@ public class DocBuilder {
addStatusMessage("Optimized");
}
try {
- writer.persist(lastIndexTimeProps);
+ propWriter.persist(lastIndexTimeProps);
} catch (Exception e) {
LOG.error("Could not write property file", e);
statusMessages.put("error", "Could not write property file. Delta imports will not work. " +
@@ -433,11 +473,11 @@ public class DocBuilder {
private void runAThread(ThreadedEntityProcessorWrapper epw, EntityRow rows, String currProcess) throws Exception {
currentEntityProcWrapper.set(epw);
epw.threadedInit(context);
- initEntity();
try {
+ Context.CURRENT_CONTEXT.set(context);
epw.init(rows);
+ initEntity();
DocWrapper docWrapper = this.docWrapper;
- Context.CURRENT_CONTEXT.set(context);
for (; ;) {
if(DocBuilder.this.stop.get()) break;
try {
@@ -556,11 +596,11 @@ public class DocBuilder {
Context.CURRENT_CONTEXT.set(ctx);
if (requestParameters.start > 0) {
- writer.log(SolrWriter.DISABLE_LOGGING, null, null);
+ getDebugLogger().log(DIHLogLevels.DISABLE_LOGGING, null, null);
}
if (verboseDebug) {
- writer.log(SolrWriter.START_ENTITY, entity.name, null);
+ getDebugLogger().log(DIHLogLevels.START_ENTITY, entity.name, null);
}
int seenDocCount = 0;
@@ -574,11 +614,11 @@ public class DocBuilder {
seenDocCount++;
if (seenDocCount > requestParameters.start) {
- writer.log(SolrWriter.ENABLE_LOGGING, null, null);
+ getDebugLogger().log(DIHLogLevels.ENABLE_LOGGING, null, null);
}
if (verboseDebug && entity.isDocRoot) {
- writer.log(SolrWriter.START_DOC, entity.name, null);
+ getDebugLogger().log(DIHLogLevels.START_DOC, entity.name, null);
}
if (doc == null && entity.isDocRoot) {
doc = new DocWrapper();
@@ -607,7 +647,7 @@ public class DocBuilder {
}
if (verboseDebug) {
- writer.log(SolrWriter.ENTITY_OUT, entity.name, arow);
+ getDebugLogger().log(DIHLogLevels.ENTITY_OUT, entity.name, arow);
}
importStatistics.rowsCount.incrementAndGet();
if (doc != null) {
@@ -643,7 +683,7 @@ public class DocBuilder {
} catch (DataImportHandlerException e) {
if (verboseDebug) {
- writer.log(SolrWriter.ENTITY_EXCEPTION, entity.name, e);
+ getDebugLogger().log(DIHLogLevels.ENTITY_EXCEPTION, entity.name, e);
}
if(e.getErrCode() == DataImportHandlerException.SKIP_ROW){
continue;
@@ -662,21 +702,21 @@ public class DocBuilder {
throw e;
} catch (Throwable t) {
if (verboseDebug) {
- writer.log(SolrWriter.ENTITY_EXCEPTION, entity.name, t);
+ getDebugLogger().log(DIHLogLevels.ENTITY_EXCEPTION, entity.name, t);
}
throw new DataImportHandlerException(DataImportHandlerException.SEVERE, t);
} finally {
if (verboseDebug) {
- writer.log(SolrWriter.ROW_END, entity.name, null);
+ getDebugLogger().log(DIHLogLevels.ROW_END, entity.name, null);
if (entity.isDocRoot)
- writer.log(SolrWriter.END_DOC, null, null);
+ getDebugLogger().log(DIHLogLevels.END_DOC, null, null);
Context.CURRENT_CONTEXT.remove();
}
}
}
} finally {
if (verboseDebug) {
- writer.log(SolrWriter.END_ENTITY, null, null);
+ getDebugLogger().log(DIHLogLevels.END_ENTITY, null, null);
}
entityProcessor.destroy();
}
Modified: lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java (original)
+++ lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java Fri Jul 22 11:47:38 2011
@@ -83,7 +83,7 @@ public class EntityProcessorWrapper exte
@Override
public boolean add(Transformer transformer) {
if (docBuilder != null && docBuilder.verboseDebug) {
- transformer = docBuilder.writer.getDebugLogger().wrapTransformer(transformer);
+ transformer = docBuilder.getDebugLogger().wrapTransformer(transformer);
}
return super.add(transformer);
}
Modified: lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EvaluatorBag.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EvaluatorBag.java?rev=1149576&r1=1149575&r2=1149576&view=diff
==============================================================================
--- lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EvaluatorBag.java (original)
+++ lucene/dev/branches/positions/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EvaluatorBag.java Fri Jul 22 11:47:38 2011
@@ -217,7 +217,6 @@ public class EvaluatorBag {
Evaluator evaluator = evaluators.get(fname);
if (evaluator == null)
return null;
- VariableResolverImpl vri = VariableResolverImpl.CURRENT_VARIABLE_RESOLVER.get();
return evaluator.evaluate(m.group(2), Context.CURRENT_CONTEXT.get());
}