You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/08/13 13:17:06 UTC
svn commit: r1372366 [6/8] - in /lucene/dev/branches/pforcodec_3892: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/maven/ dev-tools/maven/lucene/
dev-tools/maven/lucene/analysis/common/
dev-tools/maven/lucene/analysis/icu/ ...
Modified: lucene/dev/branches/pforcodec_3892/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java Mon Aug 13 11:16:57 2012
@@ -99,16 +99,13 @@ public class PayloadIterator {
// Prepare for payload extraction
tp.nextPosition();
- // TODO: fix bug in SepCodec and then remove this check (the null check should be enough)
- if (!tp.hasPayload()) {
- return false;
- }
-
BytesRef br = tp.getPayload();
-
- if (br == null || br.length == 0) {
+
+ if (br == null) {
return false;
}
+
+ assert br.length > 0;
this.payloadLength = br.length;
Modified: lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.java Mon Aug 13 11:16:57 2012
@@ -60,18 +60,14 @@ public final class QueryTermExtractor
public static final WeightedTerm[] getIdfWeightedTerms(Query query, IndexReader reader, String fieldName)
{
WeightedTerm[] terms=getTerms(query,false, fieldName);
- int totalNumDocs=reader.numDocs();
+ int totalNumDocs=reader.maxDoc();
for (int i = 0; i < terms.length; i++)
{
try
{
int docFreq=reader.docFreq(new Term(fieldName,terms[i].term));
- // docFreq counts deletes
- if(totalNumDocs < docFreq) {
- docFreq = totalNumDocs;
- }
//IDF algorithm taken from DefaultSimilarity class
- float idf=(float)(Math.log((float)totalNumDocs/(double)(docFreq+1)) + 1.0);
+ float idf=(float)(Math.log(totalNumDocs/(double)(docFreq+1)) + 1.0);
terms[i].weight*=idf;
}
catch (IOException e)
Modified: lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java Mon Aug 13 11:16:57 2012
@@ -125,20 +125,7 @@ public class TokenSources {
}
private static boolean hasPositions(Terms vector) throws IOException {
- final TermsEnum termsEnum = vector.iterator(null);
- if (termsEnum.next() != null) {
- DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, DocsAndPositionsEnum.FLAG_PAYLOADS);
- if (dpEnum != null) {
- int doc = dpEnum.nextDoc();
- assert doc >= 0 && doc != DocIdSetIterator.NO_MORE_DOCS;
- int pos = dpEnum.nextPosition();
- if (pos >= 0) {
- return true;
- }
- }
- }
-
- return false;
+ return vector.hasPositions();
}
/**
Modified: lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java Mon Aug 13 11:16:57 2012
@@ -56,18 +56,17 @@ public final class TokenStreamFromTermPo
termAttribute = addAttribute(CharTermAttribute.class);
positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
offsetAttribute = addAttribute(OffsetAttribute.class);
+ final boolean hasOffsets = vector.hasOffsets();
final TermsEnum termsEnum = vector.iterator(null);
BytesRef text;
DocsAndPositionsEnum dpEnum = null;
while((text = termsEnum.next()) != null) {
dpEnum = termsEnum.docsAndPositions(null, dpEnum);
assert dpEnum != null; // presumably checked by TokenSources.hasPositions earlier
- boolean hasOffsets = true;
dpEnum.nextDoc();
final int freq = dpEnum.freq();
for (int j = 0; j < freq; j++) {
int pos = dpEnum.nextPosition();
- hasOffsets &= dpEnum.startOffset() >= 0;
Token token;
if (hasOffsets) {
token = new Token(text.utf8ToString(),
Modified: lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java Mon Aug 13 11:16:57 2012
@@ -431,7 +431,7 @@ public class WeightedSpanTermExtractor {
Map<String,WeightedSpanTerm> terms = new PositionCheckingMap<String>();
extract(query, terms);
- int totalNumDocs = reader.numDocs();
+ int totalNumDocs = reader.maxDoc();
Set<String> weightedTerms = terms.keySet();
Iterator<String> it = weightedTerms.iterator();
@@ -439,12 +439,8 @@ public class WeightedSpanTermExtractor {
while (it.hasNext()) {
WeightedSpanTerm weightedSpanTerm = terms.get(it.next());
int docFreq = reader.docFreq(new Term(fieldName, weightedSpanTerm.term));
- // docFreq counts deletes
- if(totalNumDocs < docFreq) {
- docFreq = totalNumDocs;
- }
// IDF algorithm taken from DefaultSimilarity class
- float idf = (float) (Math.log((float) totalNumDocs / (double) (docFreq + 1)) + 1.0);
+ float idf = (float) (Math.log(totalNumDocs / (double) (docFreq + 1)) + 1.0);
weightedSpanTerm.weight *= idf;
}
} finally {
Modified: lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java Mon Aug 13 11:16:57 2012
@@ -95,8 +95,7 @@ public class FieldTermStack {
DocsAndPositionsEnum dpEnum = null;
BytesRef text;
- int numDocs = reader.numDocs() - reader.numDeletedDocs();
- float weight = 0;
+ int numDocs = reader.maxDoc();
while ((text = termsEnum.next()) != null) {
UnicodeUtil.UTF8toUTF16(text, spare);
@@ -111,13 +110,14 @@ public class FieldTermStack {
}
dpEnum.nextDoc();
+
+ // For weight look here: http://lucene.apache.org/core/3_6_0/api/core/org/apache/lucene/search/DefaultSimilarity.html
+ final float weight = ( float ) ( Math.log( numDocs / ( double ) ( reader.docFreq( fieldName, text ) + 1 ) ) + 1.0 );
final int freq = dpEnum.freq();
for(int i = 0;i < freq;i++) {
int pos = dpEnum.nextPosition();
- // For weight look here: http://lucene.apache.org/core/3_6_0/api/core/org/apache/lucene/search/DefaultSimilarity.html
- weight = ( float ) ( Math.log( numDocs / ( double ) ( reader.docFreq( fieldName, text ) + 1 ) ) + 1.0 );
if (dpEnum.startOffset() < 0) {
return; // no offsets, null snippet
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/ivy-settings.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/ivy-settings.xml?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/ivy-settings.xml (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/ivy-settings.xml Mon Aug 13 11:16:57 2012
@@ -19,6 +19,8 @@
<ivysettings>
<settings defaultResolver="default"/>
+ <property name="local-maven2-dir" value="${user.home}/.m2/repository/" />
+
<include url="${ivy.default.settings.dir}/ivysettings-public.xml"/>
<include url="${ivy.default.settings.dir}/ivysettings-shared.xml"/>
<include url="${ivy.default.settings.dir}/ivysettings-local.xml"/>
@@ -26,11 +28,22 @@
<resolvers>
<ibiblio name="sonatype-releases" root="http://oss.sonatype.org/content/repositories/releases" m2compatible="true" />
+
<!-- you might need to tweak this from china so it works -->
<ibiblio name="working-chinese-mirror" root="http://mirror.netcologne.de/maven2" m2compatible="true" />
+ <!--
+ <filesystem name="local-maven-2" m2compatible="true" local="true">
+ <artifact
+ pattern="${local-maven2-dir}/[organisation]/[module]/[revision]/[module]-[revision].[ext]" />
+ <ivy
+ pattern="${local-maven2-dir}/[organisation]/[module]/[revision]/[module]-[revision].pom" />
+ </filesystem>
+ -->
+
<chain name="default" returnFirst="true" checkmodified="true" changingPattern=".*SNAPSHOT">
<resolver ref="local"/>
+ <!-- <resolver ref="local-maven-2" /> -->
<resolver ref="main"/>
<resolver ref="sonatype-releases" />
<resolver ref="working-chinese-mirror" />
Modified: lucene/dev/branches/pforcodec_3892/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Mon Aug 13 11:16:57 2012
@@ -25,6 +25,7 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
+import java.util.NoSuchElementException;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
@@ -42,7 +43,6 @@ import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.FieldsEnum;
import org.apache.lucene.index.OrdTermState;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.TermState;
@@ -718,22 +718,27 @@ public class MemoryIndex {
private class MemoryFields extends Fields {
@Override
- public FieldsEnum iterator() {
- return new FieldsEnum() {
+ public Iterator<String> iterator() {
+ return new Iterator<String>() {
int upto = -1;
@Override
public String next() {
upto++;
if (upto >= sortedFields.length) {
- return null;
+ throw new NoSuchElementException();
}
return sortedFields[upto].getKey();
}
@Override
- public Terms terms() {
- return MemoryFields.this.terms(sortedFields[upto].getKey());
+ public boolean hasNext() {
+ return upto+1 < sortedFields.length;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
}
};
}
@@ -778,8 +783,21 @@ public class MemoryIndex {
public int getDocCount() {
return info.sortedTerms.length > 0 ? 1 : 0;
}
-
-
+
+ @Override
+ public boolean hasOffsets() {
+ return stride == 3;
+ }
+
+ @Override
+ public boolean hasPositions() {
+ return true;
+ }
+
+ @Override
+ public boolean hasPayloads() {
+ return false;
+ }
};
}
}
@@ -1003,11 +1021,6 @@ public class MemoryIndex {
}
@Override
- public boolean hasPayload() {
- return false;
- }
-
- @Override
public BytesRef getPayload() {
return null;
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java Mon Aug 13 11:16:57 2012
@@ -25,7 +25,6 @@ import org.apache.lucene.index.MultiFiel
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.FieldsEnum;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.search.DocIdSetIterator;
@@ -133,16 +132,10 @@ public class HighFreqTerms {
throw new RuntimeException("no fields found for this index");
}
tiq = new TermStatsQueue(numTerms);
- FieldsEnum fieldsEnum = fields.iterator();
- while (true) {
- field = fieldsEnum.next();
- if (field != null) {
- Terms terms = fieldsEnum.terms();
- if (terms != null) {
- tiq.fill(field, terms.iterator(null));
- }
- } else {
- break;
+ for (String fieldName : fields) {
+ Terms terms = fields.terms(fieldName);
+ if (terms != null) {
+ tiq.fill(fieldName, terms.iterator(null));
}
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/module-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/module-build.xml?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/module-build.xml (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/module-build.xml Mon Aug 13 11:16:57 2012
@@ -49,8 +49,8 @@
<path id="test.classpath" refid="test.base.classpath"/>
<path id="junit.classpath">
- <path refid="test.classpath"/>
<pathelement location="${build.dir}/classes/test"/>
+ <path refid="test.classpath"/>
<pathelement path="${java.class.path}"/>
</path>
Modified: lucene/dev/branches/pforcodec_3892/lucene/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java Mon Aug 13 11:16:57 2012
@@ -39,6 +39,7 @@ import org.apache.lucene.store.AlreadyCl
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.English;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.NamedThreadFactory;
/**
* Spell checker test case
@@ -413,7 +414,7 @@ public class TestSpellChecker extends Lu
int num_field2 = this.numdoc();
assertEquals(num_field2, num_field1 + 1);
int numThreads = 5 + random().nextInt(5);
- ExecutorService executor = Executors.newFixedThreadPool(numThreads);
+ ExecutorService executor = Executors.newFixedThreadPool(numThreads, new NamedThreadFactory("testConcurrentAccess"));
SpellCheckWorker[] workers = new SpellCheckWorker[numThreads];
for (int i = 0; i < numThreads; i++) {
SpellCheckWorker spellCheckWorker = new SpellCheckWorker(r);
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/ivy.xml?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/ivy.xml (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/ivy.xml Mon Aug 13 11:16:57 2012
@@ -33,8 +33,8 @@
<dependency org="org.apache.ant" name="ant-junit" rev="1.8.2" transitive="false" />
<dependency org="junit" name="junit" rev="4.10" transitive="false" conf="default->*;junit4-stdalone->*" />
- <dependency org="com.carrotsearch.randomizedtesting" name="junit4-ant" rev="1.6.0" transitive="false" conf="default->*;junit4-stdalone->*" />
- <dependency org="com.carrotsearch.randomizedtesting" name="randomizedtesting-runner" rev="1.6.0" transitive="false" conf="default->*;junit4-stdalone->*" />
+ <dependency org="com.carrotsearch.randomizedtesting" name="junit4-ant" rev="2.0.0.rc5" transitive="false" conf="default->*;junit4-stdalone->*" />
+ <dependency org="com.carrotsearch.randomizedtesting" name="randomizedtesting-runner" rev="2.0.0.rc5" transitive="false" conf="default->*;junit4-stdalone->*" />
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
</dependencies>
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Mon Aug 13 11:16:57 2012
@@ -486,6 +486,9 @@ public abstract class BaseTokenStreamTes
ft.setStoreTermVectors(true);
ft.setStoreTermVectorOffsets(random.nextBoolean());
ft.setStoreTermVectorPositions(random.nextBoolean());
+ if (ft.storeTermVectorPositions()) {
+ ft.setStoreTermVectorPayloads(random.nextBoolean());
+ }
}
if (random.nextBoolean()) {
ft.setOmitNorms(true);
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java Mon Aug 13 11:16:57 2012
@@ -19,6 +19,7 @@ package org.apache.lucene.codecs.asserti
import java.io.IOException;
import java.util.Comparator;
+import java.util.Iterator;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
@@ -30,7 +31,6 @@ import org.apache.lucene.codecs.lucene40
import org.apache.lucene.index.AssertingAtomicReader;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.index.FieldsEnum;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Terms;
@@ -70,10 +70,10 @@ public class AssertingPostingsFormat ext
}
@Override
- public FieldsEnum iterator() throws IOException {
- FieldsEnum iterator = in.iterator();
+ public Iterator<String> iterator() {
+ Iterator<String> iterator = in.iterator();
assert iterator != null;
- return new AssertingAtomicReader.AssertingFieldsEnum(iterator);
+ return iterator;
}
@Override
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java Mon Aug 13 11:16:57 2012
@@ -39,7 +39,6 @@ import org.apache.lucene.index.DocsAndPo
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldsEnum;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -50,6 +49,7 @@ import org.apache.lucene.store.IndexOutp
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.UnmodifiableIterator;
/** Stores all postings data in RAM, but writes a small
* token (header + single int) to identify which "slot" the
@@ -112,8 +112,8 @@ public class RAMOnlyPostingsFormat exten
}
@Override
- public FieldsEnum iterator() {
- return new RAMFieldsEnum(this);
+ public Iterator<String> iterator() {
+ return new UnmodifiableIterator<String>(fieldToTerms.keySet().iterator());
}
@Override
@@ -127,9 +127,11 @@ public class RAMOnlyPostingsFormat exten
long sumTotalTermFreq;
long sumDocFreq;
int docCount;
+ final FieldInfo info;
- RAMField(String field) {
+ RAMField(String field, FieldInfo info) {
this.field = field;
+ this.info = info;
}
@Override
@@ -161,6 +163,21 @@ public class RAMOnlyPostingsFormat exten
public Comparator<BytesRef> getComparator() {
return reverseUnicodeComparator;
}
+
+ @Override
+ public boolean hasOffsets() {
+ return info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+ }
+
+ @Override
+ public boolean hasPositions() {
+ return info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ }
+
+ @Override
+ public boolean hasPayloads() {
+ return info.hasPayloads();
+ }
}
static class RAMTerm {
@@ -198,7 +215,7 @@ public class RAMOnlyPostingsFormat exten
if (field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0) {
throw new UnsupportedOperationException("this codec cannot index offsets");
}
- RAMField ramField = new RAMField(field.name);
+ RAMField ramField = new RAMField(field.name, field);
postings.fieldToTerms.put(field.name, ramField);
termsConsumer.reset(ramField);
return termsConsumer;
@@ -286,33 +303,6 @@ public class RAMOnlyPostingsFormat exten
}
}
- // Classes for reading from the postings state
- static class RAMFieldsEnum extends FieldsEnum {
- private final RAMPostings postings;
- private final Iterator<String> it;
- private String current;
-
- public RAMFieldsEnum(RAMPostings postings) {
- this.postings = postings;
- this.it = postings.fieldToTerms.keySet().iterator();
- }
-
- @Override
- public String next() {
- if (it.hasNext()) {
- current = it.next();
- } else {
- current = null;
- }
- return current;
- }
-
- @Override
- public Terms terms() {
- return postings.fieldToTerms.get(current);
- }
- }
-
static class RAMTermsEnum extends TermsEnum {
Iterator<String> it;
String current;
@@ -507,13 +497,12 @@ public class RAMOnlyPostingsFormat exten
}
@Override
- public boolean hasPayload() {
- return current.payloads != null && current.payloads[posUpto-1] != null;
- }
-
- @Override
public BytesRef getPayload() {
- return new BytesRef(current.payloads[posUpto-1]);
+ if (current.payloads != null && current.payloads[posUpto-1] != null) {
+ return new BytesRef(current.payloads[posUpto-1]);
+ } else {
+ return null;
+ }
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java Mon Aug 13 11:16:57 2012
@@ -1,6 +1,7 @@
package org.apache.lucene.index;
import java.io.IOException;
+import java.util.Iterator;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.Bits;
@@ -60,10 +61,10 @@ public class AssertingAtomicReader exten
}
@Override
- public FieldsEnum iterator() throws IOException {
- FieldsEnum fieldsEnum = super.iterator();
- assert fieldsEnum != null;
- return new AssertingFieldsEnum(fieldsEnum);
+ public Iterator<String> iterator() {
+ Iterator<String> iterator = super.iterator();
+ assert iterator != null;
+ return iterator;
}
@Override
@@ -74,21 +75,6 @@ public class AssertingAtomicReader exten
}
/**
- * Wraps a FieldsEnum but with additional asserts
- */
- public static class AssertingFieldsEnum extends FilterFieldsEnum {
- public AssertingFieldsEnum(FieldsEnum in) {
- super(in);
- }
-
- @Override
- public Terms terms() throws IOException {
- Terms terms = super.terms();
- return terms == null ? null : new AssertingTerms(terms);
- }
- }
-
- /**
* Wraps a Terms but with additional asserts
*/
public static class AssertingTerms extends FilterTerms {
@@ -365,15 +351,9 @@ public class AssertingAtomicReader exten
assert state != DocsEnumState.START : "getPayload() called before nextDoc()/advance()";
assert state != DocsEnumState.FINISHED : "getPayload() called after NO_MORE_DOCS";
assert positionCount > 0 : "getPayload() called before nextPosition()!";
- return super.getPayload();
- }
-
- @Override
- public boolean hasPayload() {
- assert state != DocsEnumState.START : "hasPayload() called before nextDoc()/advance()";
- assert state != DocsEnumState.FINISHED : "hasPayload() called after NO_MORE_DOCS";
- assert positionCount > 0 : "hasPayload() called before nextPosition()!";
- return super.hasPayload();
+ BytesRef payload = super.getPayload();
+ assert payload == null || payload.length > 0 : "getPayload() returned payload with invalid length!";
+ return payload;
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java Mon Aug 13 11:16:57 2012
@@ -19,6 +19,8 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
import java.util.Set;
public final class FieldFilterAtomicReader extends FilterAtomicReader {
@@ -125,15 +127,16 @@ public final class FieldFilterAtomicRead
}
private class FieldFilterFields extends FilterFields {
+
public FieldFilterFields(Fields in) {
super(in);
}
@Override
- public int size() throws IOException {
+ public int size() {
// TODO: add faster implementation!
int c = 0;
- final FieldsEnum it = iterator();
+ final Iterator<String> it = iterator();
while (it.next() != null) {
c++;
}
@@ -141,16 +144,46 @@ public final class FieldFilterAtomicRead
}
@Override
- public FieldsEnum iterator() throws IOException {
- return new FilterFieldsEnum(super.iterator()) {
+ public Iterator<String> iterator() {
+ final Iterator<String> in = super.iterator();
+ return new Iterator<String>() {
+ String cached = null;
+
+ @Override
+ public String next() {
+ if (cached != null) {
+ String next = cached;
+ cached = null;
+ return next;
+ } else {
+ String next = doNext();
+ if (next == null) {
+ throw new NoSuchElementException();
+ } else {
+ return next;
+ }
+ }
+ }
+
@Override
- public String next() throws IOException {
- String f;
- while ((f = super.next()) != null) {
- if (hasField(f)) return f;
+ public boolean hasNext() {
+ return cached != null || (cached = doNext()) != null;
+ }
+
+ private String doNext() {
+ while (in.hasNext()) {
+ String field = in.next();
+ if (hasField(field)) {
+ return field;
+ }
}
return null;
- }
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
};
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/search/RandomSimilarityProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/search/RandomSimilarityProvider.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/search/RandomSimilarityProvider.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/search/RandomSimilarityProvider.java Mon Aug 13 11:16:57 2012
@@ -67,12 +67,12 @@ public class RandomSimilarityProvider ex
final List<Similarity> knownSims;
Map<String,Similarity> previousMappings = new HashMap<String,Similarity>();
final int perFieldSeed;
- final boolean shouldCoord;
+ final int coordType; // 0 = no coord, 1 = coord, 2 = crazy coord
final boolean shouldQueryNorm;
public RandomSimilarityProvider(Random random) {
perFieldSeed = random.nextInt();
- shouldCoord = random.nextBoolean();
+ coordType = random.nextInt(3);
shouldQueryNorm = random.nextBoolean();
knownSims = new ArrayList<Similarity>(allSims);
Collections.shuffle(knownSims, random);
@@ -80,10 +80,12 @@ public class RandomSimilarityProvider ex
@Override
public float coord(int overlap, int maxOverlap) {
- if (shouldCoord) {
+ if (coordType == 0) {
+ return 1.0f;
+ } else if (coordType == 1) {
return defaultSim.coord(overlap, maxOverlap);
} else {
- return 1.0f;
+ return overlap / ((float)maxOverlap + 1);
}
}
@@ -161,6 +163,14 @@ public class RandomSimilarityProvider ex
@Override
public synchronized String toString() {
- return "RandomSimilarityProvider(queryNorm=" + shouldQueryNorm + ",coord=" + shouldCoord + "): " + previousMappings.toString();
+ final String coordMethod;
+ if (coordType == 0) {
+ coordMethod = "no";
+ } else if (coordType == 1) {
+ coordMethod = "yes";
+ } else {
+ coordMethod = "crazy";
+ }
+ return "RandomSimilarityProvider(queryNorm=" + shouldQueryNorm + ",coord=" + coordMethod + "): " + previousMappings.toString();
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/LuceneJUnit3MethodProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/LuceneJUnit3MethodProvider.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/LuceneJUnit3MethodProvider.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/LuceneJUnit3MethodProvider.java Mon Aug 13 11:16:57 2012
@@ -17,13 +17,14 @@ package org.apache.lucene.util;
* limitations under the License.
*/
-import static com.carrotsearch.randomizedtesting.MethodCollector.flatten;
-import static com.carrotsearch.randomizedtesting.MethodCollector.mutableCopy1;
-
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Map;
+import com.carrotsearch.randomizedtesting.ClassModel;
+import com.carrotsearch.randomizedtesting.ClassModel.MethodModel;
import com.carrotsearch.randomizedtesting.TestMethodProvider;
/**
@@ -31,20 +32,21 @@ import com.carrotsearch.randomizedtestin
*/
public final class LuceneJUnit3MethodProvider implements TestMethodProvider {
@Override
- public Collection<Method> getTestMethods(Class<?> suiteClass, List<List<Method>> methods) {
- // We will return all methods starting with test* and rely on further validation to weed
- // out static or otherwise invalid test methods.
- List<Method> copy = mutableCopy1(flatten(methods));
- Iterator<Method> i =copy.iterator();
- while (i.hasNext()) {
- Method m= i.next();
- if (!m.getName().startsWith("test") ||
- !Modifier.isPublic(m.getModifiers()) ||
- Modifier.isStatic(m.getModifiers()) ||
- m.getParameterTypes().length != 0) {
- i.remove();
+ public Collection<Method> getTestMethods(Class<?> suiteClass, ClassModel classModel) {
+ Map<Method,MethodModel> methods = classModel.getMethods();
+ ArrayList<Method> result = new ArrayList<Method>();
+ for (MethodModel mm : methods.values()) {
+ // Skip any methods that have overrieds/ shadows.
+ if (mm.getDown() != null) continue;
+
+ Method m = mm.element;
+ if (m.getName().startsWith("test") &&
+ Modifier.isPublic(m.getModifiers()) &&
+ !Modifier.isStatic(m.getModifiers()) &&
+ m.getParameterTypes().length == 0) {
+ result.add(m);
}
}
- return copy;
+ return result;
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Mon Aug 13 11:16:57 2012
@@ -45,7 +45,13 @@ import org.junit.rules.TestRule;
import org.junit.runner.RunWith;
import com.carrotsearch.randomizedtesting.*;
import com.carrotsearch.randomizedtesting.annotations.*;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction.Action;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakGroup.Group;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies.Consequence;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import com.carrotsearch.randomizedtesting.rules.NoClassHooksShadowingRule;
+import com.carrotsearch.randomizedtesting.rules.NoInstanceHooksOverridesRule;
import com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule;
import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsBoolean;
@@ -113,7 +119,15 @@ import static com.carrotsearch.randomize
RunListenerPrintReproduceInfo.class
})
@SeedDecorators({MixWithSuiteName.class}) // See LUCENE-3995 for rationale.
-@ThreadLeaks(failTestIfLeaking = false)
+@ThreadLeakScope(Scope.SUITE)
+@ThreadLeakGroup(Group.MAIN)
+@ThreadLeakAction({Action.WARN, Action.INTERRUPT})
+@ThreadLeakLingering(linger = 20000) // Wait long for leaked threads to complete before failure. zk needs this.
+@ThreadLeakZombies(Consequence.IGNORE_REMAINING_TESTS)
+@TimeoutSuite(millis = 2 * TimeUnits.HOUR)
+@ThreadLeakFilters(defaultFilters = true, filters = {
+ QuickPatchThreadsFilter.class
+})
public abstract class LuceneTestCase extends Assert {
// --------------------------------------------------------------------
@@ -124,6 +138,7 @@ public abstract class LuceneTestCase ext
public static final String SYSPROP_WEEKLY = "tests.weekly";
public static final String SYSPROP_AWAITSFIX = "tests.awaitsfix";
public static final String SYSPROP_SLOW = "tests.slow";
+ public static final String SYSPROP_BADAPPLES = "tests.badapples";
/** @see #ignoreAfterMaxFailures*/
private static final String SYSPROP_MAXFAILURES = "tests.maxfailures";
@@ -170,7 +185,21 @@ public abstract class LuceneTestCase ext
@Retention(RetentionPolicy.RUNTIME)
@TestGroup(enabled = true, sysProperty = SYSPROP_SLOW)
public @interface Slow {}
-
+
+ /**
+ * Annotation for tests that fail frequently. You can disable them
+ * if you want to run a long build and not stop on something that
+ * is a known problem.
+ * <pre>
+ * -Dtests.badapples=false
+ * </pre>
+ */
+ @Documented
+ @Inherited
+ @Retention(RetentionPolicy.RUNTIME)
+ @TestGroup(enabled = true, sysProperty = SYSPROP_BADAPPLES)
+ public @interface BadApple {}
+
/**
* Annotation for test classes that should avoid certain codec types
* (because they are expensive, for example).
@@ -182,7 +211,6 @@ public abstract class LuceneTestCase ext
public @interface SuppressCodecs {
String[] value();
}
-
// -----------------------------------------------------------------
// Truly immutable fields and constants, initialized once and valid
@@ -347,11 +375,16 @@ public abstract class LuceneTestCase ext
.around(ignoreAfterMaxFailures)
.around(suiteFailureMarker)
.around(new TestRuleAssertionsRequired())
- .around(new TestRuleNoStaticHooksShadowing())
- .around(new TestRuleNoInstanceHooksOverrides())
+ .around(new NoClassHooksShadowingRule())
+ .around(new NoInstanceHooksOverridesRule() {
+ @Override
+ protected boolean verify(Method key) {
+ String name = key.getName();
+ return !(name.equals("setUp") || name.equals("tearDown"));
+ }
+ })
.around(new SystemPropertiesInvariantRule(IGNORED_INVARIANT_PROPERTIES))
.around(classNameRule = new TestRuleStoreClassName())
- .around(new TestRuleReportUncaughtExceptions())
.around(classEnvRule = new TestRuleSetupAndRestoreClassEnv());
@@ -378,7 +411,6 @@ public abstract class LuceneTestCase ext
.outerRule(testFailureMarker)
.around(ignoreAfterMaxFailures)
.around(threadAndTestNameRule)
- .around(new TestRuleReportUncaughtExceptions())
.around(new SystemPropertiesInvariantRule(IGNORED_INVARIANT_PROPERTIES))
.around(new TestRuleSetupAndRestoreInstanceEnv())
.around(new TestRuleFieldCacheSanity())
@@ -934,6 +966,10 @@ public abstract class LuceneTestCase ext
}
if (!newType.storeTermVectorPositions()) {
newType.setStoreTermVectorPositions(random.nextBoolean());
+
+ if (newType.storeTermVectorPositions() && !newType.storeTermVectorPayloads()) {
+ newType.setStoreTermVectorPayloads(random.nextBoolean());
+ }
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleAssertionsRequired.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleAssertionsRequired.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleAssertionsRequired.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleAssertionsRequired.java Mon Aug 13 11:16:57 2012
@@ -21,30 +21,24 @@ import org.junit.rules.TestRule;
import org.junit.runner.Description;
import org.junit.runners.model.Statement;
-import com.carrotsearch.randomizedtesting.ClassValidator;
-
/**
* Require assertions for Lucene/Solr packages.
*/
-public class TestRuleAssertionsRequired implements TestRule, ClassValidator {
+public class TestRuleAssertionsRequired implements TestRule {
@Override
public Statement apply(final Statement base, final Description description) {
return new Statement() {
@Override
public void evaluate() throws Throwable {
- validate(description.getTestClass());
+ try {
+ assert false;
+ throw new Exception("Test class requires assertions, enable assertions globally (-ea) or for Solr/Lucene subpackages only.");
+ } catch (AssertionError e) {
+ // Ok, enabled.
+ }
+
base.evaluate();
}
};
}
-
- @Override
- public void validate(Class<?> clazz) throws Throwable {
- try {
- assert false;
- throw new Exception("Test class requires assertions, enable assertions globally (-ea) or for Solr/Lucene subpackages only.");
- } catch (AssertionError e) {
- // Ok, enabled.
- }
- }
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java Mon Aug 13 11:16:57 2012
@@ -956,9 +956,18 @@ public class _TestUtil {
while (true) {
try {
Pattern p = Pattern.compile(_TestUtil.randomRegexpishString(random));
+ String replacement = null;
+ // ignore bugs in Sun's regex impl
+ try {
+ replacement = p.matcher(nonBmpString).replaceAll("_");
+ } catch (StringIndexOutOfBoundsException jdkBug) {
+ System.out.println("WARNING: your jdk is buggy!");
+ System.out.println("Pattern.compile(\"" + p.pattern() +
+ "\").matcher(\"AB\\uD840\\uDC00C\").replaceAll(\"_\"); should not throw IndexOutOfBounds!");
+ }
// Make sure the result of applying the pattern to a string with extended
// unicode characters is a valid utf16 string. See LUCENE-4078 for discussion.
- if (UnicodeUtil.validUTF16String(p.matcher(nonBmpString).replaceAll("_"))) {
+ if (replacement != null && UnicodeUtil.validUTF16String(replacement)) {
return p;
}
} catch (PatternSyntaxException ignored) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/tools/custom-tasks.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/tools/custom-tasks.xml?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/tools/custom-tasks.xml (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/tools/custom-tasks.xml Mon Aug 13 11:16:57 2012
@@ -51,8 +51,9 @@
<additional-filters />
<!-- Typical version patterns. -->
+ <replaceregex pattern="\.rc[0-9]+" replace="" flags="gi" />
<replaceregex pattern="\-(r)?([0-9\-\_\.])+(b(eta)?([0-9\-\.])*)?$" replace="" flags="gi" />
-
+
<!-- git hashcode pattern: its always 40 chars right? -->
<replaceregex pattern="\-[a-z0-9]{40,40}$" replace="" flags="gi" />
</filtermapper>
Modified: lucene/dev/branches/pforcodec_3892/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/CHANGES.txt?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/CHANGES.txt (original)
+++ lucene/dev/branches/pforcodec_3892/solr/CHANGES.txt Mon Aug 13 11:16:57 2012
@@ -24,8 +24,39 @@ $Id$
(No changes)
+================== 4.0.0 ==================
+
+Upgrading from Solr 4.0.0-BETA
+----------------------
+
+In order to better support distributed search mode, the TermVectorComponent's response format has been changed so that if the schema defines a uniqueKeyField, then that field value is used as the "key" for each document in it's response section, instead of the internal lucene doc id. Users w/o a uniqueKeyField will continue to see the same response format. See SOLR-3229 for more details.
+
+Optimizations
+----------------------
+
+* SOLR-3715: improve concurrency of the transaction log by removing
+ synchronization around log record serialization. (yonik)
+
+
+Bug Fixes
+----------------------
+
+* SOLR-3685: Solr Cloud sometimes skipped peersync attempt and replicated instead due
+ to tlog flags not being cleared when no updates were buffered during a previous
+ replication. (Markus Jelsma, Mark Miller, yonik)
+
+* SOLR-3229: Fixed TermVectorComponent to work with distributed search
+ (Hang Xie, hossman)
+
+* SOLR-3725: Fixed package-local-src-tgz target to not bring in unnecessary jars
+ and binary contents. (Michael Dodsworth via rmuir)
+
+* SOLR-3649: Fixed bug in JavabinLoader that caused deleteById(List<String> ids)
+ to not work in SolrJ (siren)
+
================== 4.0.0-BETA ===================
+
Versions of Major Components
---------------------
Apache Tika 1.1
@@ -109,6 +140,18 @@ New Features
* SOLR-3672: SimplePostTool: Improvements for posting files
Support for auto mode, recursive and wildcards (janhoy)
+Optimizations
+----------------------
+
+* SOLR-3708: Add hashCode to ClusterState so that structures built based on the
+ ClusterState can be easily cached. (Mark Miller)
+
+* SOLR-3709: Cache the url list created from the ClusterState in CloudSolrServer on each
+ request. (Mark Miller, yonik)
+
+* SOLR-3710: Change CloudSolrServer so that update requests are only sent to leaders by
+ default. (Mark Miller)
+
Bug Fixes
----------------------
@@ -176,6 +219,9 @@ Bug Fixes
* SOLR-3428: SolrCmdDistributor flushAdds/flushDeletes can cause repeated
adds/deletes to be sent (Mark Miller, Per Steffensen)
+
+* SOLR-3647: DistributedQueue should use our Solr zk client rather than the std zk
+ client. ZooKeeper expiration can be permanent otherwise. (Mark Miller)
Other Changes
----------------------
@@ -204,7 +250,7 @@ Other Changes
* SOLR-3215: Clone SolrInputDocument when distrib indexing so that update processors after
the distrib update process do not process the document twice. (Mark Miller)
-
+
* SOLR-3683: Improved error handling if an <analyzer> contains both an
explicit class attribute, as well as nested factories. (hossman)
@@ -217,6 +263,13 @@ Other Changes
using a file (this previously worked in debug mode only). When configuration errors are
encountered, the error message is returned in XML format. (James Dyer)
+* SOLR-3439: Make SolrCell easier to use out of the box. Also improves "/browse" to display
+ rich-text documents correctly, along with facets for author and content_type.
+ With the new "content" field, highlighting of body is supported. See also SOLR-3672 for
+ easier posting of a whole directory structure. (Jack Krupansky, janhoy)
+
+* SOLR-3579: SolrCloud view should default to the graph view rather than tree view.
+ (steffkes, Mark Miller)
================== 4.0.0-ALPHA ==================
More information about this release, including any errata related to the
Modified: lucene/dev/branches/pforcodec_3892/solr/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/build.xml?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/build.xml (original)
+++ lucene/dev/branches/pforcodec_3892/solr/build.xml Mon Aug 13 11:16:57 2012
@@ -200,6 +200,7 @@
<include name="jdk.txt" />
<include name="jdk-deprecated.txt" />
<include name="commons-io.txt" />
+ <include name="executors.txt" />
</apiFileSet>
<fileset dir="${basedir}/build">
<include name="**/*.class" />
@@ -258,27 +259,6 @@
</delete>
</target>
- <!--
- Run after Junit tests.
- -->
- <target name="generate-clover-reports" depends="clover">
- <fail unless="run.clover">Clover not enabled!</fail>
- <mkdir dir="${clover.report.dir}"/>
- <fileset dir="build" id="clover.test.result.files">
- <include name="**/test/TEST-*.xml"/>
- </fileset>
- <clover-report>
- <current outfile="${clover.report.dir}" title="${final.name}" numThreads="0">
- <format type="html" filter="assert"/>
- <testresults refid="clover.test.result.files"/>
- </current>
- <current outfile="${clover.report.dir}/clover.xml" title="${final.name}">
- <format type="xml" filter="assert"/>
- <testresults refid="clover.test.result.files"/>
- </current>
- </clover-report>
- </target>
-
<!-- ========================================================================= -->
<!-- ===================== DISTRIBUTION-RELATED TASKS ======================== -->
<!-- ========================================================================= -->
@@ -361,12 +341,17 @@
value="${common-solr.dir}/build/${fullnamever}-src.tgz"/>
<delete file="${source.package.file}" failonerror="false" />
+ <!-- includes/excludes requires a relative path -->
+ <property name="dist.rel" location="${dist}" relative="yes"/>
+ <property name="package.dir.rel" location="${package.dir}" relative="yes"/>
+
<tar destfile="${source.package.file}" compression="gzip" longfile="gnu">
<tarfileset dir=".." prefix="${fullnamever}" includes="*.txt *.xml dev-tools/" />
<tarfileset dir="." prefix="${fullnamever}" includes="LICENSE.txt NOTICE.txt"/>
<tarfileset dir="." prefix="${fullnamever}/solr"
- excludes="build ${package.dir}/** ${dist}/**
- example/webapps/*.war example/exampledocs/post.jar
+ excludes="build/** ${package.dir.rel}/** ${dist.rel}/**
+ example/webapps/*.war example/lib/**
+ **/*.jar
lib/README.committers.txt **/data/ **/logs/*
**/*.sh **/bin/ scripts/
.idea/ **/*.iml **/pom.xml" />
Modified: lucene/dev/branches/pforcodec_3892/solr/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/common-build.xml?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/common-build.xml (original)
+++ lucene/dev/branches/pforcodec_3892/solr/common-build.xml Mon Aug 13 11:16:57 2012
@@ -49,8 +49,6 @@
<property name="tests.loggingfile" value="${common-solr.dir}/testlogging.properties"/>
<property name="tests.cleanthreads.sysprop" value="perClass"/>
- <property name="clover.report.dir" location="${dest}/clover/reports"/>
-
<import file="${common-solr.dir}/../lucene/module-build.xml"/>
<path id="additional.dependencies">
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java Mon Aug 13 11:16:57 2012
@@ -30,6 +30,8 @@ import org.apache.velocity.runtime.Runti
import org.apache.velocity.tools.generic.*;
import java.io.*;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Properties;
public class VelocityResponseWriter implements QueryResponseWriter {
@@ -81,6 +83,64 @@ public class VelocityResponseWriter impl
context.put("engine", engine); // for $engine.resourceExists(...)
+ // Mimetype to extension map for detecting file type and show icon
+ // List of types match the icons in /solr/img/filetypes
+ Map<String, String> mimeToExt = new HashMap<String, String>() {{
+ put("application/x-7z-compressed", "7z");
+ put("application/postscript", "ai");
+ put("application/pgp-signature", "asc");
+ put("application/octet-stream", "bin");
+ put("application/x-bzip2", "bz2");
+ put("text/x-c", "c");
+ put("application/vnd.ms-htmlhelp", "chm");
+ put("application/java-vm", "class");
+ put("text/css", "css");
+ put("text/csv", "csv");
+ put("application/x-debian-package", "deb");
+ put("application/msword", "doc");
+ put("message/rfc822", "eml");
+ put("image/gif", "gif");
+ put("application/winhlp", "hlp");
+ put("text/html", "html");
+ put("application/java-archive", "jar");
+ put("text/x-java-source", "java");
+ put("image/jpeg", "jpeg");
+ put("application/javascript", "js");
+ put("application/vnd.oasis.opendocument.chart", "odc");
+ put("application/vnd.oasis.opendocument.formula", "odf");
+ put("application/vnd.oasis.opendocument.graphics", "odg");
+ put("application/vnd.oasis.opendocument.image", "odi");
+ put("application/vnd.oasis.opendocument.presentation", "odp");
+ put("application/vnd.oasis.opendocument.spreadsheet", "ods");
+ put("application/vnd.oasis.opendocument.text", "odt");
+ put("application/pdf", "pdf");
+ put("application/pgp-encrypted", "pgp");
+ put("image/png", "png");
+ put("application/vnd.ms-powerpoint", "ppt");
+ put("audio/x-pn-realaudio", "ram");
+ put("application/x-rar-compressed", "rar");
+ put("application/vnd.rn-realmedia", "rm");
+ put("application/rtf", "rtf");
+ put("application/x-shockwave-flash", "swf");
+ put("application/vnd.sun.xml.calc", "sxc");
+ put("application/vnd.sun.xml.draw", "sxd");
+ put("application/vnd.sun.xml.impress", "sxi");
+ put("application/vnd.sun.xml.writer", "sxw");
+ put("application/x-tar", "tar");
+ put("application/x-tex", "tex");
+ put("text/plain", "txt");
+ put("text/x-vcard", "vcf");
+ put("application/vnd.visio", "vsd");
+ put("audio/x-wav", "wav");
+ put("audio/x-ms-wma", "wma");
+ put("video/x-ms-wmv", "wmv");
+ put("application/vnd.ms-excel", "xls");
+ put("application/xml", "xml");
+ put("application/x-xpinstall", "xpi");
+ put("application/zip", "zip");
+ }};
+ context.put("mimeToExt", mimeToExt);
+
String layout_template = request.getParams().get("v.layout");
String json_wrapper = request.getParams().get("v.json");
boolean wrap_response = (layout_template != null) || (json_wrapper != null);
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java Mon Aug 13 11:16:57 2012
@@ -152,17 +152,19 @@ final class ShardLeaderElectionContext e
if (zkClient.exists(leaderPath, true)) {
zkClient.delete(leaderPath, -1, true);
}
-// System.out.println("I may be the new Leader:" + leaderPath
-// + " - I need to try and sync");
+ log.info("I may be the new leader - try and sync");
+ // we are going to attempt to be the leader
+ // first cancel any current recovery
+ core.getUpdateHandler().getSolrCoreState().cancelRecovery();
boolean success = syncStrategy.sync(zkController, core, leaderProps);
if (!success && anyoneElseActive()) {
rejoinLeaderElection(leaderSeqPath, core);
return;
}
}
+ log.info("I am the new leader: " + ZkCoreNodeProps.getCoreUrl(leaderProps));
// If I am going to be the leader I have to be active
- // System.out.println("I am leader go active");
core.getUpdateHandler().getSolrCoreState().cancelRecovery();
zkController.publish(core.getCoreDescriptor(), ZkStateReader.ACTIVE);
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java Mon Aug 13 11:16:57 2012
@@ -100,7 +100,7 @@ public class RecoveryStrategy extends Th
private void recoveryFailed(final SolrCore core,
final ZkController zkController, final String baseUrl,
final String shardZkNodeName, final CoreDescriptor cd) throws KeeperException, InterruptedException {
- SolrException.log(log, "Recovery failed - I give up.");
+ SolrException.log(log, "Recovery failed - I give up. core=" + coreName);
try {
zkController.publish(cd, ZkStateReader.RECOVERY_FAILED);
} finally {
@@ -115,7 +115,7 @@ public class RecoveryStrategy extends Th
ZkCoreNodeProps leaderCNodeProps = new ZkCoreNodeProps(leaderprops);
String leaderUrl = leaderCNodeProps.getCoreUrl();
- log.info("Attempting to replicate from " + leaderUrl);
+ log.info("Attempting to replicate from " + leaderUrl + ". core=" + coreName);
// if we are the leader, either we are trying to recover faster
// then our ephemeral timed out or we are the only node
@@ -205,7 +205,7 @@ public class RecoveryStrategy extends Th
SolrQueryResponse rsp = new SolrQueryResponse();
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
- log.info("Starting recovery process. recoveringAfterStartup=" + recoveringAfterStartup);
+ log.info("Starting recovery process. core=" + coreName + " recoveringAfterStartup=" + recoveringAfterStartup);
try {
doRecovery(core);
@@ -233,7 +233,7 @@ public class RecoveryStrategy extends Th
UpdateLog ulog;
ulog = core.getUpdateHandler().getUpdateLog();
if (ulog == null) {
- SolrException.log(log, "No UpdateLog found - cannot recover");
+ SolrException.log(log, "No UpdateLog found - cannot recover. core=" + coreName);
recoveryFailed(core, zkController, baseUrl, coreZkNodeName,
core.getCoreDescriptor());
return;
@@ -241,14 +241,17 @@ public class RecoveryStrategy extends Th
List<Long> recentVersions;
- UpdateLog.RecentUpdates recentUpdates = ulog.getRecentUpdates();
+ UpdateLog.RecentUpdates recentUpdates = null;
try {
+ recentUpdates = ulog.getRecentUpdates();
recentVersions = recentUpdates.getVersions(ulog.numRecordsToKeep);
} catch (Throwable t) {
- SolrException.log(log, "Corrupt tlog - ignoring", t);
+ SolrException.log(log, "Corrupt tlog - ignoring. core=" + coreName, t);
recentVersions = new ArrayList<Long>(0);
- }finally {
- recentUpdates.close();
+ } finally {
+ if (recentUpdates != null) {
+ recentUpdates.close();
+ }
}
List<Long> startingVersions = ulog.getStartingVersions();
@@ -282,7 +285,7 @@ public class RecoveryStrategy extends Th
// last operation at the time of startup had the GAP flag set...
// this means we were previously doing a full index replication
// that probably didn't complete and buffering updates in the meantime.
- log.info("Looks like a previous replication recovery did not complete - skipping peer sync");
+ log.info("Looks like a previous replication recovery did not complete - skipping peer sync. core=" + coreName);
firstTime = false; // skip peersync
}
}
@@ -308,7 +311,7 @@ public class RecoveryStrategy extends Th
// first thing we just try to sync
if (firstTime) {
firstTime = false; // only try sync the first time through the loop
- log.info("Attempting to PeerSync from " + leaderUrl + " recoveringAfterStartup="+recoveringAfterStartup);
+ log.info("Attempting to PeerSync from " + leaderUrl + " core=" + coreName + " - recoveringAfterStartup="+recoveringAfterStartup);
// System.out.println("Attempting to PeerSync from " + leaderUrl
// + " i am:" + zkController.getNodeName());
PeerSync peerSync = new PeerSync(core,
@@ -319,7 +322,7 @@ public class RecoveryStrategy extends Th
SolrQueryRequest req = new LocalSolrQueryRequest(core,
new ModifiableSolrParams());
core.getUpdateHandler().commit(new CommitUpdateCommand(req, false));
- log.info("PeerSync Recovery was successful - registering as Active");
+ log.info("PeerSync Recovery was successful - registering as Active. core=" + coreName);
// System.out
// .println("Sync Recovery was successful - registering as Active "
// + zkController.getNodeName());
@@ -348,11 +351,11 @@ public class RecoveryStrategy extends Th
return;
}
- log.info("PeerSync Recovery was not successful - trying replication");
+ log.info("PeerSync Recovery was not successful - trying replication. core=" + coreName);
}
//System.out.println("Sync Recovery was not successful - trying replication");
- log.info("Starting Replication Recovery");
- log.info("Begin buffering updates");
+ log.info("Starting Replication Recovery. core=" + coreName);
+ log.info("Begin buffering updates. core=" + coreName);
ulog.bufferUpdates();
replayed = false;
@@ -364,7 +367,7 @@ public class RecoveryStrategy extends Th
replay(ulog);
replayed = true;
- log.info("Replication Recovery was successful - registering as Active");
+ log.info("Replication Recovery was successful - registering as Active. core=" + coreName);
// if there are pending recovery requests, don't advert as active
zkController.publish(core.getCoreDescriptor(), ZkStateReader.ACTIVE);
close = true;
@@ -387,7 +390,7 @@ public class RecoveryStrategy extends Th
}
} catch (Throwable t) {
- log.error("Error while trying to recover.", t);
+ log.error("Error while trying to recover. core=" + coreName, t);
}
if (!successfulRecovery) {
@@ -396,13 +399,13 @@ public class RecoveryStrategy extends Th
// Or do a fall off retry...
try {
- log.error("Recovery failed - trying again...");
+ log.error("Recovery failed - trying again... core=" + coreName);
retries++;
if (retries >= MAX_RETRIES) {
if (retries == INTERRUPTED) {
} else {
- log.error("Recovery failed - max retries exceeded.");
+ log.error("Recovery failed - max retries exceeded. core=" + coreName);
recoveryFailed(core, zkController, baseUrl, coreZkNodeName,
core.getCoreDescriptor());
}
@@ -410,7 +413,7 @@ public class RecoveryStrategy extends Th
}
} catch (Exception e) {
- log.error("", e);
+ log.error("core=" + coreName, e);
}
try {
@@ -422,13 +425,13 @@ public class RecoveryStrategy extends Th
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
- log.warn("Recovery was interrupted", e);
+ log.warn("Recovery was interrupted. core=" + coreName, e);
retries = INTERRUPTED;
}
}
}
- log.info("Finished recovery process");
+ log.info("Finished recovery process. core=" + coreName);
}
@@ -437,9 +440,9 @@ public class RecoveryStrategy extends Th
Future<RecoveryInfo> future = ulog.applyBufferedUpdates();
if (future == null) {
// no replay needed\
- log.info("No replay needed");
+ log.info("No replay needed. core=" + coreName);
} else {
- log.info("Replaying buffered documents");
+ log.info("Replaying buffered documents. core=" + coreName);
// wait for replay
future.get();
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java Mon Aug 13 11:16:57 2012
@@ -34,7 +34,7 @@ public abstract class DirectoryFactory i
/**
* Indicates a Directory will no longer be used, and when it's ref count
* hits 0, it can be closed. On shutdown all directories will be closed
- * with this has been called or not. This is simply to allow early cleanup.
+ * whether this has been called or not. This is simply to allow early cleanup.
*
* @param directory
* @throws IOException
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/core/SolrCore.java Mon Aug 13 11:16:57 2012
@@ -70,6 +70,7 @@ import org.apache.solr.update.processor.
import org.apache.solr.update.processor.RunUpdateProcessorFactory;
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
+import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.solr.util.RefCounted;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
import org.apache.solr.util.plugin.PluginInfoInitialized;
@@ -1070,7 +1071,8 @@ public final class SolrCore implements S
private final LinkedList<RefCounted<SolrIndexSearcher>> _searchers = new LinkedList<RefCounted<SolrIndexSearcher>>();
private final LinkedList<RefCounted<SolrIndexSearcher>> _realtimeSearchers = new LinkedList<RefCounted<SolrIndexSearcher>>();
- final ExecutorService searcherExecutor = Executors.newSingleThreadExecutor();
+ final ExecutorService searcherExecutor = Executors.newSingleThreadExecutor(
+ new DefaultSolrThreadFactory("searcherExecutor"));
private int onDeckSearchers; // number of searchers preparing
// Lock ordering: one can acquire the openSearcherLock and then the searcherLock, but not vice-versa.
private Object searcherLock = new Object(); // the sync object for the searcher
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java Mon Aug 13 11:16:57 2012
@@ -36,6 +36,7 @@ import org.apache.lucene.analysis.util.T
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.util.hash.HashFunction;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.handler.admin.CoreAdminHandler;
@@ -175,6 +176,8 @@ public class SolrResourceLoader implemen
* this ResourceLoader.
*/
void reloadLuceneSPI() {
+ // Hash functions:
+ HashFunction.reloadHashFunctions(this.classLoader);
// Codecs:
PostingsFormat.reloadPostingsFormats(this.classLoader);
Codec.reloadCodecs(this.classLoader);
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/SnapPuller.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/SnapPuller.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/SnapPuller.java Mon Aug 13 11:16:57 2012
@@ -31,6 +31,7 @@ import org.apache.solr.common.params.Com
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.FastInputStream;
+import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.solr.util.FileUtils;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CachingDirectoryFactory.CloseListener;
@@ -178,7 +179,8 @@ public class SnapPuller {
}
}
};
- executorService = Executors.newSingleThreadScheduledExecutor();
+ executorService = Executors.newSingleThreadScheduledExecutor(
+ new DefaultSolrThreadFactory("snapPuller"));
long initialDelay = pollInterval - (System.currentTimeMillis() % pollInterval);
executorService.scheduleAtFixedRate(task, initialDelay, pollInterval, TimeUnit.MILLISECONDS);
LOG.info("Poll Scheduled at an interval of " + pollInterval + "ms");
@@ -311,7 +313,7 @@ public class SnapPuller {
LOG.info("Number of files in latest index in master: " + filesToDownload.size());
// Create the sync service
- fsyncService = Executors.newSingleThreadExecutor();
+ fsyncService = Executors.newSingleThreadExecutor(new DefaultSolrThreadFactory("fsyncService"));
// use a synchronized list because the list is read by other threads (to show details)
filesDownloaded = Collections.synchronizedList(new ArrayList<Map<String, Object>>());
// if the generateion of master is older than that of the slave , it means they are not compatible to be copied
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java Mon Aug 13 11:16:57 2012
@@ -11,11 +11,11 @@ import java.util.Set;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.FieldsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.Terms;
@@ -141,6 +141,15 @@ public class TermVectorComponent extends
NamedList<Object> termVectors = new NamedList<Object>();
rb.rsp.add(TERM_VECTORS, termVectors);
+
+ IndexSchema schema = rb.req.getSchema();
+ SchemaField keyField = schema.getUniqueKeyField();
+ String uniqFieldName = null;
+ if (keyField != null) {
+ uniqFieldName = keyField.getName();
+ termVectors.add("uniqueKeyFieldName", uniqFieldName);
+ }
+
FieldOptions allFields = new FieldOptions();
//figure out what options we have, and try to get the appropriate vector
allFields.termFreq = params.getBool(TermVectorParams.TF, false);
@@ -158,8 +167,6 @@ public class TermVectorComponent extends
allFields.tfIdf = true;
}
- //use this to validate our fields
- IndexSchema schema = rb.req.getSchema();
//Build up our per field mapping
Map<String, FieldOptions> fieldOptions = new HashMap<String, FieldOptions>();
NamedList<List<String>> warnings = new NamedList<List<String>>();
@@ -175,6 +182,11 @@ public class TermVectorComponent extends
// workarround SOLR-3523
if (null == field || "score".equals(field)) continue;
+ // we don't want to issue warnings about the uniqueKey field
+ // since it can cause lots of confusion in distributed requests
+ // where the uniqueKey field is injected into the fl for merging
+ final boolean fieldIsUniqueKey = field.equals(uniqFieldName);
+
SchemaField sf = schema.getFieldOrNull(field);
if (sf != null) {
if (sf.storeTermVector()) {
@@ -190,15 +202,15 @@ public class TermVectorComponent extends
option.tfIdf = params.getFieldBool(field, TermVectorParams.TF_IDF, allFields.tfIdf);
//Validate these are even an option
option.positions = params.getFieldBool(field, TermVectorParams.POSITIONS, allFields.positions);
- if (option.positions && !sf.storeTermPositions()){
+ if (option.positions && !sf.storeTermPositions() && !fieldIsUniqueKey){
noPos.add(field);
}
option.offsets = params.getFieldBool(field, TermVectorParams.OFFSETS, allFields.offsets);
- if (option.offsets && !sf.storeTermOffsets()){
+ if (option.offsets && !sf.storeTermOffsets() && !fieldIsUniqueKey){
noOff.add(field);
}
} else {//field doesn't have term vectors
- noTV.add(field);
+ if (!fieldIsUniqueKey) noTV.add(field);
}
} else {
//field doesn't exist
@@ -206,6 +218,11 @@ public class TermVectorComponent extends
}
}
} //else, deal with all fields
+
+ // NOTE: currently all typs of warnings are schema driven, and garunteed
+ // to be consistent across all shards - if additional types of warnings
+ // are added that might be differnet between shards, finishStage() needs
+ // to be changed to account for that.
boolean hasWarnings = false;
if (!noTV.isEmpty()) {
warnings.add("noTermVectors", noTV);
@@ -236,11 +253,7 @@ public class TermVectorComponent extends
IndexReader reader = searcher.getIndexReader();
//the TVMapper is a TermVectorMapper which can be used to optimize loading of Term Vectors
- SchemaField keyField = schema.getUniqueKeyField();
- String uniqFieldName = null;
- if (keyField != null) {
- uniqFieldName = keyField.getName();
- }
+
//Only load the id field to get the uniqueKey of that
//field
@@ -277,7 +290,6 @@ public class TermVectorComponent extends
while (iter.hasNext()) {
Integer docId = iter.next();
NamedList<Object> docNL = new NamedList<Object>();
- termVectors.add("doc-" + docId, docNL);
if (keyField != null) {
reader.document(docId, getUniqValue);
@@ -286,9 +298,13 @@ public class TermVectorComponent extends
uniqVal = uniqValues.get(0);
uniqValues.clear();
docNL.add("uniqueKey", uniqVal);
- termVectors.add("uniqueKeyFieldName", uniqFieldName);
+ termVectors.add(uniqVal, docNL);
}
+ } else {
+ // support for schemas w/o a unique key,
+ termVectors.add("doc-" + docId, docNL);
}
+
if ( null != fields ) {
for (Map.Entry<String, FieldOptions> entry : fieldOptions.entrySet()) {
final String field = entry.getKey();
@@ -301,10 +317,8 @@ public class TermVectorComponent extends
} else {
// extract all fields
final Fields vectors = reader.getTermVectors(docId);
- final FieldsEnum fieldsEnum = vectors.iterator();
- String field;
- while((field = fieldsEnum.next()) != null) {
- Terms terms = fieldsEnum.terms();
+ for (String field : vectors) {
+ Terms terms = vectors.terms(field);
if (terms != null) {
termsEnum = terms.iterator(termsEnum);
mapOneVector(docNL, allFields, reader, docId, termsEnum, field);
@@ -394,46 +408,6 @@ public class TermVectorComponent extends
return result;
}
- @Override
- public int distributedProcess(ResponseBuilder rb) throws IOException {
- int result = ResponseBuilder.STAGE_DONE;
- if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
- //Go ask each shard for it's vectors
- // for each shard, collect the documents for that shard.
- HashMap<String, Collection<ShardDoc>> shardMap = new HashMap<String, Collection<ShardDoc>>();
- for (ShardDoc sdoc : rb.resultIds.values()) {
- Collection<ShardDoc> shardDocs = shardMap.get(sdoc.shard);
- if (shardDocs == null) {
- shardDocs = new ArrayList<ShardDoc>();
- shardMap.put(sdoc.shard, shardDocs);
- }
- shardDocs.add(sdoc);
- }
- // Now create a request for each shard to retrieve the stored fields
- for (Collection<ShardDoc> shardDocs : shardMap.values()) {
- ShardRequest sreq = new ShardRequest();
- sreq.purpose = ShardRequest.PURPOSE_GET_FIELDS;
-
- sreq.shards = new String[]{shardDocs.iterator().next().shard};
-
- sreq.params = new ModifiableSolrParams();
-
- // add original params
- sreq.params.add(rb.req.getParams());
- sreq.params.remove(CommonParams.Q);//remove the query
- ArrayList<String> ids = new ArrayList<String>(shardDocs.size());
- for (ShardDoc shardDoc : shardDocs) {
- ids.add(shardDoc.id.toString());
- }
- sreq.params.add(TermVectorParams.DOC_IDS, StrUtils.join(ids, ','));
-
- rb.addRequest(this, sreq);
- }
- result = ResponseBuilder.STAGE_DONE;
- }
- return result;
- }
-
private static int getDocFreq(IndexReader reader, String field, BytesRef term) {
int result = 1;
try {
@@ -449,6 +423,40 @@ public class TermVectorComponent extends
}
+ @Override
+ public void finishStage(ResponseBuilder rb) {
+ if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
+
+ NamedList termVectors = new NamedList<Object>();
+ Map.Entry<String, Object>[] arr = new NamedList.NamedListEntry[rb.resultIds.size()];
+
+ for (ShardRequest sreq : rb.finished) {
+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) == 0 || !sreq.params.getBool(COMPONENT_NAME, false)) {
+ continue;
+ }
+ for (ShardResponse srsp : sreq.responses) {
+ NamedList<Object> nl = (NamedList<Object>)srsp.getSolrResponse().getResponse().get(TERM_VECTORS);
+ for (int i=0; i < nl.size(); i++) {
+ String key = nl.getName(i);
+ ShardDoc sdoc = rb.resultIds.get(key);
+ if (null == sdoc) {
+ // metadata, only need from one node, leave in order
+ if (termVectors.indexOf(key,0) < 0) {
+ termVectors.add(key, nl.getVal(i));
+ }
+ } else {
+ int idx = sdoc.positionInResponse;
+ arr[idx] = new NamedList.NamedListEntry<Object>(key, nl.getVal(i));
+ }
+ }
+ }
+ }
+ // remove nulls in case not all docs were able to be retrieved
+ termVectors.addAll(SolrPluginUtils.removeNulls(new NamedList<Object>(arr)));
+ rb.rsp.add(TERM_VECTORS, termVectors);
+ }
+ }
+
//////////////////////// NamedListInitializedPlugin methods //////////////////////
@Override
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java Mon Aug 13 11:16:57 2012
@@ -122,8 +122,8 @@ public class JavabinLoader extends Conte
for (String s : update.getDeleteById()) {
delcmd.id = s;
processor.processDelete(delcmd);
+ delcmd.clear();
}
- delcmd.id = null;
}
if(update.getDeleteQuery() != null) {
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/request/SimpleFacets.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/request/SimpleFacets.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/request/SimpleFacets.java Mon Aug 13 11:16:57 2012
@@ -397,7 +397,7 @@ public class SimpleFacets {
Integer.MAX_VALUE,
10, TimeUnit.SECONDS, // terminate idle threads after 10 sec
new SynchronousQueue<Runnable>() // directly hand off tasks
- , new DefaultSolrThreadFactory("facetExectutor")
+ , new DefaultSolrThreadFactory("facetExecutor")
);
/**
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/Grouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/Grouping.java?rev=1372366&r1=1372365&r2=1372366&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/Grouping.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/Grouping.java Mon Aug 13 11:16:57 2012
@@ -787,7 +787,7 @@ public class Grouping {
SchemaField schemaField = searcher.getSchema().getField(groupBy);
FieldType fieldType = schemaField.getType();
String readableValue = fieldType.indexedToReadable(group.groupValue.utf8ToString());
- IndexableField field = schemaField.createField(readableValue, 0.0f);
+ IndexableField field = schemaField.createField(readableValue, 1.0f);
nl.add("groupValue", fieldType.toObject(field));
} else {
nl.add("groupValue", null);