You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/05/31 13:25:50 UTC
svn commit: r1129631 [5/7] - in /lucene/dev/branches/docvalues: ./
dev-tools/eclipse/ dev-tools/idea/.idea/
dev-tools/idea/lucene/contrib/spellchecker/ dev-tools/idea/modules/suggest/
dev-tools/maven/lucene/contrib/ dev-tools/maven/lucene/contrib/spell...
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java Tue May 31 11:25:37 2011
@@ -30,6 +30,7 @@ import org.apache.lucene.document.Field.
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.codecs.mocksep.MockSepCodec;
+import org.apache.lucene.index.codecs.pulsing.PulsingCodec;
import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec;
import org.apache.lucene.index.codecs.standard.StandardCodec;
import org.apache.lucene.search.DocIdSetIterator;
@@ -74,7 +75,7 @@ public class TestAddIndexes extends Luce
writer.close();
writer = newWriter(aux2, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));
- // add 40 documents in compound files
+ // add 50 documents in compound files
addDocs2(writer, 50);
assertEquals(50, writer.maxDoc());
writer.close();
@@ -1084,4 +1085,126 @@ public class TestAddIndexes extends Luce
assertEquals("Only one compound segment should exist", 4, dir.listAll().length);
}
+ // LUCENE-3126: tests that if a non-CFS segment is copied, it is converted to
+ // a CFS, given MP preferences
+ public void testCopyIntoCFS() throws Exception {
+ // create an index, no CFS (so we can assert that existing segments are not affected)
+ Directory target = newDirectory();
+ LogMergePolicy lmp = newLogMergePolicy(false);
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null).setMergePolicy(lmp);
+ IndexWriter w = new IndexWriter(target, conf);
+ w.addDocument(new Document());
+ w.commit();
+ assertFalse(w.segmentInfos.info(0).getUseCompoundFile());
+
+ // prepare second index, no-CFS too + .del file + separate norms file
+ Directory src = newDirectory();
+ LogMergePolicy lmp2 = newLogMergePolicy(false);
+ IndexWriterConfig conf2 = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(random)).setMergePolicy(lmp2);
+ IndexWriter w2 = new IndexWriter(src, conf2);
+ Document doc = new Document();
+ doc.add(new Field("c", "some text", Store.YES, Index.ANALYZED));
+ w2.addDocument(doc);
+ doc = new Document();
+ doc.add(new Field("d", "delete", Store.NO, Index.NOT_ANALYZED_NO_NORMS));
+ w2.addDocument(doc);
+ w2.commit();
+ w2.deleteDocuments(new Term("d", "delete"));
+ w2.commit();
+ w2.close();
+
+ // create separate norms file
+ IndexReader r = IndexReader.open(src, false);
+ r.setNorm(0, "c", (byte) 1);
+ r.close();
+ assertTrue(".del file not found", src.fileExists("_0_1.del"));
+ assertTrue("separate norms file not found", src.fileExists("_0_1.s0"));
+
+ // Case 1: force 'CFS' on target
+ lmp.setUseCompoundFile(true);
+ lmp.setNoCFSRatio(1.0);
+ w.addIndexes(src);
+ w.commit();
+ assertFalse("existing segments should not be modified by addIndexes", w.segmentInfos.info(0).getUseCompoundFile());
+ assertTrue("segment should have been converted to a CFS by addIndexes", w.segmentInfos.info(1).getUseCompoundFile());
+ assertTrue(".del file not found", target.fileExists("_1_1.del"));
+ assertTrue("separate norms file not found", target.fileExists("_1_1.s0"));
+
+ // Case 2: LMP disallows CFS
+ lmp.setUseCompoundFile(false);
+ w.addIndexes(src);
+ w.commit();
+ assertFalse("segment should not have been converted to a CFS by addIndexes if MP disallows", w.segmentInfos.info(2).getUseCompoundFile());
+
+ w.close();
+
+ // cleanup
+ src.close();
+ target.close();
+ }
+
+ /*
+ * simple test that ensures we getting expected exceptions
+ */
+ public void testAddIndexMissingCodec() throws IOException {
+ Directory toAdd = newDirectory();
+ {
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(random));
+ CodecProvider provider = new CodecProvider();
+ provider.register(new StandardCodec());
+ conf.setCodecProvider(provider);
+ IndexWriter w = new IndexWriter(toAdd, conf);
+ Document doc = new Document();
+ doc.add(newField("foo", "bar", Index.NOT_ANALYZED));
+ w.addDocument(doc);
+ w.close();
+ }
+ {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(random));
+ CodecProvider provider = new CodecProvider();
+ provider.register(new PulsingCodec(1 + random.nextInt(10)));
+ conf.setCodecProvider(provider);
+ IndexWriter w = new IndexWriter(dir, conf);
+ try {
+ w.addIndexes(toAdd);
+ fail("no such codec");
+ } catch (IllegalArgumentException ex) {
+ // expected
+ }
+ w.close();
+ IndexReader open = IndexReader.open(dir);
+ assertEquals(0, open.numDocs());
+ open.close();
+ dir.close();
+ }
+
+ {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(random));
+ CodecProvider provider = new CodecProvider();
+ provider.register(new PulsingCodec(1 + random.nextInt(10)));
+ conf.setCodecProvider(provider);
+ IndexWriter w = new IndexWriter(dir, conf);
+ IndexReader indexReader = IndexReader.open(toAdd);
+ try {
+ w.addIndexes(indexReader);
+ fail("no such codec");
+ } catch (IllegalArgumentException ex) {
+ // expected
+ }
+ indexReader.close();
+ w.close();
+ IndexReader open = IndexReader.open(dir);
+ assertEquals(0, open.numDocs());
+ open.close();
+ dir.close();
+ }
+ toAdd.close();
+ }
+
}
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestCompoundFile.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestCompoundFile.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestCompoundFile.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestCompoundFile.java Tue May 31 11:25:37 2011
@@ -648,4 +648,25 @@ public class TestCompoundFile extends Lu
}
}
+
+ public void testAddExternalFile() throws IOException {
+ createSequenceFile(dir, "d1", (byte) 0, 15);
+
+ Directory newDir = newDirectory();
+ CompoundFileWriter csw = new CompoundFileWriter(newDir, "d.csf");
+ csw.addFile("d1", dir);
+ csw.close();
+
+ CompoundFileReader csr = new CompoundFileReader(newDir, "d.csf");
+ IndexInput expected = dir.openInput("d1");
+ IndexInput actual = csr.openInput("d1");
+ assertSameStreams("d1", expected, actual);
+ assertSameSeekBehavior("d1", expected, actual);
+ expected.close();
+ actual.close();
+ csr.close();
+
+ newDir.close();
+ }
+
}
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestCrash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestCrash.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestCrash.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestCrash.java Tue May 31 11:25:37 2011
@@ -176,6 +176,7 @@ public class TestCrash extends LuceneTes
reader = IndexReader.open(dir, false);
assertEquals(157, reader.numDocs());
reader.close();
+ dir.clearCrash();
dir.close();
}
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Tue May 31 11:25:37 2011
@@ -1688,6 +1688,7 @@ public class TestIndexReader extends Luc
r2.close();
assertTrue(ints == ints2);
+ writer.close();
dir.close();
}
@@ -1735,6 +1736,7 @@ public class TestIndexReader extends Luc
assertTrue(((SegmentReader) subs[1]).readOnly);
assertTrue(ints == ints2);
+ writer.close();
dir.close();
}
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Tue May 31 11:25:37 2011
@@ -68,6 +68,7 @@ import org.apache.lucene.store.NoLockFac
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.SingleInstanceLockFactory;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util.UnicodeUtil;
@@ -1631,7 +1632,7 @@ public class TestIndexWriter extends Luc
public void testAllUnicodeChars() throws Throwable {
BytesRef utf8 = new BytesRef(10);
- UnicodeUtil.UTF16Result utf16 = new UnicodeUtil.UTF16Result();
+ CharsRef utf16 = new CharsRef(10);
char[] chars = new char[2];
for(int ch=0;ch<0x0010FFFF;ch++) {
@@ -1654,7 +1655,7 @@ public class TestIndexWriter extends Luc
assertEquals("codepoint " + ch, s1, s2);
UnicodeUtil.UTF8toUTF16(utf8.bytes, 0, utf8.length, utf16);
- assertEquals("codepoint " + ch, s1, new String(utf16.result, 0, utf16.length));
+ assertEquals("codepoint " + ch, s1, new String(utf16.chars, 0, utf16.length));
byte[] b = s1.getBytes("UTF-8");
assertEquals(utf8.length, b.length);
@@ -1721,7 +1722,7 @@ public class TestIndexWriter extends Luc
char[] expected = new char[20];
BytesRef utf8 = new BytesRef(20);
- UnicodeUtil.UTF16Result utf16 = new UnicodeUtil.UTF16Result();
+ CharsRef utf16 = new CharsRef(20);
int num = 100000 * RANDOM_MULTIPLIER;
for (int iter = 0; iter < num; iter++) {
@@ -1738,62 +1739,7 @@ public class TestIndexWriter extends Luc
UnicodeUtil.UTF8toUTF16(utf8.bytes, 0, utf8.length, utf16);
assertEquals(utf16.length, 20);
for(int i=0;i<20;i++)
- assertEquals(expected[i], utf16.result[i]);
- }
- }
-
- // LUCENE-510
- public void testIncrementalUnicodeStrings() throws Throwable {
- char[] buffer = new char[20];
- char[] expected = new char[20];
-
- BytesRef utf8 = new BytesRef(new byte[20]);
- UnicodeUtil.UTF16Result utf16 = new UnicodeUtil.UTF16Result();
- UnicodeUtil.UTF16Result utf16a = new UnicodeUtil.UTF16Result();
-
- boolean hasIllegal = false;
- byte[] last = new byte[60];
-
- int num = 100000 * RANDOM_MULTIPLIER;
- for (int iter = 0; iter < num; iter++) {
-
- final int prefix;
-
- if (iter == 0 || hasIllegal)
- prefix = 0;
- else
- prefix = nextInt(20);
-
- hasIllegal = fillUnicode(buffer, expected, prefix, 20-prefix);
-
- UnicodeUtil.UTF16toUTF8(buffer, 0, 20, utf8);
- if (!hasIllegal) {
- byte[] b = new String(buffer, 0, 20).getBytes("UTF-8");
- assertEquals(b.length, utf8.length);
- for(int i=0;i<b.length;i++)
- assertEquals(b[i], utf8.bytes[i]);
- }
-
- int bytePrefix = 20;
- if (iter == 0 || hasIllegal)
- bytePrefix = 0;
- else
- for(int i=0;i<20;i++)
- if (last[i] != utf8.bytes[i]) {
- bytePrefix = i;
- break;
- }
- System.arraycopy(utf8.bytes, 0, last, 0, utf8.length);
-
- UnicodeUtil.UTF8toUTF16(utf8.bytes, bytePrefix, utf8.length-bytePrefix, utf16);
- assertEquals(20, utf16.length);
- for(int i=0;i<20;i++)
- assertEquals(expected[i], utf16.result[i]);
-
- UnicodeUtil.UTF8toUTF16(utf8.bytes, 0, utf8.length, utf16a);
- assertEquals(20, utf16a.length);
- for(int i=0;i<20;i++)
- assertEquals(expected[i], utf16a.result[i]);
+ assertEquals(expected[i], utf16.chars[i]);
}
}
@@ -2177,6 +2123,10 @@ public class TestIndexWriter extends Luc
allowInterrupt = true;
}
} catch (ThreadInterruptedException re) {
+ if (VERBOSE) {
+ System.out.println("TEST: got interrupt");
+ re.printStackTrace(System.out);
+ }
Throwable e = re.getCause();
assertTrue(e instanceof InterruptedException);
if (finish) {
@@ -2774,7 +2724,7 @@ public class TestIndexWriter extends Luc
// or, at most the write.lock file
final int extraFileCount;
if (files.length == 1) {
- assertEquals("write.lock", files[0]);
+ assertTrue(files[0].endsWith("write.lock"));
extraFileCount = 1;
} else {
assertEquals(0, files.length);
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Tue May 31 11:25:37 2011
@@ -833,6 +833,10 @@ public class TestIndexWriterDelete exten
try {
modifier.addDocument(doc);
} catch (IOException io) {
+ if (VERBOSE) {
+ System.out.println("TEST: got expected exc:");
+ io.printStackTrace(System.out);
+ }
break;
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Tue May 31 11:25:37 2011
@@ -17,24 +17,16 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.io.Reader;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.MockDirectoryWrapper;
-import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
@@ -43,9 +35,54 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
public class TestIndexWriterExceptions extends LuceneTestCase {
+ private static class DocCopyIterator implements Iterable<Document> {
+ private final Document doc;
+ private final int count;
+
+ public DocCopyIterator(Document doc, int count) {
+ this.count = count;
+ this.doc = doc;
+ }
+
+ // @Override -- not until Java 1.6
+ public Iterator<Document> iterator() {
+ return new Iterator<Document>() {
+ int upto;
+
+ // @Override -- not until Java 1.6
+ public boolean hasNext() {
+ return upto < count;
+ }
+
+ // @Override -- not until Java 1.6
+ public Document next() {
+ upto++;
+ return doc;
+ }
+
+ // @Override -- not until Java 1.6
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+ }
+
private class IndexerThread extends Thread {
IndexWriter writer;
@@ -87,7 +124,11 @@ public class TestIndexWriterExceptions e
idField.setValue(id);
Term idTerm = new Term("id", id);
try {
- writer.updateDocument(idTerm, doc);
+ if (r.nextBoolean()) {
+ writer.updateDocuments(idTerm, new DocCopyIterator(doc, _TestUtil.nextInt(r, 1, 20)));
+ } else {
+ writer.updateDocument(idTerm, doc);
+ }
} catch (RuntimeException re) {
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": EXC: ");
@@ -136,7 +177,7 @@ public class TestIndexWriterExceptions e
@Override
boolean testPoint(String name) {
- if (doFail.get() != null && !name.equals("startDoFlush") && r.nextInt(20) == 17) {
+ if (doFail.get() != null && !name.equals("startDoFlush") && r.nextInt(40) == 17) {
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": NOW FAIL: " + name);
new Throwable().printStackTrace(System.out);
@@ -267,6 +308,8 @@ public class TestIndexWriterExceptions e
}
}
+ private static String CRASH_FAIL_MESSAGE = "I'm experiencing problems";
+
private class CrashingFilter extends TokenFilter {
String fieldName;
int count;
@@ -279,7 +322,7 @@ public class TestIndexWriterExceptions e
@Override
public boolean incrementToken() throws IOException {
if (this.fieldName.equals("crash") && count++ >= 4)
- throw new IOException("I'm experiencing problems");
+ throw new IOException(CRASH_FAIL_MESSAGE);
return input.incrementToken();
}
@@ -876,7 +919,7 @@ public class TestIndexWriterExceptions e
assertTrue(failure.failOnCommit && failure.failOnDeleteFile);
w.rollback();
assertFalse(dir.fileExists("1.fnx"));
- // FIXME: on windows, this often fails! assertEquals(0, dir.listAll().length);
+ assertEquals(0, dir.listAll().length);
dir.close();
}
}
@@ -1278,4 +1321,141 @@ public class TestIndexWriterExceptions e
}
}
}
+
+ public void testAddDocsNonAbortingException() throws Exception {
+ final Directory dir = newDirectory();
+ final RandomIndexWriter w = new RandomIndexWriter(random, dir);
+ final int numDocs1 = random.nextInt(25);
+ for(int docCount=0;docCount<numDocs1;docCount++) {
+ Document doc = new Document();
+ doc.add(newField("content", "good content", Field.Index.ANALYZED));
+ w.addDocument(doc);
+ }
+
+ final List<Document> docs = new ArrayList<Document>();
+ for(int docCount=0;docCount<7;docCount++) {
+ Document doc = new Document();
+ docs.add(doc);
+ doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
+ doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
+ if (docCount == 4) {
+ Field f = newField("crash", "", Field.Index.ANALYZED);
+ doc.add(f);
+ MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);
+ tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
+ f.setTokenStream(new CrashingFilter("crash", tokenizer));
+ }
+ }
+ try {
+ w.addDocuments(docs);
+ // BUG: CrashingFilter didn't
+ fail("did not hit expected exception");
+ } catch (IOException ioe) {
+ // expected
+ assertEquals(CRASH_FAIL_MESSAGE, ioe.getMessage());
+ }
+
+ final int numDocs2 = random.nextInt(25);
+ for(int docCount=0;docCount<numDocs2;docCount++) {
+ Document doc = new Document();
+ doc.add(newField("content", "good content", Field.Index.ANALYZED));
+ w.addDocument(doc);
+ }
+
+ final IndexReader r = w.getReader();
+ w.close();
+
+ final IndexSearcher s = new IndexSearcher(r);
+ PhraseQuery pq = new PhraseQuery();
+ pq.add(new Term("content", "silly"));
+ pq.add(new Term("content", "content"));
+ assertEquals(0, s.search(pq, 1).totalHits);
+
+ pq = new PhraseQuery();
+ pq.add(new Term("content", "good"));
+ pq.add(new Term("content", "content"));
+ assertEquals(numDocs1+numDocs2, s.search(pq, 1).totalHits);
+ r.close();
+ dir.close();
+ }
+
+
+ public void testUpdateDocsNonAbortingException() throws Exception {
+ final Directory dir = newDirectory();
+ final RandomIndexWriter w = new RandomIndexWriter(random, dir);
+ final int numDocs1 = random.nextInt(25);
+ for(int docCount=0;docCount<numDocs1;docCount++) {
+ Document doc = new Document();
+ doc.add(newField("content", "good content", Field.Index.ANALYZED));
+ w.addDocument(doc);
+ }
+
+ // Use addDocs (no exception) to get docs in the index:
+ final List<Document> docs = new ArrayList<Document>();
+ final int numDocs2 = random.nextInt(25);
+ for(int docCount=0;docCount<numDocs2;docCount++) {
+ Document doc = new Document();
+ docs.add(doc);
+ doc.add(newField("subid", "subs", Field.Index.NOT_ANALYZED));
+ doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
+ doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
+ }
+ w.addDocuments(docs);
+
+ final int numDocs3 = random.nextInt(25);
+ for(int docCount=0;docCount<numDocs3;docCount++) {
+ Document doc = new Document();
+ doc.add(newField("content", "good content", Field.Index.ANALYZED));
+ w.addDocument(doc);
+ }
+
+ docs.clear();
+ final int limit = _TestUtil.nextInt(random, 2, 25);
+ final int crashAt = random.nextInt(limit);
+ for(int docCount=0;docCount<limit;docCount++) {
+ Document doc = new Document();
+ docs.add(doc);
+ doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
+ doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
+ if (docCount == crashAt) {
+ Field f = newField("crash", "", Field.Index.ANALYZED);
+ doc.add(f);
+ MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);
+ tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
+ f.setTokenStream(new CrashingFilter("crash", tokenizer));
+ }
+ }
+
+ try {
+ w.updateDocuments(new Term("subid", "subs"), docs);
+ // BUG: CrashingFilter didn't
+ fail("did not hit expected exception");
+ } catch (IOException ioe) {
+ // expected
+ assertEquals(CRASH_FAIL_MESSAGE, ioe.getMessage());
+ }
+
+ final int numDocs4 = random.nextInt(25);
+ for(int docCount=0;docCount<numDocs4;docCount++) {
+ Document doc = new Document();
+ doc.add(newField("content", "good content", Field.Index.ANALYZED));
+ w.addDocument(doc);
+ }
+
+ final IndexReader r = w.getReader();
+ w.close();
+
+ final IndexSearcher s = new IndexSearcher(r);
+ PhraseQuery pq = new PhraseQuery();
+ pq.add(new Term("content", "silly"));
+ pq.add(new Term("content", "content"));
+ assertEquals(numDocs2, s.search(pq, 1).totalHits);
+
+ pq = new PhraseQuery();
+ pq.add(new Term("content", "good"));
+ pq.add(new Term("content", "content"));
+ assertEquals(numDocs1+numDocs3+numDocs4, s.search(pq, 1).totalHits);
+ r.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java Tue May 31 11:25:37 2011
@@ -65,6 +65,10 @@ public class TestIndexWriterWithThreads
writer.updateDocument(new Term("id", ""+(idUpto++)), doc);
addCount++;
} catch (IOException ioe) {
+ if (VERBOSE) {
+ System.out.println("TEST: expected exc:");
+ ioe.printStackTrace(System.out);
+ }
//System.out.println(Thread.currentThread().getName() + ": hit exc");
//ioe.printStackTrace(System.out);
if (ioe.getMessage().startsWith("fake disk full at") ||
@@ -218,6 +222,9 @@ public class TestIndexWriterWithThreads
int NUM_THREADS = 3;
for(int iter=0;iter<2;iter++) {
+ if (VERBOSE) {
+ System.out.println("TEST: iter=" + iter);
+ }
MockDirectoryWrapper dir = newDirectory();
IndexWriter writer = new IndexWriter(
@@ -228,6 +235,7 @@ public class TestIndexWriterWithThreads
setMergePolicy(newLogMergePolicy(4))
);
((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions();
+ writer.setInfoStream(VERBOSE ? System.out : null);
IndexerThread[] threads = new IndexerThread[NUM_THREADS];
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java Tue May 31 11:25:37 2011
@@ -21,33 +21,35 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import java.util.HashSet;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.Executors;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.document.Field;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.MockDirectoryWrapper;
-import org.apache.lucene.util.NamedThreadFactory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.NamedThreadFactory;
import org.apache.lucene.util._TestUtil;
import org.junit.Test;
@@ -57,6 +59,39 @@ import org.junit.Test;
public class TestNRTThreads extends LuceneTestCase {
+ private static class SubDocs {
+ public final String packID;
+ public final List<String> subIDs;
+ public boolean deleted;
+
+ public SubDocs(String packID, List<String> subIDs) {
+ this.packID = packID;
+ this.subIDs = subIDs;
+ }
+ }
+
+ // TODO: is there a pre-existing way to do this!!!
+ private Document cloneDoc(Document doc1) {
+ final Document doc2 = new Document();
+ for(Fieldable f : doc1.getFields()) {
+ Field field1 = (Field) f;
+
+ Field field2 = new Field(field1.name(),
+ field1.stringValue(),
+ field1.isStored() ? Field.Store.YES : Field.Store.NO,
+ field1.isIndexed() ? (field1.isTokenized() ? Field.Index.ANALYZED : Field.Index.NOT_ANALYZED) : Field.Index.NO);
+ if (field1.getOmitNorms()) {
+ field2.setOmitNorms(true);
+ }
+ if (field1.getOmitTermFreqAndPositions()) {
+ field2.setOmitTermFreqAndPositions(true);
+ }
+ doc2.add(field2);
+ }
+
+ return doc2;
+ }
+
@Test
public void testNRTThreads() throws Exception {
@@ -121,13 +156,16 @@ public class TestNRTThreads extends Luce
final int NUM_INDEX_THREADS = 2;
final int NUM_SEARCH_THREADS = 3;
+
final int RUN_TIME_SEC = LuceneTestCase.TEST_NIGHTLY ? 300 : 5;
final AtomicBoolean failed = new AtomicBoolean();
final AtomicInteger addCount = new AtomicInteger();
final AtomicInteger delCount = new AtomicInteger();
+ final AtomicInteger packCount = new AtomicInteger();
final Set<String> delIDs = Collections.synchronizedSet(new HashSet<String>());
+ final List<SubDocs> allSubDocs = Collections.synchronizedList(new ArrayList<SubDocs>());
final long stopTime = System.currentTimeMillis() + RUN_TIME_SEC*1000;
Thread[] threads = new Thread[NUM_INDEX_THREADS];
@@ -135,7 +173,9 @@ public class TestNRTThreads extends Luce
threads[thread] = new Thread() {
@Override
public void run() {
+ // TODO: would be better if this were cross thread, so that we make sure one thread deleting anothers added docs works:
final List<String> toDeleteIDs = new ArrayList<String>();
+ final List<SubDocs> toDeleteSubDocs = new ArrayList<SubDocs>();
while(System.currentTimeMillis() < stopTime && !failed.get()) {
try {
Document doc = docs.nextDoc();
@@ -153,7 +193,92 @@ public class TestNRTThreads extends Luce
if (VERBOSE) {
//System.out.println(Thread.currentThread().getName() + ": add doc id:" + doc.get("docid"));
}
- writer.addDocument(doc);
+
+ if (random.nextBoolean()) {
+ // Add a pack of adjacent sub-docs
+ final String packID;
+ final SubDocs delSubDocs;
+ if (toDeleteSubDocs.size() > 0 && random.nextBoolean()) {
+ delSubDocs = toDeleteSubDocs.get(random.nextInt(toDeleteSubDocs.size()));
+ assert !delSubDocs.deleted;
+ toDeleteSubDocs.remove(delSubDocs);
+ // reuse prior packID
+ packID = delSubDocs.packID;
+ } else {
+ delSubDocs = null;
+ // make new packID
+ packID = packCount.getAndIncrement() + "";
+ }
+
+ final Field packIDField = newField("packID", packID, Field.Store.YES, Field.Index.NOT_ANALYZED);
+ final List<String> docIDs = new ArrayList<String>();
+ final SubDocs subDocs = new SubDocs(packID, docIDs);
+ final List<Document> docsList = new ArrayList<Document>();
+
+ allSubDocs.add(subDocs);
+ doc.add(packIDField);
+ docsList.add(cloneDoc(doc));
+ docIDs.add(doc.get("docid"));
+
+ final int maxDocCount = _TestUtil.nextInt(random, 1, 10);
+ while(docsList.size() < maxDocCount) {
+ doc = docs.nextDoc();
+ if (doc == null) {
+ break;
+ }
+ docsList.add(cloneDoc(doc));
+ docIDs.add(doc.get("docid"));
+ }
+ addCount.addAndGet(docsList.size());
+
+ if (delSubDocs != null) {
+ delSubDocs.deleted = true;
+ delIDs.addAll(delSubDocs.subIDs);
+ delCount.addAndGet(delSubDocs.subIDs.size());
+ if (VERBOSE) {
+ System.out.println("TEST: update pack packID=" + delSubDocs.packID + " count=" + docsList.size() + " docs=" + docIDs);
+ }
+ writer.updateDocuments(new Term("packID", delSubDocs.packID), docsList);
+ /*
+ // non-atomic:
+ writer.deleteDocuments(new Term("packID", delSubDocs.packID));
+ for(Document subDoc : docsList) {
+ writer.addDocument(subDoc);
+ }
+ */
+ } else {
+ if (VERBOSE) {
+ System.out.println("TEST: add pack packID=" + packID + " count=" + docsList.size() + " docs=" + docIDs);
+ }
+ writer.addDocuments(docsList);
+
+ /*
+ // non-atomic:
+ for(Document subDoc : docsList) {
+ writer.addDocument(subDoc);
+ }
+ */
+ }
+ doc.removeField("packID");
+
+ if (random.nextInt(5) == 2) {
+ if (VERBOSE) {
+ //System.out.println(Thread.currentThread().getName() + ": buffer del id:" + packID);
+ }
+ toDeleteSubDocs.add(subDocs);
+ }
+
+ } else {
+ writer.addDocument(doc);
+ addCount.getAndIncrement();
+
+ if (random.nextInt(5) == 3) {
+ if (VERBOSE) {
+ //System.out.println(Thread.currentThread().getName() + ": buffer del id:" + doc.get("docid"));
+ }
+ toDeleteIDs.add(doc.get("docid"));
+ }
+ }
} else {
// we use update but it never replaces a
// prior doc
@@ -161,14 +286,17 @@ public class TestNRTThreads extends Luce
//System.out.println(Thread.currentThread().getName() + ": update doc id:" + doc.get("docid"));
}
writer.updateDocument(new Term("docid", doc.get("docid")), doc);
- }
- if (random.nextInt(5) == 3) {
- if (VERBOSE) {
- //System.out.println(Thread.currentThread().getName() + ": buffer del id:" + doc.get("docid"));
+ addCount.getAndIncrement();
+
+ if (random.nextInt(5) == 3) {
+ if (VERBOSE) {
+ //System.out.println(Thread.currentThread().getName() + ": buffer del id:" + doc.get("docid"));
+ }
+ toDeleteIDs.add(doc.get("docid"));
}
- toDeleteIDs.add(doc.get("docid"));
}
- if (random.nextInt(50) == 17) {
+
+ if (random.nextInt(30) == 17) {
if (VERBOSE) {
//System.out.println(Thread.currentThread().getName() + ": apply " + toDeleteIDs.size() + " deletes");
}
@@ -184,8 +312,19 @@ public class TestNRTThreads extends Luce
}
delIDs.addAll(toDeleteIDs);
toDeleteIDs.clear();
+
+ for(SubDocs subDocs : toDeleteSubDocs) {
+ assert !subDocs.deleted;
+ writer.deleteDocuments(new Term("packID", subDocs.packID));
+ subDocs.deleted = true;
+ if (VERBOSE) {
+ System.out.println(" del subs: " + subDocs.subIDs + " packID=" + subDocs.packID);
+ }
+ delIDs.addAll(subDocs.subIDs);
+ delCount.addAndGet(subDocs.subIDs.size());
+ }
+ toDeleteSubDocs.clear();
}
- addCount.getAndIncrement();
if (addedField != null) {
doc.removeField(addedField);
}
@@ -356,7 +495,7 @@ public class TestNRTThreads extends Luce
if (VERBOSE) {
System.out.println("TEST: done join [" + (System.currentTimeMillis()-t0) + " ms]; addCount=" + addCount + " delCount=" + delCount);
}
-
+
final IndexReader r2 = writer.getReader();
final IndexSearcher s = newSearcher(r2);
boolean doFail = false;
@@ -367,6 +506,43 @@ public class TestNRTThreads extends Luce
doFail = true;
}
}
+
+ // Make sure each group of sub-docs are still in docID order:
+ for(SubDocs subDocs : allSubDocs) {
+ if (!subDocs.deleted) {
+ // We sort by relevance but the scores should be identical so sort falls back to by docID:
+ TopDocs hits = s.search(new TermQuery(new Term("packID", subDocs.packID)), 20);
+ assertEquals(subDocs.subIDs.size(), hits.totalHits);
+ int lastDocID = -1;
+ int startDocID = -1;
+ for(ScoreDoc scoreDoc : hits.scoreDocs) {
+ final int docID = scoreDoc.doc;
+ if (lastDocID != -1) {
+ assertEquals(1+lastDocID, docID);
+ } else {
+ startDocID = docID;
+ }
+ lastDocID = docID;
+ final Document doc = s.doc(docID);
+ assertEquals(subDocs.packID, doc.get("packID"));
+ }
+
+ lastDocID = startDocID - 1;
+ for(String subID : subDocs.subIDs) {
+ hits = s.search(new TermQuery(new Term("docid", subID)), 1);
+ assertEquals(1, hits.totalHits);
+ final int docID = hits.scoreDocs[0].doc;
+ if (lastDocID != -1) {
+ assertEquals(1+lastDocID, docID);
+ }
+ lastDocID = docID;
+ }
+ } else {
+ for(String subID : subDocs.subIDs) {
+ assertEquals(0, s.search(new TermQuery(new Term("docid", subID)), 1).totalHits);
+ }
+ }
+ }
final int endID = Integer.parseInt(docs.nextDoc().get("docid"));
for(int id=0;id<endID;id++) {
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java Tue May 31 11:25:37 2011
@@ -30,9 +30,13 @@ import org.junit.Before;
import org.junit.Test;
public class TestPersistentSnapshotDeletionPolicy extends TestSnapshotDeletionPolicy {
+
// Keep it a class member so that getDeletionPolicy can use it
private Directory snapshotDir;
+ // so we can close it if called by SDP tests
+ private PersistentSnapshotDeletionPolicy psdp;
+
@Before
@Override
public void setUp() throws Exception {
@@ -43,15 +47,17 @@ public class TestPersistentSnapshotDelet
@After
@Override
public void tearDown() throws Exception {
+ if (psdp != null) psdp.close();
snapshotDir.close();
super.tearDown();
}
@Override
protected SnapshotDeletionPolicy getDeletionPolicy() throws IOException {
+ if (psdp != null) psdp.close();
snapshotDir.close();
snapshotDir = newDirectory();
- return new PersistentSnapshotDeletionPolicy(
+ return psdp = new PersistentSnapshotDeletionPolicy(
new KeepOnlyLastCommitDeletionPolicy(), snapshotDir, OpenMode.CREATE,
TEST_VERSION_CURRENT);
}
@@ -173,6 +179,8 @@ public class TestPersistentSnapshotDelet
fail("should not have reached here - the snapshots directory should be locked!");
} catch (LockObtainFailedException e) {
// expected
+ } finally {
+ psdp.close();
}
// Reading the snapshots info should succeed though
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java Tue May 31 11:25:37 2011
@@ -20,8 +20,12 @@ package org.apache.lucene.index;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
-import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.util.BytesRef;
import java.io.IOException;
@@ -127,4 +131,50 @@ public class TestSegmentMerger extends L
TestSegmentReader.checkNorms(mergedReader);
mergedReader.close();
}
+
+ // LUCENE-3143
+ public void testInvalidFilesToCreateCompound() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
+ IndexWriter w = new IndexWriter(dir, iwc);
+
+ // Create an index w/ .del file
+ w.addDocument(new Document());
+ Document doc = new Document();
+ doc.add(new Field("c", "test", Store.NO, Index.ANALYZED));
+ w.addDocument(doc);
+ w.commit();
+ w.deleteDocuments(new Term("c", "test"));
+ w.close();
+
+ // Assert that SM fails if .del exists
+ SegmentMerger sm = new SegmentMerger(dir, 1, "a", null, null, null);
+ try {
+ sm.createCompoundFile("b1", w.segmentInfos.info(0));
+ fail("should not have been able to create a .cfs with .del and .s* files");
+ } catch (AssertionError e) {
+ // expected
+ }
+
+ // Create an index w/ .s*
+ w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));
+ doc = new Document();
+ doc.add(new Field("c", "test", Store.NO, Index.ANALYZED));
+ w.addDocument(doc);
+ w.close();
+ IndexReader r = IndexReader.open(dir, false);
+ r.setNorm(0, "c", (byte) 1);
+ r.close();
+
+ // Assert that SM fails if .s* exists
+ try {
+ sm.createCompoundFile("b2", w.segmentInfos.info(0));
+ fail("should not have been able to create a .cfs with .del and .s* files");
+ } catch (AssertionError e) {
+ // expected
+ }
+
+ dir.close();
+ }
+
}
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/search/TestFieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/search/TestFieldCache.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/search/TestFieldCache.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/search/TestFieldCache.java Tue May 31 11:25:37 2011
@@ -217,6 +217,7 @@ public class TestFieldCache extends Luce
IndexReader r = IndexReader.open(writer, true);
FieldCache.DocTerms terms = FieldCache.DEFAULT.getTerms(r, "foobar");
FieldCache.DocTermsIndex termsIndex = FieldCache.DEFAULT.getTermsIndex(r, "foobar");
+ writer.close();
r.close();
dir.close();
}
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java Tue May 31 11:25:37 2011
@@ -35,6 +35,7 @@ import org.apache.lucene.index.RandomInd
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util._TestUtil;
@@ -114,7 +115,7 @@ public class TestRegexpRandom2 extends L
private class SimpleAutomatonTermsEnum extends FilteredTermsEnum {
CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton);
- UnicodeUtil.UTF16Result utf16 = new UnicodeUtil.UTF16Result();
+ CharsRef utf16 = new CharsRef(10);
private SimpleAutomatonTermsEnum(TermsEnum tenum) throws IOException {
super(tenum);
@@ -124,7 +125,7 @@ public class TestRegexpRandom2 extends L
@Override
protected AcceptStatus accept(BytesRef term) throws IOException {
UnicodeUtil.UTF8toUTF16(term.bytes, term.offset, term.length, utf16);
- return runAutomaton.run(utf16.result, 0, utf16.length) ?
+ return runAutomaton.run(utf16.chars, 0, utf16.length) ?
AcceptStatus.YES : AcceptStatus.NO;
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java Tue May 31 11:25:37 2011
@@ -87,7 +87,7 @@ public class TestBufferedIndexInput exte
// NOTE: this does only test the chunked reads and NOT if the Bug is triggered.
//final int tmpFileSize = 1024 * 1024 * 5;
final int inputBufferSize = 128;
- File tmpInputFile = File.createTempFile("IndexInput", "tmpFile");
+ File tmpInputFile = _TestUtil.createTempFile("IndexInput", "tmpFile", TEMP_DIR);
tmpInputFile.deleteOnExit();
writeBytes(tmpInputFile, TEST_FILE_LENGTH);
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/store/TestLockFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/store/TestLockFactory.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/store/TestLockFactory.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/store/TestLockFactory.java Tue May 31 11:25:37 2011
@@ -255,15 +255,21 @@ public class TestLockFactory extends Luc
// write.lock is stored in index):
public void testDefaultFSLockFactoryPrefix() throws IOException {
- // Make sure we get null prefix:
+ // Make sure we get null prefix, which wont happen if setLockFactory is ever called.
File dirName = _TestUtil.getTempDir("TestLockFactory.10");
- Directory dir = newFSDirectory(dirName);
-
- String prefix = dir.getLockFactory().getLockPrefix();
-
- assertTrue("Default lock prefix should be null", null == prefix);
+ Directory dir = new SimpleFSDirectory(dirName);
+ assertNull("Default lock prefix should be null", dir.getLockFactory().getLockPrefix());
+ dir.close();
+
+ dir = new MMapDirectory(dirName);
+ assertNull("Default lock prefix should be null", dir.getLockFactory().getLockPrefix());
+ dir.close();
+
+ dir = new NIOFSDirectory(dirName);
+ assertNull("Default lock prefix should be null", dir.getLockFactory().getLockPrefix());
dir.close();
+
_TestUtil.rmDir(dirName);
}
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/store/TestMultiMMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/store/TestMultiMMap.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/store/TestMultiMMap.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/store/TestMultiMMap.java Tue May 31 11:25:37 2011
@@ -51,7 +51,7 @@ public class TestMultiMMap extends Lucen
}
private void assertChunking(Random random, int chunkSize) throws Exception {
- File path = File.createTempFile("mmap" + chunkSize, "tmp", workDir);
+ File path = _TestUtil.createTempFile("mmap" + chunkSize, "tmp", workDir);
path.delete();
path.mkdirs();
MMapDirectory dir = new MMapDirectory(path);
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/util/TestUnicodeUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/util/TestUnicodeUtil.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/util/TestUnicodeUtil.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/util/TestUnicodeUtil.java Tue May 31 11:25:37 2011
@@ -85,37 +85,6 @@ package org.apache.lucene.util;
*/
public class TestUnicodeUtil extends LuceneTestCase {
- public void testNextValidUTF16String() {
- // valid UTF-16
- assertEquals("dogs", UnicodeUtil.nextValidUTF16String("dogs"));
- assertEquals("dogs\uD802\uDC02", UnicodeUtil
- .nextValidUTF16String("dogs\uD802\uDC02"));
-
- // an illegal combination, where we have not yet enumerated into the supp
- // plane so we increment to H + \uDC00 (the lowest possible trail surrogate)
- assertEquals("dogs\uD801\uDC00", UnicodeUtil
- .nextValidUTF16String("dogs\uD801"));
- assertEquals("dogs\uD801\uDC00", UnicodeUtil
- .nextValidUTF16String("dogs\uD801b"));
- assertEquals("dogs\uD801\uDC00", UnicodeUtil
- .nextValidUTF16String("dogs\uD801\uD800"));
-
- // an illegal combination where we have already enumerated the trail
- // we must increment the lead and start the trail back at the beginning.
- assertEquals("dogs\uD802\uDC00", UnicodeUtil
- .nextValidUTF16String("dogs\uD801\uE001"));
-
- // an illegal combination where we have exhausted the supp plane
- // we must now move to the lower bmp.
- assertEquals("dogs\uE000", UnicodeUtil
- .nextValidUTF16String("dogs\uDBFF\uE001"));
-
- // an unpaired trail surrogate. this is invalid when not preceded by a lead
- // surrogate. in this case we have to bump to \uE000 (the lowest possible
- // "upper BMP")
- assertEquals("dogs\uE000", UnicodeUtil.nextValidUTF16String("dogs\uDC00"));
- assertEquals("\uE000", UnicodeUtil.nextValidUTF16String("\uDC00dogs"));
- }
public void testCodePointCount() {
BytesRef utf8 = new BytesRef(20);
@@ -197,4 +166,19 @@ public class TestUnicodeUtil extends Luc
assertTrue(rc == -1);
}
}
+
+ public void testUTF8UTF16CharsRef() {
+ for (int i = 0; i < 3989 * RANDOM_MULTIPLIER; i++) {
+ String unicode = _TestUtil.randomRealisticUnicodeString(random);
+ BytesRef ref = new BytesRef(unicode);
+ char[] arr = new char[1 + random.nextInt(100)];
+ int offset = random.nextInt(arr.length);
+ int len = random.nextInt(arr.length - offset);
+ CharsRef cRef = new CharsRef(arr, offset, len);
+ UnicodeUtil.UTF8toUTF16(ref, cRef);
+ assertEquals(cRef.toString(), unicode);
+ assertEquals(cRef, unicode); // CharSeq
+ assertEquals(cRef, ref.utf8ToString()); // CharSeq
+ }
+ }
}
Modified: lucene/dev/branches/docvalues/modules/analysis/LICENSE.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/analysis/LICENSE.txt?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/analysis/LICENSE.txt (original)
+++ lucene/dev/branches/docvalues/modules/analysis/LICENSE.txt Tue May 31 11:25:37 2011
@@ -200,3 +200,32 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+
+The following license applies to the Snowball stemmers:
+
+Copyright (c) 2001, Dr Martin Porter
+Copyright (c) 2002, Richard Boulton
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * Neither the name of the copyright holders nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Modified: lucene/dev/branches/docvalues/modules/analysis/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/analysis/build.xml?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/analysis/build.xml (original)
+++ lucene/dev/branches/docvalues/modules/analysis/build.xml Tue May 31 11:25:37 2011
@@ -57,6 +57,13 @@
<ant dir="smartcn" target="clean" />
<ant dir="stempel" target="clean" />
</target>
+ <target name="validate">
+ <ant dir="common" target="validate" />
+ <ant dir="icu" target="validate" />
+ <ant dir="phonetic" target="validate" />
+ <ant dir="smartcn" target="validate" />
+ <ant dir="stempel" target="validate" />
+ </target>
<target name="compile-core">
<ant dir="common" target="compile-core" />
<ant dir="icu" target="compile-core" />
Modified: lucene/dev/branches/docvalues/modules/analysis/common/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/analysis/common/build.xml?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/analysis/common/build.xml (original)
+++ lucene/dev/branches/docvalues/modules/analysis/common/build.xml Tue May 31 11:25:37 2011
@@ -106,7 +106,7 @@
</java>
</target>
- <target name="compile-tools">
+ <target name="compile-tools" depends="common.compile-tools">
<compile
srcdir="src/tools/java"
destdir="${build.dir}/classes/tools">
Modified: lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java (original)
+++ lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java Tue May 31 11:25:37 2011
@@ -24,6 +24,7 @@ import org.apache.lucene.index.MultiFiel
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.StopFilter;
+import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.Version;
import org.apache.lucene.util.BytesRef;
@@ -143,13 +144,14 @@ public final class QueryAutoStopWordAnal
*/
public int addStopWords(IndexReader reader, String fieldName, int maxDocFreq) throws IOException {
HashSet<String> stopWords = new HashSet<String>();
- Terms terms = MultiFields.getTerms(reader, fieldName);
+ final Terms terms = MultiFields.getTerms(reader, fieldName);
+ final CharsRef spare = new CharsRef();
if (terms != null) {
- TermsEnum te = terms.iterator();
+ final TermsEnum te = terms.iterator();
BytesRef text;
while ((text = te.next()) != null) {
if (te.docFreq() > maxDocFreq) {
- stopWords.add(text.utf8ToString());
+ stopWords.add(text.utf8ToChars(spare).toString());
}
}
}
Modified: lucene/dev/branches/docvalues/modules/analysis/icu/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/analysis/icu/build.xml?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/analysis/icu/build.xml (original)
+++ lucene/dev/branches/docvalues/modules/analysis/icu/build.xml Tue May 31 11:25:37 2011
@@ -125,7 +125,7 @@ are part of the ICU4C package. See http:
</java>
</target>
- <target name="compile-tools">
+ <target name="compile-tools" depends="common.compile-tools">
<compile
srcdir="src/tools/java"
destdir="${build.dir}/classes/tools">
Modified: lucene/dev/branches/docvalues/modules/benchmark/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/benchmark/CHANGES.txt?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/benchmark/CHANGES.txt (original)
+++ lucene/dev/branches/docvalues/modules/benchmark/CHANGES.txt Tue May 31 11:25:37 2011
@@ -2,6 +2,9 @@ Lucene Benchmark Contrib Change Log
The Benchmark contrib package contains code for benchmarking Lucene in a variety of ways.
+05/25/2011
+ LUCENE-3137: ExtractReuters supports out-dir param suffixed by a slash. (Doron Cohen)
+
03/31/2011
Updated ReadTask to the new method for obtaining a top-level deleted docs
bitset. Also checking the bitset for null, when there are no deleted docs.
Modified: lucene/dev/branches/docvalues/modules/benchmark/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/benchmark/build.xml?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/benchmark/build.xml (original)
+++ lucene/dev/branches/docvalues/modules/benchmark/build.xml Tue May 31 11:25:37 2011
@@ -1,4 +1,22 @@
<?xml version="1.0"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
<project name="benchmark" default="default">
<description>
Modified: lucene/dev/branches/docvalues/modules/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java (original)
+++ lucene/dev/branches/docvalues/modules/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java Tue May 31 11:25:37 2011
@@ -122,17 +122,19 @@ public class ExtractReuters {
public static void main(String[] args) {
if (args.length != 2) {
- printUsage();
+ usage("Wrong number of arguments ("+args.length+")");
+ return;
}
File reutersDir = new File(args[0]);
if (!reutersDir.exists()) {
- printUsage();
+ usage("Cannot find Path to Reuters SGM files ("+reutersDir+")");
return;
}
// First, extract to a tmp directory and only if everything succeeds, rename
// to output directory.
- File outputDir = new File(args[1] + "-tmp");
+ File outputDir = new File(args[1]);
+ outputDir = new File(outputDir.getAbsolutePath() + "-tmp");
outputDir.mkdirs();
ExtractReuters extractor = new ExtractReuters(reutersDir, outputDir);
extractor.extract();
@@ -140,8 +142,8 @@ public class ExtractReuters {
outputDir.renameTo(new File(args[1]));
}
- private static void printUsage() {
- System.err.println("Usage: java -cp <...> org.apache.lucene.benchmark.utils.ExtractReuters <Path to Reuters SGM files> <Output Path>");
+ private static void usage(String msg) {
+ System.err.println("Usage: "+msg+" :: java -cp <...> org.apache.lucene.benchmark.utils.ExtractReuters <Path to Reuters SGM files> <Output Path>");
}
}
Modified: lucene/dev/branches/docvalues/modules/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/build.xml?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/build.xml (original)
+++ lucene/dev/branches/docvalues/modules/build.xml Tue May 31 11:25:37 2011
@@ -18,23 +18,24 @@
-->
<project name="modules" default="test" basedir=".">
- <import file="../common-build.xml"/>
<target name="test" description="Test all modules">
<sequential>
<subant target="test" inheritall="false" failonerror="true">
<fileset dir="analysis" includes="build.xml" />
<fileset dir="benchmark" includes="build.xml" />
<fileset dir="grouping" includes="build.xml" />
+ <fileset dir="suggest" includes="build.xml" />
</subant>
</sequential>
</target>
- <target name="compile" description="Compile all modules" depends="validate-modules">
+ <target name="compile" description="Compile all modules" depends="validate">
<sequential>
<subant target="compile" inheritall="false" failonerror="true">
<fileset dir="analysis" includes="build.xml" />
<fileset dir="benchmark" includes="build.xml" />
<fileset dir="grouping" includes="build.xml" />
+ <fileset dir="suggest" includes="build.xml" />
</subant>
</sequential>
</target>
@@ -45,6 +46,7 @@
<fileset dir="analysis" includes="build.xml" />
<fileset dir="benchmark" includes="build.xml" />
<fileset dir="grouping" includes="build.xml" />
+ <fileset dir="suggest" includes="build.xml" />
</subant>
</sequential>
</target>
@@ -55,6 +57,7 @@
<fileset dir="analysis" includes="build.xml" />
<fileset dir="benchmark" includes="build.xml" />
<fileset dir="grouping" includes="build.xml" />
+ <fileset dir="suggest" includes="build.xml" />
</subant>
</sequential>
</target>
@@ -66,27 +69,19 @@
<fileset dir="analysis" includes="build.xml" />
<fileset dir="benchmark" includes="build.xml" />
<fileset dir="grouping" includes="build.xml" />
+ <fileset dir="suggest" includes="build.xml" />
</subant>
</sequential>
</target>
- <target name="validate" depends="validate-modules"/>
- <target name="validate-modules" depends="check-legal-modules" unless="validated-modules"/>
- <target name="check-legal-modules" depends="compile-tools">
- <java classname="org.apache.lucene.validation.DependencyChecker" failonerror="true" fork="true">
- <classpath>
- <path refid="tools.runtime.classpath" />
- </classpath>
- <!-- TODO: it might be better to just automatically find all directories that contain jar files, but that could take a
- long time. This should be faster, but we could miss a directory
- -->
- <!-- Modules -->
- <arg value="-c" />
- <arg value="${basedir}/analysis/icu/lib" />
- <arg value="-c" />
- <arg value="${basedir}/analysis/phonetic/lib" />
- <arg value="-c" />
- <arg value="${basedir}/benchmark/lib" />
- </java>
+ <target name="validate">
+ <sequential>
+ <subant target="validate" inheritall="false" failonerror="true">
+ <fileset dir="analysis" includes="build.xml" />
+ <fileset dir="benchmark" includes="build.xml" />
+ <fileset dir="grouping" includes="build.xml" />
+ <fileset dir="suggest" includes="build.xml" />
+ </subant>
+ </sequential>
</target>
<target name="clean" description="Clean all modules">
@@ -96,6 +91,7 @@
<fileset dir="analysis" includes="build.xml" />
<fileset dir="benchmark" includes="build.xml" />
<fileset dir="grouping" includes="build.xml" />
+ <fileset dir="suggest" includes="build.xml" />
</subant>
</sequential>
</target>
Modified: lucene/dev/branches/docvalues/modules/grouping/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/grouping/build.xml?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/grouping/build.xml (original)
+++ lucene/dev/branches/docvalues/modules/grouping/build.xml Tue May 31 11:25:37 2011
@@ -1,4 +1,22 @@
<?xml version="1.0"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
<project name="grouping" default="default">
<description>
Collectors for grouping search results
Modified: lucene/dev/branches/docvalues/modules/grouping/src/java/org/apache/lucene/search/grouping/AllGroupsCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/grouping/src/java/org/apache/lucene/search/grouping/AllGroupsCollector.java?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/grouping/src/java/org/apache/lucene/search/grouping/AllGroupsCollector.java (original)
+++ lucene/dev/branches/docvalues/modules/grouping/src/java/org/apache/lucene/search/grouping/AllGroupsCollector.java Tue May 31 11:25:37 2011
@@ -88,7 +88,7 @@ public class AllGroupsCollector extends
int key = index.getOrd(doc);
if (!ordSet.exists(key)) {
ordSet.put(key);
- BytesRef term = key == 0 ? null : index.getTerm(doc, new BytesRef());
+ BytesRef term = key == 0 ? null : index.lookup(key, new BytesRef());
groups.add(term);
}
}
Modified: lucene/dev/branches/docvalues/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/CHANGES.txt?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/CHANGES.txt (original)
+++ lucene/dev/branches/docvalues/solr/CHANGES.txt Tue May 31 11:25:37 2011
@@ -24,14 +24,13 @@ $Id$
================== 4.0.0-dev ==================
Versions of Major Components
---------------------
-Apache Lucene trunk
Apache Tika 0.8
Carrot2 3.5.0
Velocity 1.6.4 and Velocity Tools 2.0
Apache UIMA 2.3.1-SNAPSHOT
-Upgrading from Solr 3.2-dev
+Upgrading from Solr 3.3-dev
----------------------
* The Lucene index format has changed and as a result, once you upgrade,
@@ -144,6 +143,11 @@ New Features
fq={!join from=name to=parent}eyes:blue
(yonik)
+* SOLR-1942: Added the ability to select codec per fieldType in schema.xml
+ as well as support custom CodecProviders in solrconfig.xml.
+ NOTE: IndexReaderFactory now has a codecProvider that should be passed
+ to IndexReader.open (in the case you have a custom IndexReaderFactory).
+ (simonw via rmuir)
Optimizations
@@ -246,15 +250,27 @@ Other Changes
variance in asserting score comparisons in unit tests.
(David Smiley, Chris Hostetter)
+* LUCENE-2995: Moved some spellchecker and suggest APIs to modules/suggest:
+ HighFrequencyDictionary, SortedIterator, TermFreqIterator, and the
+ suggester APIs and implementations. (rmuir)
+
Documentation
----------------------
* SOLR-2232: Improved README info on solr.solr.home in examples
(Eric Pugh and hossman)
-* LUCENE-3006: Building javadocs will fail on warnings by default. Override with -Dfailonjavadocwarning=false (sarowe, gsingers)
+======================= 3.x (not yet released) ================
+
+Bug Fixes
+----------------------
-================== 3.2.0-dev ==================
+* SOLR-2519: Improve text_* fieldTypes in example schema.xml: improve
+ cross-language defaults for text_general; break out separate
+ English-specific fieldTypes (Jan Høydahl, hossman, Robert Muir,
+ yonik, Mike McCandless)
+
+================== 3.2.0 ==================
Versions of Major Components
---------------------
Apache Lucene trunk
@@ -333,6 +349,12 @@ Bug Fixes
in strings since those characters are not valid in javascript strings
(although they are valid in JSON strings). (yonik)
+* SOLR-2536: Add ReloadCacheRequestHandler to fix ExternalFileField bug (if reopenReaders
+ set to true and no index segments have been changed, commit cannot trigger reload
+ external file). (koji)
+
+* SOLR-2539: VectorValueSource.floatVal incorrectly used byteVal on sub-sources.
+ (Tom Liu via yonik)
Other Changes
----------------------
@@ -346,6 +368,9 @@ Other Changes
Build
----------------------
+* LUCENE-3006: Building javadocs will fail on warnings by default. Override with -Dfailonjavadocwarning=false (sarowe, gsingers)
+
+
Documentation
----------------------
@@ -1804,7 +1829,7 @@ Documentation
3. SOLR-1409: Added Solr Powered By Logos
-================== Release 1.3.0 20080915 ==================
+================== Release 1.3.0 ==================
Upgrading from Solr 1.2
-----------------------
@@ -2465,7 +2490,7 @@ Build
client and contrib. The target can publish artifacts with source and javadocs.
(Spencer Crissman, Craig McClanahan, shalin)
-================== Release 1.2, 20070602 ==================
+================== Release 1.2 ==================
Upgrading from Solr 1.1
-------------------------------------
@@ -2814,7 +2839,7 @@ Other Changes
2. Updated to Lucene 2007-05-20_00-04-53
-================== Release 1.1.0, 20061222 ==================
+================== Release 1.1.0 ==================
Status
------
Modified: lucene/dev/branches/docvalues/solr/LICENSE.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/LICENSE.txt?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/LICENSE.txt (original)
+++ lucene/dev/branches/docvalues/solr/LICENSE.txt Tue May 31 11:25:37 2011
@@ -1067,3 +1067,32 @@ Definitions.
10. RESPONSIBILITY FOR CLAIMS. As between Initial Developer and the Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License and You agree to work with Initial Developer and Contributors to distribute such responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute any admission of liability.
NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) The code released under the CDDL shall be governed by the laws of the State of California (excluding conflict-of-law provisions). Any litigation relating to this License shall be subject to the jurisdiction of the Federal Courts of the Northern District of California and the state courts of the State of California, with venue lying in Santa Clara County, California.
+
+The following license applies to the Snowball stemmers:
+
+Copyright (c) 2001, Dr Martin Porter
+Copyright (c) 2002, Richard Boulton
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * Neither the name of the copyright holders nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Modified: lucene/dev/branches/docvalues/solr/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/build.xml?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/build.xml (original)
+++ lucene/dev/branches/docvalues/solr/build.xml Tue May 31 11:25:37 2011
@@ -859,7 +859,7 @@
<tar destfile="${package.dir}/${fullnamever}.tgz" compression="gzip" longfile="gnu">
<tarfileset dir="."
prefix="${fullnamever}"
- includes="LICENSE.txt NOTICE.txt *.txt example/** client/** contrib/**/lib/ contrib/**/lucene-libs/ contrib/**/README.txt contrib/**/CHANGES.txt"
+ includes="LICENSE.txt NOTICE.txt CHANGES.txt README.txt example/** client/** contrib/**/lib/ contrib/**/lucene-libs/ contrib/**/README.txt contrib/**/CHANGES.txt"
excludes="${dist}/** ${dest}/** lib/README.committers.txt **/data/ **/logs/* **/classes/ **/*.sh **/bin/ src/scripts/** src/site/build/** **/target/** client/ruby/flare/** client/python/** client/javascript/** contrib/**/build/** **/*.iml **/*.ipr **/*.iws **/*pom.xml.template" />
<tarfileset dir="."
mode="755"
Modified: lucene/dev/branches/docvalues/solr/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/common-build.xml?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/common-build.xml (original)
+++ lucene/dev/branches/docvalues/solr/common-build.xml Tue May 31 11:25:37 2011
@@ -22,7 +22,6 @@
</description>
<dirname file="${ant.file.common-solr}" property="common-solr.dir"/>
- <import file="${common-solr.dir}/../common-build.xml"/>
<!-- change this together with the default and test's solrconfig.xml after starting a new development branch: -->
<property name="tests.luceneMatchVersion" value="4.0"/>
@@ -188,12 +187,12 @@
<pathelement location="${common-solr.dir}/../lucene/build/classes/java" />
<pathelement location="${common-solr.dir}/../modules/analysis/build/common/classes/java" />
<pathelement location="${common-solr.dir}/../modules/analysis/build/phonetic/classes/java" />
+ <pathelement location="${common-solr.dir}/../modules/suggest/build/classes/java" />
<pathelement location="${common-solr.dir}/../lucene/build/contrib/highlighter/classes/java" />
<pathelement location="${common-solr.dir}/../lucene/build/contrib/memory/classes/java" />
<pathelement location="${common-solr.dir}/../lucene/build/contrib/misc/classes/java" />
<pathelement location="${common-solr.dir}/../lucene/build/contrib/queries/classes/java" />
<pathelement location="${common-solr.dir}/../lucene/build/contrib/spatial/classes/java" />
- <pathelement location="${common-solr.dir}/../lucene/build/contrib/spellchecker/classes/java" />
</path>
<target name="prep-lucene-jars">
@@ -204,12 +203,12 @@
<subant target="jar" inheritall="false" failonerror="true">
<fileset dir="../modules/analysis/common" includes="build.xml" />
<fileset dir="../modules/analysis/phonetic" includes="build.xml" />
+ <fileset dir="../modules/suggest" includes="build.xml" />
<fileset dir="../lucene/contrib/highlighter" includes="build.xml" />
<fileset dir="../lucene/contrib/memory" includes="build.xml" />
<fileset dir="../lucene/contrib/misc" includes="build.xml" />
<fileset dir="../lucene/contrib/queries" includes="build.xml" />
<fileset dir="../lucene/contrib/spatial" includes="build.xml" />
- <fileset dir="../lucene/contrib/spellchecker" includes="build.xml" />
</subant>
</sequential>
</target>
@@ -226,6 +225,9 @@
<fileset dir="../modules/analysis/build/phonetic">
<include name="lucene-analyzers-phonetic-${version}.jar" />
</fileset>
+ <fileset dir="../modules/suggest/build">
+ <include name="lucene-suggest-${version}.jar" />
+ </fileset>
<fileset dir="../lucene/build/contrib/highlighter">
<include name="lucene-highlighter-${version}.jar" />
</fileset>
@@ -241,9 +243,6 @@
<fileset dir="../lucene/build/contrib/spatial">
<include name="lucene-spatial-${version}.jar" />
</fileset>
- <fileset dir="../lucene/build/contrib/spellchecker">
- <include name="lucene-spellchecker-${version}.jar" />
- </fileset>
</copy>
</target>
@@ -252,12 +251,12 @@
<subant target="default">
<fileset dir="../modules/analysis/common" includes="build.xml"/>
<fileset dir="../modules/analysis/phonetic" includes="build.xml"/>
+ <fileset dir="../modules/suggest" includes="build.xml"/>
<fileset dir="../lucene/contrib/highlighter" includes="build.xml"/>
<fileset dir="../lucene/contrib/memory" includes="build.xml"/>
<fileset dir="../lucene/contrib/misc" includes="build.xml"/>
<fileset dir="../lucene/contrib/queries" includes="build.xml"/>
<fileset dir="../lucene/contrib/spatial" includes="build.xml"/>
- <fileset dir="../lucene/contrib/spellchecker" includes="build.xml"/>
</subant>
</target>
@@ -636,4 +635,16 @@
</java>
</target>
+ <path id="tools.runtime.classpath">
+ <pathelement location="${common-solr.dir}/../lucene/build/classes/tools"/>
+ </path>
+ <target name="compile-tools" description="Compile the Test Framework and Validation tools">
+ <sequential>
+ <subant target="compile-tools" inheritall="false" failonerror="true">
+ <fileset dir="${common-solr.dir}/../lucene" includes="build.xml" />
+ </subant>
+ </sequential>
+ </target>
+
+
</project>
Modified: lucene/dev/branches/docvalues/solr/contrib/analysis-extras/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/analysis-extras/CHANGES.txt?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/analysis-extras/CHANGES.txt (original)
+++ lucene/dev/branches/docvalues/solr/contrib/analysis-extras/CHANGES.txt Tue May 31 11:25:37 2011
@@ -11,17 +11,21 @@ analyzers for Chinese and Polish.
$Id$
-================== Release 4.0-dev ==================
+================== 4.0.0-dev ==============
* SOLR-2396: Add ICUCollationField, which is much more efficient than
the Solr 3.x ICUCollationKeyFilterFactory, and also supports
Locale-sensitive range queries. (rmuir)
-================== Release 3.2-dev ==================
+================== 3.3.0-dev ==============
(No Changes)
-================== Release 3.1-dev ==================
+================== 3.2.0 ==================
+
+(No Changes)
+
+================== 3.1.0 ==================
* SOLR-2210: Add icu-based tokenizer and filters to contrib/analysis-extras (rmuir)
Modified: lucene/dev/branches/docvalues/solr/contrib/clustering/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/clustering/CHANGES.txt?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/clustering/CHANGES.txt (original)
+++ lucene/dev/branches/docvalues/solr/contrib/clustering/CHANGES.txt Tue May 31 11:25:37 2011
@@ -7,7 +7,7 @@ See http://wiki.apache.org/solr/Clusteri
CHANGES
$Id$
-================== Release 4.0.0-dev ==================
+================== Release 4.0.0-dev ==============
* SOLR-2448: Search results clustering updates: bisecting k-means
clustering algorithm added, loading of Carrot2 stop words from
@@ -15,7 +15,11 @@ $Id$
for clustering (SOLR-2450), output of cluster scores (SOLR-2505)
(Stanislaw Osinski, Dawid Weiss).
-================== Release 3.2.0-dev ==================
+================== Release 3.3.0-dev ==============
+
+(No Changes)
+
+================== Release 3.2.0 ==================
* SOLR-2448: Search results clustering updates: bisecting k-means
clustering algorithm added, loading of Carrot2 stop words from
@@ -23,7 +27,7 @@ $Id$
for clustering (SOLR-2450), output of cluster scores (SOLR-2505)
(Stanislaw Osinski, Dawid Weiss).
-================== Release 3.1.0-dev ==================
+================== Release 3.1.0 ==================
* SOLR-1684: Switch to use the SolrIndexSearcher.doc(int, Set<String>) method b/c it can use the document cache (gsingers)
Modified: lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/CHANGES.txt?rev=1129631&r1=1129630&r2=1129631&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/CHANGES.txt Tue May 31 11:25:37 2011
@@ -8,15 +8,19 @@ HTTP data sources quick and easy.
$Id$
-================== 4.0.0-dev ==================
+================== 4.0.0-dev ==============
(No Changes)
-================== 3.2.0-dev ==================
+================== 3.3.0-dev ==============
(No Changes)
-================== 3.1.0-dev ==================
+================== 3.2.0 ==================
+
+(No Changes)
+
+================== 3.1.0 ==================
Upgrading from Solr 1.4
----------------------
@@ -440,7 +444,7 @@ Other
11.SOLR-1269: Better error messages from JdbcDataSource when JDBC Driver name or SQL is incorrect.
(ehatcher, shalin)
-================== Release 1.3.0 20080915 ==================
+================== Release 1.3.0 ==================
Status
------