You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/05/02 16:44:57 UTC
svn commit: r1098604 - in /lucene/dev/branches/docvalues: ./ lucene/
lucene/contrib/queries/src/test/org/apache/lucene/search/regex/
lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/builders/
lucene/src/java/org/apache/lucene/index/...
Author: simonw
Date: Mon May 2 14:44:57 2011
New Revision: 1098604
URL: http://svn.apache.org/viewvc?rev=1098604&view=rev
Log:
merged with trunk
Modified:
lucene/dev/branches/docvalues/ (props changed)
lucene/dev/branches/docvalues/lucene/ (props changed)
lucene/dev/branches/docvalues/lucene/CHANGES.txt
lucene/dev/branches/docvalues/lucene/contrib/queries/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java (props changed)
lucene/dev/branches/docvalues/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/builders/TestNumericRangeFilterBuilder.java (props changed)
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java
lucene/dev/branches/docvalues/modules/analysis/common/src/test/org/apache/lucene/collation/CollationTestBase.java
lucene/dev/branches/docvalues/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java
lucene/dev/branches/docvalues/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java
lucene/dev/branches/docvalues/solr/ (props changed)
Modified: lucene/dev/branches/docvalues/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/CHANGES.txt?rev=1098604&r1=1098603&r2=1098604&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/docvalues/lucene/CHANGES.txt Mon May 2 14:44:57 2011
@@ -280,6 +280,10 @@ API Changes
* LUCENE-2953: In addition to changes in 3.x, PriorityQueue#initialize(int)
function was moved into the ctor. (Uwe Schindler, Yonik Seeley)
+* LUCENE-3061: IndexWriter's getNextMerge() and merge(OneMerge) are now public
+ (though @lucene.experimental), allowing for custom MergeScheduler
+ implementations. (Shai Erera)
+
New features
* LUCENE-2604: Added RegexpQuery support to QueryParser. Regular expressions
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1098604&r1=1098603&r2=1098604&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java Mon May 2 14:44:57 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Collection;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -30,8 +31,7 @@ import org.apache.lucene.index.Documents
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.codecs.docvalues.DocValuesConsumer;
-import org.apache.lucene.index.values.PerDocFieldValues;
-import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.ArrayUtil;
/**
@@ -262,7 +262,7 @@ final class DocFieldProcessor extends Do
// sort the subset of fields that have vectors
// enabled; we could save [small amount of] CPU
// here.
- quickSort(fields, 0, fieldCount-1);
+ ArrayUtil.quickSort(fields, 0, fieldCount, fieldsComp);
for(int i=0;i<fieldCount;i++)
fields[i].consumer.processFields(fields[i].fields, fields[i].fieldCount);
@@ -273,6 +273,12 @@ final class DocFieldProcessor extends Do
}
}
+ private static final Comparator<DocFieldProcessorPerField> fieldsComp = new Comparator<DocFieldProcessorPerField>() {
+ public int compare(DocFieldProcessorPerField o1, DocFieldProcessorPerField o2) {
+ return o1.fieldInfo.name.compareTo(o2.fieldInfo.name);
+ }
+ };
+
@Override
void finishDocument() throws IOException {
try {
@@ -282,66 +288,6 @@ final class DocFieldProcessor extends Do
}
}
- void quickSort(DocFieldProcessorPerField[] array, int lo, int hi) {
- if (lo >= hi)
- return;
- else if (hi == 1+lo) {
- if (array[lo].fieldInfo.name.compareTo(array[hi].fieldInfo.name) > 0) {
- final DocFieldProcessorPerField tmp = array[lo];
- array[lo] = array[hi];
- array[hi] = tmp;
- }
- return;
- }
-
- int mid = (lo + hi) >>> 1;
-
- if (array[lo].fieldInfo.name.compareTo(array[mid].fieldInfo.name) > 0) {
- DocFieldProcessorPerField tmp = array[lo];
- array[lo] = array[mid];
- array[mid] = tmp;
- }
-
- if (array[mid].fieldInfo.name.compareTo(array[hi].fieldInfo.name) > 0) {
- DocFieldProcessorPerField tmp = array[mid];
- array[mid] = array[hi];
- array[hi] = tmp;
-
- if (array[lo].fieldInfo.name.compareTo(array[mid].fieldInfo.name) > 0) {
- DocFieldProcessorPerField tmp2 = array[lo];
- array[lo] = array[mid];
- array[mid] = tmp2;
- }
- }
-
- int left = lo + 1;
- int right = hi - 1;
-
- if (left >= right)
- return;
-
- DocFieldProcessorPerField partition = array[mid];
-
- for (; ;) {
- while (array[right].fieldInfo.name.compareTo(partition.fieldInfo.name) > 0)
- --right;
-
- while (left < right && array[left].fieldInfo.name.compareTo(partition.fieldInfo.name) <= 0)
- ++left;
-
- if (left < right) {
- DocFieldProcessorPerField tmp = array[left];
- array[left] = array[right];
- array[right] = tmp;
- --right;
- } else {
- break;
- }
- }
-
- quickSort(array, lo, left);
- quickSort(array, left + 1, hi);
- }
final private Map<String, DocValuesConsumer> docValues = new HashMap<String, DocValuesConsumer>();
final private Map<Integer, PerDocConsumer> perDocConsumers = new HashMap<Integer, PerDocConsumer>();
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1098604&r1=1098603&r2=1098604&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexWriter.java Mon May 2 14:44:57 2011
@@ -1813,10 +1813,13 @@ public class IndexWriter implements Clos
return mergingSegments;
}
- /** Expert: the {@link MergeScheduler} calls this method
- * to retrieve the next merge requested by the
- * MergePolicy */
- synchronized MergePolicy.OneMerge getNextMerge() {
+ /**
+ * Expert: the {@link MergeScheduler} calls this method to retrieve the next
+ * merge requested by the MergePolicy
+ *
+ * @lucene.experimental
+ */
+ public synchronized MergePolicy.OneMerge getNextMerge() {
if (pendingMerges.size() == 0)
return null;
else {
@@ -2936,9 +2939,10 @@ public class IndexWriter implements Clos
/**
* Merges the indicated segments, replacing them in the stack with a
* single segment.
+ *
+ * @lucene.experimental
*/
-
- final void merge(MergePolicy.OneMerge merge)
+ public final void merge(MergePolicy.OneMerge merge)
throws CorruptIndexException, IOException {
boolean success = false;
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java?rev=1098604&r1=1098603&r2=1098604&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java Mon May 2 14:44:57 2011
@@ -19,17 +19,21 @@ package org.apache.lucene;
import java.io.IOException;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LogMergePolicy;
import org.apache.lucene.index.MergePolicy;
import org.apache.lucene.index.ConcurrentMergeScheduler;
+import org.apache.lucene.index.MergeScheduler;
+import org.apache.lucene.index.MergePolicy.OneMerge;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-
/**
* Holds tests cases to verify external APIs are accessible
* while not being in org.apache.lucene.index package.
@@ -106,4 +110,40 @@ public class TestMergeSchedulerExternal
assertTrue(excCalled);
dir.close();
}
+
+ private static class ReportingMergeScheduler extends MergeScheduler {
+
+ @Override
+ public void merge(IndexWriter writer) throws CorruptIndexException, IOException {
+ OneMerge merge = null;
+ while ((merge = writer.getNextMerge()) != null) {
+ if (VERBOSE) {
+ System.out.println("executing merge " + merge.segString(writer.getDirectory()));
+ }
+ writer.merge(merge);
+ }
+ }
+
+ @Override
+ public void close() throws CorruptIndexException, IOException {}
+
+ }
+
+ public void testCustomMergeScheduler() throws Exception {
+ // we don't really need to execute anything, just to make sure the custom MS
+ // compiles. But ensure that it can be used as well, e.g., no other hidden
+ // dependencies or something. Therefore, don't use any random API !
+ Directory dir = new RAMDirectory();
+ IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null);
+ conf.setMergeScheduler(new ReportingMergeScheduler());
+ IndexWriter writer = new IndexWriter(dir, conf);
+ writer.addDocument(new Document());
+ writer.commit(); // trigger flush
+ writer.addDocument(new Document());
+ writer.commit(); // trigger flush
+ writer.optimize();
+ writer.close();
+ dir.close();
+ }
+
}
Modified: lucene/dev/branches/docvalues/modules/analysis/common/src/test/org/apache/lucene/collation/CollationTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/analysis/common/src/test/org/apache/lucene/collation/CollationTestBase.java?rev=1098604&r1=1098603&r2=1098604&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/analysis/common/src/test/org/apache/lucene/collation/CollationTestBase.java (original)
+++ lucene/dev/branches/docvalues/modules/analysis/common/src/test/org/apache/lucene/collation/CollationTestBase.java Mon May 2 14:44:57 2011
@@ -257,27 +257,6 @@ public abstract class CollationTestBase
}
assertEquals(expectedResult, buff.toString());
}
-
- private String randomString() {
- // ideally we could do this!
- // return _TestUtil.randomUnicodeString(random);
- //
- // http://bugs.icu-project.org/trac/ticket/8060
- // http://bugs.icu-project.org/trac/ticket/7732
- // ...
- //
- // as a workaround, just test the BMP for now (and avoid 0xFFFF etc)
- int length = _TestUtil.nextInt(random, 0, 10);
- char chars[] = new char[length];
- for (int i = 0; i < length; i++) {
- if (random.nextBoolean()) {
- chars[i] = (char) _TestUtil.nextInt(random, 0, 0xD7FF);
- } else {
- chars[i] = (char) _TestUtil.nextInt(random, 0xE000, 0xFFFD);
- }
- }
- return new String(chars, 0, length);
- }
public void assertThreadSafe(final Analyzer analyzer) throws Exception {
int numTestPoints = 100;
@@ -289,7 +268,7 @@ public abstract class CollationTestBase
// and ensure they are the same as the ones we produced in serial fashion.
for (int i = 0; i < numTestPoints; i++) {
- String term = randomString();
+ String term = _TestUtil.randomSimpleString(random);
TokenStream ts = analyzer.reusableTokenStream("fake", new StringReader(term));
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
BytesRef bytes = termAtt.getBytesRef();
Modified: lucene/dev/branches/docvalues/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java?rev=1098604&r1=1098603&r2=1098604&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java (original)
+++ lucene/dev/branches/docvalues/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java Mon May 2 14:44:57 2011
@@ -87,8 +87,7 @@ public class TestCollationKeyAnalyzer ex
public void testThreadSafe() throws Exception {
int iters = 20 * RANDOM_MULTIPLIER;
for (int i = 0; i < iters; i++) {
- Locale locale = randomLocale(random);
- Collator collator = Collator.getInstance(locale);
+ Collator collator = Collator.getInstance(Locale.GERMAN);
collator.setStrength(Collator.PRIMARY);
assertThreadSafe(new CollationKeyAnalyzer(TEST_VERSION_CURRENT, collator));
}
Modified: lucene/dev/branches/docvalues/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java?rev=1098604&r1=1098603&r2=1098604&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java (original)
+++ lucene/dev/branches/docvalues/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java Mon May 2 14:44:57 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.collation;
import com.ibm.icu.text.Collator;
+
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.util.BytesRef;
@@ -88,7 +89,7 @@ public class TestICUCollationKeyAnalyzer
public void testThreadSafe() throws Exception {
int iters = 20 * RANDOM_MULTIPLIER;
for (int i = 0; i < iters; i++) {
- Locale locale = randomLocale(random);
+ Locale locale = Locale.GERMAN;
Collator collator = Collator.getInstance(locale);
collator.setStrength(Collator.IDENTICAL);
assertThreadSafe(new ICUCollationKeyAnalyzer(TEST_VERSION_CURRENT, collator));