You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by bu...@apache.org on 2011/01/17 10:26:00 UTC
svn commit: r1059823 - in /lucene/dev/branches/realtime_search: ./ lucene/
lucene/src/java/org/apache/lucene/index/
lucene/src/java/org/apache/lucene/search/
lucene/src/test/org/apache/lucene/index/ modules/
modules/analysis/common/src/java/org/apache/...
Author: buschmi
Date: Mon Jan 17 09:26:00 2011
New Revision: 1059823
URL: http://svn.apache.org/viewvc?rev=1059823&view=rev
Log:
Merging r1059432 through r1059822 into realtime branch
Modified:
lucene/dev/branches/realtime_search/ (props changed)
lucene/dev/branches/realtime_search/lucene/ (props changed)
lucene/dev/branches/realtime_search/lucene/CHANGES.txt
lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/LogByteSizeMergePolicy.java
lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/LogDocMergePolicy.java
lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentReader.java
lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/FieldValueHitQueue.java
lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java
lucene/dev/branches/realtime_search/modules/ (props changed)
lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
lucene/dev/branches/realtime_search/solr/ (props changed)
lucene/dev/branches/realtime_search/solr/CHANGES.txt (props changed)
lucene/dev/branches/realtime_search/solr/KEYS (props changed)
lucene/dev/branches/realtime_search/solr/LICENSE.txt (props changed)
lucene/dev/branches/realtime_search/solr/NOTICE.txt (props changed)
lucene/dev/branches/realtime_search/solr/README.txt (props changed)
lucene/dev/branches/realtime_search/solr/build.xml (props changed)
lucene/dev/branches/realtime_search/solr/client/ (props changed)
lucene/dev/branches/realtime_search/solr/common-build.xml (props changed)
lucene/dev/branches/realtime_search/solr/contrib/ (props changed)
lucene/dev/branches/realtime_search/solr/example/ (props changed)
lucene/dev/branches/realtime_search/solr/lib/ (props changed)
lucene/dev/branches/realtime_search/solr/site/ (props changed)
lucene/dev/branches/realtime_search/solr/src/ (props changed)
lucene/dev/branches/realtime_search/solr/testlogging.properties (props changed)
Modified: lucene/dev/branches/realtime_search/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/CHANGES.txt?rev=1059823&r1=1059822&r2=1059823&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/realtime_search/lucene/CHANGES.txt Mon Jan 17 09:26:00 2011
@@ -507,9 +507,10 @@ Changes in runtime behavior
usage, allowing applications to accidentally open two writers on the
same directory. (Mike McCandless)
-* LUCENE-2701: maxMergeMB and maxMergeDocs constraints set on LogMergePolicy now
- affect optimize() as well (as opposed to only regular merges). This means that
- you can run optimize() and too large segments won't be merged. (Shai Erera)
+* LUCENE-2701: maxMergeMBForOptimize and maxMergeDocs constraints set on
+ LogMergePolicy now affect optimize() as well (as opposed to only regular
+ merges). This means that you can run optimize() and too large segments won't
+ be merged. (Shai Erera)
API Changes
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/LogByteSizeMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/LogByteSizeMergePolicy.java?rev=1059823&r1=1059822&r2=1059823&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/LogByteSizeMergePolicy.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/LogByteSizeMergePolicy.java Mon Jan 17 09:26:00 2011
@@ -30,9 +30,14 @@ public class LogByteSizeMergePolicy exte
* or larger will never be merged. @see setMaxMergeMB */
public static final double DEFAULT_MAX_MERGE_MB = 2048;
+ /** Default maximum segment size. A segment of this size
+ * or larger will never be merged during optimize. @see setMaxMergeMBForOptimize */
+ public static final double DEFAULT_MAX_MERGE_MB_FOR_OPTIMIZE = Long.MAX_VALUE;
+
public LogByteSizeMergePolicy() {
minMergeSize = (long) (DEFAULT_MIN_MERGE_MB*1024*1024);
maxMergeSize = (long) (DEFAULT_MAX_MERGE_MB*1024*1024);
+ maxMergeSizeForOptimize = (long) (DEFAULT_MAX_MERGE_MB_FOR_OPTIMIZE*1024*1024);
}
@Override
@@ -63,6 +68,23 @@ public class LogByteSizeMergePolicy exte
return ((double) maxMergeSize)/1024/1024;
}
+ /** <p>Determines the largest segment (measured by total
+ * byte size of the segment's files, in MB) that may be
+ * merged with other segments during optimize. Setting
+ * it low will leave the index with more than 1 segment,
+ * even if {@link IndexWriter#optimize()} is called.*/
+ public void setMaxMergeMBForOptimize(double mb) {
+ maxMergeSizeForOptimize = (long) (mb*1024*1024);
+ }
+
+ /** Returns the largest segment (measured by total byte
+ * size of the segment's files, in MB) that may be merged
+ * with other segments during optimize.
+ * @see #setMaxMergeMBForOptimize */
+ public double getMaxMergeMBForOptimize() {
+ return ((double) maxMergeSizeForOptimize)/1024/1024;
+ }
+
/** Sets the minimum size for the lowest level segments.
* Any segments below this size are considered to be on
* the same level (even if they vary drastically in size)
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/LogDocMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/LogDocMergePolicy.java?rev=1059823&r1=1059822&r2=1059823&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/LogDocMergePolicy.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/LogDocMergePolicy.java Mon Jan 17 09:26:00 2011
@@ -31,9 +31,10 @@ public class LogDocMergePolicy extends L
public LogDocMergePolicy() {
minMergeSize = DEFAULT_MIN_MERGE_DOCS;
- // maxMergeSize is never used by LogDocMergePolicy; set
+ // maxMergeSize(ForOptimize) are never used by LogDocMergePolicy; set
// it to Long.MAX_VALUE to disable it
maxMergeSize = Long.MAX_VALUE;
+ maxMergeSizeForOptimize = Long.MAX_VALUE;
}
@Override
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java?rev=1059823&r1=1059822&r2=1059823&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java Mon Jan 17 09:26:00 2011
@@ -63,6 +63,9 @@ public abstract class LogMergePolicy ext
protected long minMergeSize;
protected long maxMergeSize;
+ // Although the core MPs set it explicitly, we must default in case someone
+ // out there wrote his own LMP ...
+ protected long maxMergeSizeForOptimize = Long.MAX_VALUE;
protected int maxMergeDocs = DEFAULT_MAX_MERGE_DOCS;
protected double noCFSRatio = DEFAULT_NO_CFS_RATIO;
@@ -240,9 +243,9 @@ public abstract class LogMergePolicy ext
int start = last - 1;
while (start >= 0) {
SegmentInfo info = infos.info(start);
- if (size(info) > maxMergeSize || sizeDocs(info) > maxMergeDocs) {
+ if (size(info) > maxMergeSizeForOptimize || sizeDocs(info) > maxMergeDocs) {
if (verbose()) {
- message("optimize: skip segment=" + info + ": size is > maxMergeSize (" + maxMergeSize + ") or sizeDocs is > maxMergeDocs (" + maxMergeDocs + ")");
+ message("optimize: skip segment=" + info + ": size is > maxMergeSize (" + maxMergeSizeForOptimize + ") or sizeDocs is > maxMergeDocs (" + maxMergeDocs + ")");
}
// need to skip that segment + add a merge for the 'right' segments,
// unless there is only 1 which is optimized.
@@ -326,9 +329,12 @@ public abstract class LogMergePolicy ext
}
/** Returns the merges necessary to optimize the index.
- * This merge policy defines "optimized" to mean only one
- * segment in the index, where that segment has no
- * deletions pending nor separate norms, and it is in
+ * This merge policy defines "optimized" to mean only the
+ * requested number of segments is left in the index, and
+ * respects the {@link #maxMergeSizeForOptimize} setting.
+ * By default, and assuming {@code maxNumSegments=1}, only
+ * one segment will be left in the index, where that segment
+ * has no deletions pending nor separate norms, and it is in
* compound file format if the current useCompoundFile
* setting is true. This method returns multiple merges
* (mergeFactor at a time) so the {@link MergeScheduler}
@@ -382,7 +388,7 @@ public abstract class LogMergePolicy ext
boolean anyTooLarge = false;
for (int i = 0; i < last; i++) {
SegmentInfo info = infos.info(i);
- if (size(info) > maxMergeSize || sizeDocs(info) > maxMergeDocs) {
+ if (size(info) > maxMergeSizeForOptimize || sizeDocs(info) > maxMergeDocs) {
anyTooLarge = true;
break;
}
@@ -588,6 +594,7 @@ public abstract class LogMergePolicy ext
sb.append("minMergeSize=").append(minMergeSize).append(", ");
sb.append("mergeFactor=").append(mergeFactor).append(", ");
sb.append("maxMergeSize=").append(maxMergeSize).append(", ");
+ sb.append("maxMergeSizeForOptimize=").append(maxMergeSizeForOptimize).append(", ");
sb.append("calibrateSizeByDeletes=").append(calibrateSizeByDeletes).append(", ");
sb.append("maxMergeDocs=").append(maxMergeDocs).append(", ");
sb.append("useCompoundFile=").append(useCompoundFile);
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1059823&r1=1059822&r2=1059823&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentReader.java Mon Jan 17 09:26:00 2011
@@ -335,29 +335,6 @@ public class SegmentReader extends Index
}
}
- // Load bytes but do not cache them if they were not
- // already cached
- public synchronized void bytes(byte[] bytesOut, int offset, int len) throws IOException {
- assert refCount > 0 && (origNorm == null || origNorm.refCount > 0);
- if (bytes != null) {
- // Already cached -- copy from cache:
- assert len <= maxDoc();
- System.arraycopy(bytes, 0, bytesOut, offset, len);
- } else {
- // Not cached
- if (origNorm != null) {
- // Ask origNorm to load
- origNorm.bytes(bytesOut, offset, len);
- } else {
- // We are orig -- read ourselves from disk:
- synchronized(in) {
- in.seek(normSeek);
- in.readBytes(bytesOut, offset, len, false);
- }
- }
- }
- }
-
// Load & cache full bytes array. Returns bytes.
public synchronized byte[] bytes() throws IOException {
assert refCount > 0 && (origNorm == null || origNorm.refCount > 0);
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/FieldValueHitQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/FieldValueHitQueue.java?rev=1059823&r1=1059822&r2=1059823&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/FieldValueHitQueue.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/FieldValueHitQueue.java Mon Jan 17 09:26:00 2011
@@ -57,9 +57,6 @@ public abstract class FieldValueHitQueue
public OneComparatorFieldValueHitQueue(SortField[] fields, int size)
throws IOException {
super(fields);
- if (fields.length == 0) {
- throw new IllegalArgumentException("Sort must contain at least one field");
- }
SortField field = fields[0];
setComparator(0,field.getComparator(size, 0));
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java?rev=1059823&r1=1059822&r2=1059823&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java Mon Jan 17 09:26:00 2011
@@ -63,7 +63,7 @@ public class TestSizeBoundedOptimize ext
conf = newWriterConfig();
LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy();
- lmp.setMaxMergeMB((min + 1) / (1 << 20));
+ lmp.setMaxMergeMBForOptimize((min + 1) / (1 << 20));
conf.setMergePolicy(lmp);
writer = new IndexWriter(dir, conf);
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=1059823&r1=1059822&r2=1059823&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Mon Jan 17 09:26:00 2011
@@ -34,14 +34,6 @@ import org.apache.lucene.util.Version;
* Unicode Text Segmentation algorithm, as specified in
* <a href="http://unicode.org/reports/tr29/">Unicode Standard Annex #29</a>.
* <p/>
- * <b>WARNING</b>: Because JFlex does not support Unicode supplementary
- * characters (characters above the Basic Multilingual Plane, which contains
- * those up to and including U+FFFF), this scanner will not recognize them
- * properly. If you need to be able to process text containing supplementary
- * characters, consider using the ICU4J-backed implementation in modules/analysis/icu
- * (org.apache.lucene.analysis.icu.segmentation.ICUTokenizer)
- * instead of this class, since the ICU4J-backed implementation does not have
- * this limitation.
* <p>Many applications have specific tokenizer needs. If this tokenizer does
* not suit your application, please consider copying this source code
* directory to your project and maintaining your own grammar-based tokenizer.