You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/11/09 16:51:22 UTC
svn commit: r1199810 - in /lucene/dev/branches/lucene2621/lucene:
contrib/misc/src/java/org/apache/lucene/index/
src/java/org/apache/lucene/index/ src/test/org/apache/lucene/index/
Author: rmuir
Date: Wed Nov 9 15:51:21 2011
New Revision: 1199810
URL: http://svn.apache.org/viewvc?rev=1199810&view=rev
Log:
LUCENE-2621: clean up SI.sizeInBytes
Modified:
lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java
lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/MergePolicy.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java
Modified: lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java?rev=1199810&r1=1199809&r2=1199810&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java (original)
+++ lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java Wed Nov 9 15:51:21 2011
@@ -57,7 +57,7 @@ public class BalancedSegmentMergePolicy
@Override
protected long size(SegmentInfo info) throws IOException {
- long byteSize = info.sizeInBytes(true);
+ long byteSize = info.sizeInBytes();
float delRatio = (info.docCount <= 0 ? 0.0f : ((float)info.getDelCount() / (float)info.docCount));
return (info.docCount <= 0 ? byteSize : (long)((1.0f - delRatio) * byteSize));
}
Modified: lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java?rev=1199810&r1=1199809&r2=1199810&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java (original)
+++ lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java Wed Nov 9 15:51:21 2011
@@ -105,7 +105,7 @@ public class IndexSplitter {
DecimalFormat formatter = new DecimalFormat("###,###.###");
for (int x = 0; x < infos.size(); x++) {
SegmentInfo info = infos.info(x);
- String sizeStr = formatter.format(info.sizeInBytes(true));
+ String sizeStr = formatter.format(info.sizeInBytes());
System.out.println(info.name + " " + sizeStr);
}
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1199810&r1=1199809&r2=1199810&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/CheckIndex.java Wed Nov 9 15:51:21 2011
@@ -495,7 +495,7 @@ public class CheckIndex {
segInfoStat.hasProx = info.getHasProx();
msg(" numFiles=" + info.files().size());
segInfoStat.numFiles = info.files().size();
- segInfoStat.sizeMB = info.sizeInBytes(true)/(1024.*1024.);
+ segInfoStat.sizeMB = info.sizeInBytes()/(1024.*1024.);
msg(" size (MB)=" + nf.format(segInfoStat.sizeMB));
Map<String,String> diagnostics = info.getDiagnostics();
segInfoStat.diagnostics = diagnostics;
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1199810&r1=1199809&r2=1199810&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/IndexWriter.java Wed Nov 9 15:51:21 2011
@@ -2220,7 +2220,7 @@ public class IndexWriter implements Clos
setDiagnostics(newSegment, "flush");
- IOContext context = new IOContext(new FlushInfo(newSegment.docCount, newSegment.sizeInBytes(true)));
+ IOContext context = new IOContext(new FlushInfo(newSegment.docCount, newSegment.sizeInBytes()));
boolean success = false;
try {
@@ -2452,7 +2452,7 @@ public class IndexWriter implements Clos
&& versionComparator.compare(info.getVersion(), "3.1") >= 0;
}
- IOContext context = new IOContext(new MergeInfo(info.docCount, info.sizeInBytes(true), true, false));
+ IOContext context = new IOContext(new MergeInfo(info.docCount, info.sizeInBytes(), true, false));
if (createCFS) {
copySegmentIntoCFS(info, newSegName, context);
@@ -3457,7 +3457,7 @@ public class IndexWriter implements Clos
final int delCount = numDeletedDocs(info);
assert delCount <= info.docCount;
final double delRatio = ((double) delCount)/info.docCount;
- merge.estimatedMergeBytes += info.sizeInBytes(true) * (1.0 - delRatio);
+ merge.estimatedMergeBytes += info.sizeInBytes() * (1.0 - delRatio);
}
}
@@ -3708,7 +3708,7 @@ public class IndexWriter implements Clos
}
if (infoStream != null) {
- infoStream.message("IW", String.format("merged segment size=%.3f MB vs estimate=%.3f MB", merge.info.sizeInBytes(true)/1024./1024., merge.estimatedMergeBytes/1024/1024.));
+ infoStream.message("IW", String.format("merged segment size=%.3f MB vs estimate=%.3f MB", merge.info.sizeInBytes()/1024./1024., merge.estimatedMergeBytes/1024/1024.));
}
final IndexReaderWarmer mergedSegmentWarmer = config.getMergedSegmentWarmer();
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java?rev=1199810&r1=1199809&r2=1199810&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java Wed Nov 9 15:51:21 2011
@@ -196,7 +196,7 @@ public abstract class LogMergePolicy ext
}
protected long sizeBytes(SegmentInfo info) throws IOException {
- long byteSize = info.sizeInBytes(true);
+ long byteSize = info.sizeInBytes();
if (calibrateSizeByDeletes) {
int delCount = writer.get().numDeletedDocs(info);
double delRatio = (info.docCount <= 0 ? 0.0f : ((float)delCount / (float)info.docCount));
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/MergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/MergePolicy.java?rev=1199810&r1=1199809&r2=1199810&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/MergePolicy.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/MergePolicy.java Wed Nov 9 15:51:21 2011
@@ -175,7 +175,7 @@ public abstract class MergePolicy implem
public long totalBytesSize() throws IOException {
long total = 0;
for (SegmentInfo info : segments) {
- total += info.sizeInBytes(true);
+ total += info.sizeInBytes();
}
return total;
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1199810&r1=1199809&r2=1199810&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Wed Nov 9 15:51:21 2011
@@ -280,11 +280,23 @@ public final class SegmentInfo implement
}
/**
+ * Returns total size in bytes of all of files used by this segment
+ */
+ public long sizeInBytes() throws IOException {
+ return sizeInBytes(true);
+ }
+
+ /**
* Returns total size in bytes of all of files used by this segment (if
* {@code includeDocStores} is true), or the size of all files except the
* store files otherwise.
+ * <p>
+ * NOTE: includeDocStores=false should only be used for debugging.
+ * Theoretically a codec could combine its files however it wants (after-
+ * the-fact or something), and this calculation is not particularly
+ * efficient.
*/
- public long sizeInBytes(boolean includeDocStores) throws IOException {
+ long sizeInBytes(boolean includeDocStores) throws IOException {
// TODO: based on how this is used, can't we just forget about all this docstore crap?
// its really an abstraction violation into the codec
if (includeDocStores) {
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java?rev=1199810&r1=1199809&r2=1199810&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java Wed Nov 9 15:51:21 2011
@@ -338,7 +338,7 @@ public class TieredMergePolicy extends M
for(int idx = tooBigCount; idx<infosSorted.size(); idx++) {
final SegmentInfo info = infosSorted.get(idx);
if (merging.contains(info)) {
- mergingBytes += info.sizeInBytes(true);
+ mergingBytes += info.sizeInBytes();
} else if (!toBeMerged.contains(info)) {
eligible.add(info);
}
@@ -430,7 +430,7 @@ public class TieredMergePolicy extends M
final long segBytes = size(info);
totAfterMergeBytes += segBytes;
totAfterMergeBytesFloored += floorSize(segBytes);
- totBeforeMergeBytes += info.sizeInBytes(true);
+ totBeforeMergeBytes += info.sizeInBytes();
}
// Measure "skew" of the merge, which can range
@@ -632,7 +632,7 @@ public class TieredMergePolicy extends M
// Segment size in bytes, pro-rated by % deleted
private long size(SegmentInfo info) throws IOException {
- final long byteSize = info.sizeInBytes(true);
+ final long byteSize = info.sizeInBytes();
final int delCount = writer.get().numDeletedDocs(info);
final double delRatio = (info.docCount <= 0 ? 0.0f : ((double)delCount / (double)info.docCount));
assert delRatio <= 1.0;
Modified: lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java?rev=1199810&r1=1199809&r2=1199810&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java Wed Nov 9 15:51:21 2011
@@ -59,7 +59,7 @@ public class TestSizeBoundedOptimize ext
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
- double min = sis.info(0).sizeInBytes(true);
+ double min = sis.info(0).sizeInBytes();
conf = newWriterConfig();
LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy();