You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2010/11/17 15:29:59 UTC
svn commit: r1036060 - in /lucene/dev/branches/docvalues/lucene/src:
java/org/apache/lucene/index/values/ java/org/apache/lucene/util/
test/org/apache/lucene/index/values/
Author: simonw
Date: Wed Nov 17 14:29:59 2010
New Revision: 1036060
URL: http://svn.apache.org/viewvc?rev=1036060&view=rev
Log:
fixed length prefix for use with PagedBytes
Modified:
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IOUtils.java
lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java?rev=1036060&r1=1036059&r2=1036060&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java Wed Nov 17 14:29:59 2010
@@ -243,7 +243,7 @@ public final class Bytes {
if (initIndex)
initIndexOut();
}
-
+
protected void initDataOut() throws IOException {
datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "",
IndexFileNames.CSF_DATA_EXTENSION));
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java?rev=1036060&r1=1036059&r2=1036060&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java Wed Nov 17 14:29:59 2010
@@ -24,6 +24,7 @@ import org.apache.lucene.index.values.By
import org.apache.lucene.index.values.Bytes.BytesReaderBase;
import org.apache.lucene.index.values.Bytes.BytesWriterBase;
import org.apache.lucene.index.values.FixedDerefBytesImpl.Reader.DerefBytesEnum;
+import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
@@ -32,7 +33,6 @@ import org.apache.lucene.util.ByteBlockP
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.CodecUtil;
-import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.ByteBlockPool.Allocator;
@@ -113,7 +113,7 @@ class VarDerefBytesImpl {
final int docAddress;
if (e >= 0) {
docAddress = array.array.address[e] = address;
- address += IOUtils.writeLength(datOut, bytes);
+ address += writePrefixLength(datOut, bytes);
datOut.writeBytes(bytes.bytes, bytes.offset, bytes.length);
address += bytes.length;
} else {
@@ -121,7 +121,18 @@ class VarDerefBytesImpl {
}
docToAddress[docID] = docAddress;
}
-
+
+ private static int writePrefixLength(DataOutput datOut, BytesRef bytes) throws IOException{
+ if (bytes.length < 128) {
+ datOut.writeByte((byte) bytes.length);
+ return 1;
+ } else {
+ datOut.writeByte((byte) (0x80 | (bytes.length >> 8)));
+ datOut.writeByte((byte) (bytes.length & 0xff));
+ return 2;
+ }
+ }
+
public long ramBytesUsed() {
return bytesUsed.get();
}
@@ -216,15 +227,14 @@ class VarDerefBytesImpl {
@Override
protected void fill(long address, BytesRef ref) throws IOException {
- // TODO(simonw): use pages here
datIn.seek(fp + --address);
final byte sizeByte = datIn.readByte();
final int size;
- if ((sizeByte & 0x80) == 0) {
+ if ((sizeByte & 128) == 0) {
// length is 1 byte
size = sizeByte;
} else {
- size = (sizeByte & 0x7f) + ((datIn.readByte() & 0xff) << 7);
+ size = ((sizeByte & 0x7f)<<8) | ((datIn.readByte() & 0xff));
}
if(ref.bytes.length < size)
ref.grow(size);
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IOUtils.java?rev=1036060&r1=1036059&r2=1036060&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IOUtils.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IOUtils.java Wed Nov 17 14:29:59 2010
@@ -20,8 +20,6 @@ package org.apache.lucene.util;
import java.io.Closeable;
import java.io.IOException;
-import org.apache.lucene.store.DataOutput;
-
/** @lucene.internal */
public final class IOUtils {
@@ -61,37 +59,10 @@ public final class IOUtils {
}
}
- if (priorException != null)
+ if (priorException != null) {
throw priorException;
- else if (firstIOE != null)
+ } else if (firstIOE != null) {
throw firstIOE;
- }
-
- /**
- * Writes the length of the {@link BytesRef} as either a one or two bytes to
- * the {@link DataOutput} and returns the number of bytes used.
- *
- * @param datOut
- * the output to write to
- * @param bytes
- * the length to write
- * @return the length of the {@link BytesRef} as either a one or two bytes to
- * the {@link DataOutput} and returns the number of bytes used.
- * @throws IOException
- * if datOut throws an {@link IOException}
- */
- public static int writeLength(DataOutput datOut, BytesRef bytes)
- throws IOException {
- final int length = bytes.length;
- if (length < 128) {
- // 1 byte to store length
- datOut.writeByte((byte) length);
- return 1;
- } else {
- // 2 byte to store length
- datOut.writeByte((byte) (0x80 | (length & 0x7f)));
- datOut.writeByte((byte) ((length >> 7) & 0xff));
- return 2;
}
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java?rev=1036060&r1=1036059&r2=1036060&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java Wed Nov 17 14:29:59 2010
@@ -273,7 +273,6 @@ public class TestDocValuesIndexing exten
bytesRef.length = b.length;
bytesRef.offset = 0;
}
- //
byte upto = 0;
for (int i = 0; i < numValues; i++) {
if (isNumeric) {
@@ -324,7 +323,7 @@ public class TestDocValuesIndexing exten
public void runTestIndexBytes(IndexWriterConfig cfg, boolean withDeletions)
throws CorruptIndexException, LockObtainFailedException, IOException {
- Directory d = newDirectory();
+ final Directory d = newDirectory();
IndexWriter w = new IndexWriter(d, cfg);
final List<Values> byteVariantList = new ArrayList<Values>(BYTES);
// run in random order to test if fill works correctly during merges
@@ -389,7 +388,6 @@ public class TestDocValuesIndexing exten
assertEquals(base + numRemainingValues, r.numDocs());
int v = 0;
for (int i = base; i < r.numDocs(); i++) {
-
String msg = " field: " + byteIndexValue.name() + " at index: " + i
+ " base: " + base + " numDocs:" + r.numDocs() + " bytesSize: "
+ bytesSize;