You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2013/01/09 11:03:38 UTC
svn commit: r1430757 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/compressing/
lucene/core/src/test/org/apache/lucene/codecs/compressing/
lucene/test-framework/ lucene/test-framework/src/jav...
Author: jpountz
Date: Wed Jan 9 10:03:37 2013
New Revision: 1430757
URL: http://svn.apache.org/viewvc?rev=1430757&view=rev
Log:
LUCENE-4666: Simplify CompressingStoredFieldsFormat merging (merged from r1430755).
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/Decompressor.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/LZ4.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java?rev=1430757&r1=1430756&r2=1430757&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java Wed Jan 9 10:03:37 2013
@@ -395,8 +395,10 @@ public final class CompressingStoredFiel
* Copy compressed data.
*/
void copyCompressedData(DataOutput out) throws IOException {
- final int chunkSize = chunkSize();
- decompressor.copyCompressedData(fieldsStream, chunkSize, out);
+ final long chunkEnd = docBase + chunkDocs == numDocs
+ ? fieldsStream.length()
+ : indexReader.getStartPointer(docBase + chunkDocs);
+ out.copyBytes(fieldsStream, chunkEnd - fieldsStream.getFilePointer());
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java?rev=1430757&r1=1430756&r2=1430757&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java Wed Jan 9 10:03:37 2013
@@ -141,14 +141,6 @@ public abstract class CompressionMode {
}
@Override
- public void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException {
- final int copied = LZ4.copyCompressedData(in, originalLength, out);
- if (copied != originalLength) {
- throw new CorruptIndexException("Currupted compressed stream: expected " + originalLength + " bytes, but got at least" + copied);
- }
- }
-
- @Override
public Decompressor clone() {
return this;
}
@@ -225,13 +217,6 @@ public abstract class CompressionMode {
}
@Override
- public void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException {
- final int compressedLength = in.readVInt();
- out.writeVInt(compressedLength);
- out.copyBytes(in, compressedLength);
- }
-
- @Override
public Decompressor clone() {
return new DeflateDecompressor();
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/Decompressor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/Decompressor.java?rev=1430757&r1=1430756&r2=1430757&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/Decompressor.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/Decompressor.java Wed Jan 9 10:03:37 2013
@@ -20,7 +20,6 @@ package org.apache.lucene.codecs.compres
import java.io.IOException;
import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.BytesRef;
/**
@@ -47,10 +46,6 @@ public abstract class Decompressor imple
*/
public abstract void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException;
- /** Copy a compressed stream whose original length is
- * <code>originalLength</code> from <code>in</code> to <code>out</code>. */
- public abstract void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException;
-
@Override
public abstract Decompressor clone();
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/LZ4.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/LZ4.java?rev=1430757&r1=1430756&r2=1430757&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/LZ4.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/LZ4.java Wed Jan 9 10:03:37 2013
@@ -506,51 +506,4 @@ class LZ4 {
encodeLastLiterals(src, anchor, srcEnd - anchor, out);
}
- /** Copy bytes from <code>in</code> to <code>out</code> where
- * <code>in</code> is a LZ4-encoded stream. This method copies enough bytes
- * so that <code>out</code> can be used later on to restore the first
- * <code>length</code> bytes of the stream. This method always reads at
- * least one byte from <code>in</code> so make sure not to call this method
- * if <code>in</code> reached the end of the stream, even if
- * <code>length=0</code>. */
- public static int copyCompressedData(DataInput in, int length, DataOutput out) throws IOException {
- int n = 0;
- do {
- // literals
- final byte token = in.readByte();
- out.writeByte(token);
- int literalLen = (token & 0xFF) >>> 4;
- if (literalLen == 0x0F) {
- byte len;
- while ((len = in.readByte()) == (byte) 0xFF) {
- literalLen += 0xFF;
- out.writeByte(len);
- }
- literalLen += len & 0xFF;
- out.writeByte(len);
- }
- out.copyBytes(in, literalLen);
- n += literalLen;
- if (n >= length) {
- break;
- }
-
- // matchs
- out.copyBytes(in, 2); // match dec
- int matchLen = token & 0x0F;
- if (matchLen == 0x0F) {
- byte len;
- while ((len = in.readByte()) == (byte) 0xFF) {
- matchLen += 0xFF;
- out.writeByte(len);
- }
- matchLen += len & 0xFF;
- out.writeByte(len);
- }
- matchLen += MIN_MATCH;
- n += matchLen;
- } while (n < length);
- return n;
- }
-
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java?rev=1430757&r1=1430756&r2=1430757&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java Wed Jan 9 10:03:37 2013
@@ -80,16 +80,6 @@ public abstract class AbstractTestCompre
return Arrays.copyOfRange(bytes.bytes, bytes.offset, bytes.offset + bytes.length);
}
- static byte[] copyCompressedData(Decompressor decompressor, byte[] compressed, int originalLength) throws IOException {
- GrowableByteArrayDataOutput out = new GrowableByteArrayDataOutput(compressed.length);
- decompressor.copyCompressedData(new ByteArrayDataInput(compressed), originalLength, out);
- return Arrays.copyOf(out.bytes, out.length);
- }
-
- byte[] copyCompressedData(byte[] compressed, int originalLength) throws IOException {
- return copyCompressedData(mode.newDecompressor(), compressed, originalLength);
- }
-
public void testDecompress() throws IOException {
final int iterations = atLeast(10);
for (int i = 0; i < iterations; ++i) {
@@ -117,17 +107,10 @@ public abstract class AbstractTestCompre
}
}
- public void testCopyCompressedData() throws IOException {
- final byte[] decompressed = randomArray();
- final byte[] compressed = compress(decompressed);
- assertArrayEquals(compressed, copyCompressedData(compressed, decompressed.length));
- }
-
public byte[] test(byte[] decompressed) throws IOException {
final byte[] compressed = compress(decompressed);
final byte[] restored = decompress(compressed, decompressed.length);
assertEquals(decompressed.length, restored.length);
- assertArrayEquals(compressed, copyCompressedData(compressed, decompressed.length));
return compressed;
}
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java?rev=1430757&r1=1430756&r2=1430757&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java Wed Jan 9 10:03:37 2013
@@ -67,11 +67,6 @@ public class DummyCompressingCodec exten
}
@Override
- public void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException {
- out.copyBytes(in, originalLength);
- }
-
- @Override
public Decompressor clone() {
return this;
}