You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2012/10/12 06:35:44 UTC
svn commit: r1397432 [3/3] - in
/hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/main/docs/
src/main/java/org/apache/hadoop/metrics/
src/main/java/org/apache/hadoop/metrics/file/
src/main/java/org/apache/hadoop/metrics/spi/ src/main/jav...
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java?rev=1397432&r1=1397431&r2=1397432&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java Fri Oct 12 04:35:42 2012
@@ -1,460 +1,460 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.util;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.zip.Checksum;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.ChecksumException;
-
-/**
- * This class provides inteface and utilities for processing checksums for
- * DFS data transfers.
- */
-@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
-@InterfaceStability.Evolving
-public class DataChecksum implements Checksum {
-
- // Misc constants
- public static final int HEADER_LEN = 5; /// 1 byte type and 4 byte len
-
- // checksum types
- public static final int CHECKSUM_NULL = 0;
- public static final int CHECKSUM_CRC32 = 1;
- public static final int CHECKSUM_CRC32C = 2;
- public static final int CHECKSUM_DEFAULT = 3;
- public static final int CHECKSUM_MIXED = 4;
-
- /** The checksum types */
- public static enum Type {
- NULL (CHECKSUM_NULL, 0),
- CRC32 (CHECKSUM_CRC32, 4),
- CRC32C(CHECKSUM_CRC32C, 4),
- DEFAULT(CHECKSUM_DEFAULT, 0), // This cannot be used to create DataChecksum
- MIXED (CHECKSUM_MIXED, 0); // This cannot be used to create DataChecksum
-
- public final int id;
- public final int size;
-
- private Type(int id, int size) {
- this.id = id;
- this.size = size;
- }
-
- /** @return the type corresponding to the id. */
- public static Type valueOf(int id) {
- if (id < 0 || id >= values().length) {
- throw new IllegalArgumentException("id=" + id
- + " out of range [0, " + values().length + ")");
- }
- return values()[id];
- }
- }
-
-
- public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) {
- if ( bytesPerChecksum <= 0 ) {
- return null;
- }
-
- switch ( type ) {
- case NULL :
- return new DataChecksum(type, new ChecksumNull(), bytesPerChecksum );
- case CRC32 :
- return new DataChecksum(type, new PureJavaCrc32(), bytesPerChecksum );
- case CRC32C:
- return new DataChecksum(type, new PureJavaCrc32C(), bytesPerChecksum);
- default:
- return null;
- }
- }
-
- /**
- * Creates a DataChecksum from HEADER_LEN bytes from arr[offset].
- * @return DataChecksum of the type in the array or null in case of an error.
- */
- public static DataChecksum newDataChecksum( byte bytes[], int offset ) {
- if ( offset < 0 || bytes.length < offset + HEADER_LEN ) {
- return null;
- }
-
- // like readInt():
- int bytesPerChecksum = ( (bytes[offset+1] & 0xff) << 24 ) |
- ( (bytes[offset+2] & 0xff) << 16 ) |
- ( (bytes[offset+3] & 0xff) << 8 ) |
- ( (bytes[offset+4] & 0xff) );
- return newDataChecksum( Type.valueOf(bytes[0]), bytesPerChecksum );
- }
-
- /**
- * This constructucts a DataChecksum by reading HEADER_LEN bytes from
- * input stream <i>in</i>
- */
- public static DataChecksum newDataChecksum( DataInputStream in )
- throws IOException {
- int type = in.readByte();
- int bpc = in.readInt();
- DataChecksum summer = newDataChecksum(Type.valueOf(type), bpc );
- if ( summer == null ) {
- throw new IOException( "Could not create DataChecksum of type " +
- type + " with bytesPerChecksum " + bpc );
- }
- return summer;
- }
-
- /**
- * Writes the checksum header to the output stream <i>out</i>.
- */
- public void writeHeader( DataOutputStream out )
- throws IOException {
- out.writeByte( type.id );
- out.writeInt( bytesPerChecksum );
- }
-
- public byte[] getHeader() {
- byte[] header = new byte[DataChecksum.HEADER_LEN];
- header[0] = (byte) (type.id & 0xff);
- // Writing in buffer just like DataOutput.WriteInt()
- header[1+0] = (byte) ((bytesPerChecksum >>> 24) & 0xff);
- header[1+1] = (byte) ((bytesPerChecksum >>> 16) & 0xff);
- header[1+2] = (byte) ((bytesPerChecksum >>> 8) & 0xff);
- header[1+3] = (byte) (bytesPerChecksum & 0xff);
- return header;
- }
-
- /**
- * Writes the current checksum to the stream.
- * If <i>reset</i> is true, then resets the checksum.
- * @return number of bytes written. Will be equal to getChecksumSize();
- */
- public int writeValue( DataOutputStream out, boolean reset )
- throws IOException {
- if ( type.size <= 0 ) {
- return 0;
- }
-
- if ( type.size == 4 ) {
- out.writeInt( (int) summer.getValue() );
- } else {
- throw new IOException( "Unknown Checksum " + type );
- }
-
- if ( reset ) {
- reset();
- }
-
- return type.size;
- }
-
- /**
- * Writes the current checksum to a buffer.
- * If <i>reset</i> is true, then resets the checksum.
- * @return number of bytes written. Will be equal to getChecksumSize();
- */
- public int writeValue( byte[] buf, int offset, boolean reset )
- throws IOException {
- if ( type.size <= 0 ) {
- return 0;
- }
-
- if ( type.size == 4 ) {
- int checksum = (int) summer.getValue();
- buf[offset+0] = (byte) ((checksum >>> 24) & 0xff);
- buf[offset+1] = (byte) ((checksum >>> 16) & 0xff);
- buf[offset+2] = (byte) ((checksum >>> 8) & 0xff);
- buf[offset+3] = (byte) (checksum & 0xff);
- } else {
- throw new IOException( "Unknown Checksum " + type );
- }
-
- if ( reset ) {
- reset();
- }
-
- return type.size;
- }
-
- /**
- * Compares the checksum located at buf[offset] with the current checksum.
- * @return true if the checksum matches and false otherwise.
- */
- public boolean compare( byte buf[], int offset ) {
- if ( type.size == 4 ) {
- int checksum = ( (buf[offset+0] & 0xff) << 24 ) |
- ( (buf[offset+1] & 0xff) << 16 ) |
- ( (buf[offset+2] & 0xff) << 8 ) |
- ( (buf[offset+3] & 0xff) );
- return checksum == (int) summer.getValue();
- }
- return type.size == 0;
- }
-
- private final Type type;
- private final Checksum summer;
- private final int bytesPerChecksum;
- private int inSum = 0;
-
- private DataChecksum( Type type, Checksum checksum, int chunkSize ) {
- this.type = type;
- summer = checksum;
- bytesPerChecksum = chunkSize;
- }
-
- // Accessors
- public Type getChecksumType() {
- return type;
- }
- public int getChecksumSize() {
- return type.size;
- }
- public int getBytesPerChecksum() {
- return bytesPerChecksum;
- }
- public int getNumBytesInSum() {
- return inSum;
- }
-
- public static final int SIZE_OF_INTEGER = Integer.SIZE / Byte.SIZE;
- static public int getChecksumHeaderSize() {
- return 1 + SIZE_OF_INTEGER; // type byte, bytesPerChecksum int
- }
- //Checksum Interface. Just a wrapper around member summer.
- @Override
- public long getValue() {
- return summer.getValue();
- }
- @Override
- public void reset() {
- summer.reset();
- inSum = 0;
- }
- @Override
- public void update( byte[] b, int off, int len ) {
- if ( len > 0 ) {
- summer.update( b, off, len );
- inSum += len;
- }
- }
- @Override
- public void update( int b ) {
- summer.update( b );
- inSum += 1;
- }
-
- /**
- * Verify that the given checksums match the given data.
- *
- * The 'mark' of the ByteBuffer parameters may be modified by this function,.
- * but the position is maintained.
- *
- * @param data the DirectByteBuffer pointing to the data to verify.
- * @param checksums the DirectByteBuffer pointing to a series of stored
- * checksums
- * @param fileName the name of the file being read, for error-reporting
- * @param basePos the file position to which the start of 'data' corresponds
- * @throws ChecksumException if the checksums do not match
- */
- public void verifyChunkedSums(ByteBuffer data, ByteBuffer checksums,
- String fileName, long basePos)
- throws ChecksumException {
- if (type.size == 0) return;
-
- if (data.hasArray() && checksums.hasArray()) {
- verifyChunkedSums(
- data.array(), data.arrayOffset() + data.position(), data.remaining(),
- checksums.array(), checksums.arrayOffset() + checksums.position(),
- fileName, basePos);
- return;
- }
- if (NativeCrc32.isAvailable()) {
- NativeCrc32.verifyChunkedSums(bytesPerChecksum, type.id, checksums, data,
- fileName, basePos);
- return;
- }
-
- int startDataPos = data.position();
- data.mark();
- checksums.mark();
- try {
- byte[] buf = new byte[bytesPerChecksum];
- byte[] sum = new byte[type.size];
- while (data.remaining() > 0) {
- int n = Math.min(data.remaining(), bytesPerChecksum);
- checksums.get(sum);
- data.get(buf, 0, n);
- summer.reset();
- summer.update(buf, 0, n);
- int calculated = (int)summer.getValue();
- int stored = (sum[0] << 24 & 0xff000000) |
- (sum[1] << 16 & 0xff0000) |
- (sum[2] << 8 & 0xff00) |
- sum[3] & 0xff;
- if (calculated != stored) {
- long errPos = basePos + data.position() - startDataPos - n;
- throw new ChecksumException(
- "Checksum error: "+ fileName + " at "+ errPos +
- " exp: " + stored + " got: " + calculated, errPos);
- }
- }
- } finally {
- data.reset();
- checksums.reset();
- }
- }
-
- /**
- * Implementation of chunked verification specifically on byte arrays. This
- * is to avoid the copy when dealing with ByteBuffers that have array backing.
- */
- private void verifyChunkedSums(
- byte[] data, int dataOff, int dataLen,
- byte[] checksums, int checksumsOff, String fileName,
- long basePos) throws ChecksumException {
-
- int remaining = dataLen;
- int dataPos = 0;
- while (remaining > 0) {
- int n = Math.min(remaining, bytesPerChecksum);
-
- summer.reset();
- summer.update(data, dataOff + dataPos, n);
- dataPos += n;
- remaining -= n;
-
- int calculated = (int)summer.getValue();
- int stored = (checksums[checksumsOff] << 24 & 0xff000000) |
- (checksums[checksumsOff + 1] << 16 & 0xff0000) |
- (checksums[checksumsOff + 2] << 8 & 0xff00) |
- checksums[checksumsOff + 3] & 0xff;
- checksumsOff += 4;
- if (calculated != stored) {
- long errPos = basePos + dataPos - n;
- throw new ChecksumException(
- "Checksum error: "+ fileName + " at "+ errPos +
- " exp: " + stored + " got: " + calculated, errPos);
- }
- }
- }
-
- /**
- * Calculate checksums for the given data.
- *
- * The 'mark' of the ByteBuffer parameters may be modified by this function,
- * but the position is maintained.
- *
- * @param data the DirectByteBuffer pointing to the data to checksum.
- * @param checksums the DirectByteBuffer into which checksums will be
- * stored. Enough space must be available in this
- * buffer to put the checksums.
- */
- public void calculateChunkedSums(ByteBuffer data, ByteBuffer checksums) {
- if (type.size == 0) return;
-
- if (data.hasArray() && checksums.hasArray()) {
- calculateChunkedSums(data.array(), data.arrayOffset() + data.position(), data.remaining(),
- checksums.array(), checksums.arrayOffset() + checksums.position());
- return;
- }
-
- data.mark();
- checksums.mark();
- try {
- byte[] buf = new byte[bytesPerChecksum];
- while (data.remaining() > 0) {
- int n = Math.min(data.remaining(), bytesPerChecksum);
- data.get(buf, 0, n);
- summer.reset();
- summer.update(buf, 0, n);
- checksums.putInt((int)summer.getValue());
- }
- } finally {
- data.reset();
- checksums.reset();
- }
- }
-
- /**
- * Implementation of chunked calculation specifically on byte arrays. This
- * is to avoid the copy when dealing with ByteBuffers that have array backing.
- */
- private void calculateChunkedSums(
- byte[] data, int dataOffset, int dataLength,
- byte[] sums, int sumsOffset) {
-
- int remaining = dataLength;
- while (remaining > 0) {
- int n = Math.min(remaining, bytesPerChecksum);
- summer.reset();
- summer.update(data, dataOffset, n);
- dataOffset += n;
- remaining -= n;
- long calculated = summer.getValue();
- sums[sumsOffset++] = (byte) (calculated >> 24);
- sums[sumsOffset++] = (byte) (calculated >> 16);
- sums[sumsOffset++] = (byte) (calculated >> 8);
- sums[sumsOffset++] = (byte) (calculated);
- }
- }
-
- @Override
- public boolean equals(Object other) {
- if (!(other instanceof DataChecksum)) {
- return false;
- }
- DataChecksum o = (DataChecksum)other;
- return o.bytesPerChecksum == this.bytesPerChecksum &&
- o.type == this.type;
- }
-
- @Override
- public int hashCode() {
- return (this.type.id + 31) * this.bytesPerChecksum;
- }
-
- @Override
- public String toString() {
- return "DataChecksum(type=" + type +
- ", chunkSize=" + bytesPerChecksum + ")";
- }
-
- /**
- * This just provides a dummy implimentation for Checksum class
- * This is used when there is no checksum available or required for
- * data
- */
- static class ChecksumNull implements Checksum {
-
- public ChecksumNull() {}
-
- //Dummy interface
- @Override
- public long getValue() { return 0; }
- @Override
- public void reset() {}
- @Override
- public void update(byte[] b, int off, int len) {}
- @Override
- public void update(int b) {}
- };
-}
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.zip.Checksum;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.ChecksumException;
+
+/**
+ * This class provides inteface and utilities for processing checksums for
+ * DFS data transfers.
+ */
+@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+@InterfaceStability.Evolving
+public class DataChecksum implements Checksum {
+
+ // Misc constants
+ public static final int HEADER_LEN = 5; /// 1 byte type and 4 byte len
+
+ // checksum types
+ public static final int CHECKSUM_NULL = 0;
+ public static final int CHECKSUM_CRC32 = 1;
+ public static final int CHECKSUM_CRC32C = 2;
+ public static final int CHECKSUM_DEFAULT = 3;
+ public static final int CHECKSUM_MIXED = 4;
+
+ /** The checksum types */
+ public static enum Type {
+ NULL (CHECKSUM_NULL, 0),
+ CRC32 (CHECKSUM_CRC32, 4),
+ CRC32C(CHECKSUM_CRC32C, 4),
+ DEFAULT(CHECKSUM_DEFAULT, 0), // This cannot be used to create DataChecksum
+ MIXED (CHECKSUM_MIXED, 0); // This cannot be used to create DataChecksum
+
+ public final int id;
+ public final int size;
+
+ private Type(int id, int size) {
+ this.id = id;
+ this.size = size;
+ }
+
+ /** @return the type corresponding to the id. */
+ public static Type valueOf(int id) {
+ if (id < 0 || id >= values().length) {
+ throw new IllegalArgumentException("id=" + id
+ + " out of range [0, " + values().length + ")");
+ }
+ return values()[id];
+ }
+ }
+
+
+ public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) {
+ if ( bytesPerChecksum <= 0 ) {
+ return null;
+ }
+
+ switch ( type ) {
+ case NULL :
+ return new DataChecksum(type, new ChecksumNull(), bytesPerChecksum );
+ case CRC32 :
+ return new DataChecksum(type, new PureJavaCrc32(), bytesPerChecksum );
+ case CRC32C:
+ return new DataChecksum(type, new PureJavaCrc32C(), bytesPerChecksum);
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Creates a DataChecksum from HEADER_LEN bytes from arr[offset].
+ * @return DataChecksum of the type in the array or null in case of an error.
+ */
+ public static DataChecksum newDataChecksum( byte bytes[], int offset ) {
+ if ( offset < 0 || bytes.length < offset + HEADER_LEN ) {
+ return null;
+ }
+
+ // like readInt():
+ int bytesPerChecksum = ( (bytes[offset+1] & 0xff) << 24 ) |
+ ( (bytes[offset+2] & 0xff) << 16 ) |
+ ( (bytes[offset+3] & 0xff) << 8 ) |
+ ( (bytes[offset+4] & 0xff) );
+ return newDataChecksum( Type.valueOf(bytes[0]), bytesPerChecksum );
+ }
+
+ /**
+ * This constructucts a DataChecksum by reading HEADER_LEN bytes from
+ * input stream <i>in</i>
+ */
+ public static DataChecksum newDataChecksum( DataInputStream in )
+ throws IOException {
+ int type = in.readByte();
+ int bpc = in.readInt();
+ DataChecksum summer = newDataChecksum(Type.valueOf(type), bpc );
+ if ( summer == null ) {
+ throw new IOException( "Could not create DataChecksum of type " +
+ type + " with bytesPerChecksum " + bpc );
+ }
+ return summer;
+ }
+
+ /**
+ * Writes the checksum header to the output stream <i>out</i>.
+ */
+ public void writeHeader( DataOutputStream out )
+ throws IOException {
+ out.writeByte( type.id );
+ out.writeInt( bytesPerChecksum );
+ }
+
+ public byte[] getHeader() {
+ byte[] header = new byte[DataChecksum.HEADER_LEN];
+ header[0] = (byte) (type.id & 0xff);
+ // Writing in buffer just like DataOutput.WriteInt()
+ header[1+0] = (byte) ((bytesPerChecksum >>> 24) & 0xff);
+ header[1+1] = (byte) ((bytesPerChecksum >>> 16) & 0xff);
+ header[1+2] = (byte) ((bytesPerChecksum >>> 8) & 0xff);
+ header[1+3] = (byte) (bytesPerChecksum & 0xff);
+ return header;
+ }
+
+ /**
+ * Writes the current checksum to the stream.
+ * If <i>reset</i> is true, then resets the checksum.
+ * @return number of bytes written. Will be equal to getChecksumSize();
+ */
+ public int writeValue( DataOutputStream out, boolean reset )
+ throws IOException {
+ if ( type.size <= 0 ) {
+ return 0;
+ }
+
+ if ( type.size == 4 ) {
+ out.writeInt( (int) summer.getValue() );
+ } else {
+ throw new IOException( "Unknown Checksum " + type );
+ }
+
+ if ( reset ) {
+ reset();
+ }
+
+ return type.size;
+ }
+
+ /**
+ * Writes the current checksum to a buffer.
+ * If <i>reset</i> is true, then resets the checksum.
+ * @return number of bytes written. Will be equal to getChecksumSize();
+ */
+ public int writeValue( byte[] buf, int offset, boolean reset )
+ throws IOException {
+ if ( type.size <= 0 ) {
+ return 0;
+ }
+
+ if ( type.size == 4 ) {
+ int checksum = (int) summer.getValue();
+ buf[offset+0] = (byte) ((checksum >>> 24) & 0xff);
+ buf[offset+1] = (byte) ((checksum >>> 16) & 0xff);
+ buf[offset+2] = (byte) ((checksum >>> 8) & 0xff);
+ buf[offset+3] = (byte) (checksum & 0xff);
+ } else {
+ throw new IOException( "Unknown Checksum " + type );
+ }
+
+ if ( reset ) {
+ reset();
+ }
+
+ return type.size;
+ }
+
+ /**
+ * Compares the checksum located at buf[offset] with the current checksum.
+ * @return true if the checksum matches and false otherwise.
+ */
+ public boolean compare( byte buf[], int offset ) {
+ if ( type.size == 4 ) {
+ int checksum = ( (buf[offset+0] & 0xff) << 24 ) |
+ ( (buf[offset+1] & 0xff) << 16 ) |
+ ( (buf[offset+2] & 0xff) << 8 ) |
+ ( (buf[offset+3] & 0xff) );
+ return checksum == (int) summer.getValue();
+ }
+ return type.size == 0;
+ }
+
+ private final Type type;
+ private final Checksum summer;
+ private final int bytesPerChecksum;
+ private int inSum = 0;
+
+ private DataChecksum( Type type, Checksum checksum, int chunkSize ) {
+ this.type = type;
+ summer = checksum;
+ bytesPerChecksum = chunkSize;
+ }
+
+ // Accessors
+ public Type getChecksumType() {
+ return type;
+ }
+ public int getChecksumSize() {
+ return type.size;
+ }
+ public int getBytesPerChecksum() {
+ return bytesPerChecksum;
+ }
+ public int getNumBytesInSum() {
+ return inSum;
+ }
+
+ public static final int SIZE_OF_INTEGER = Integer.SIZE / Byte.SIZE;
+ static public int getChecksumHeaderSize() {
+ return 1 + SIZE_OF_INTEGER; // type byte, bytesPerChecksum int
+ }
+ //Checksum Interface. Just a wrapper around member summer.
+ @Override
+ public long getValue() {
+ return summer.getValue();
+ }
+ @Override
+ public void reset() {
+ summer.reset();
+ inSum = 0;
+ }
+ @Override
+ public void update( byte[] b, int off, int len ) {
+ if ( len > 0 ) {
+ summer.update( b, off, len );
+ inSum += len;
+ }
+ }
+ @Override
+ public void update( int b ) {
+ summer.update( b );
+ inSum += 1;
+ }
+
+ /**
+ * Verify that the given checksums match the given data.
+ *
+ * The 'mark' of the ByteBuffer parameters may be modified by this function,.
+ * but the position is maintained.
+ *
+ * @param data the DirectByteBuffer pointing to the data to verify.
+ * @param checksums the DirectByteBuffer pointing to a series of stored
+ * checksums
+ * @param fileName the name of the file being read, for error-reporting
+ * @param basePos the file position to which the start of 'data' corresponds
+ * @throws ChecksumException if the checksums do not match
+ */
+ public void verifyChunkedSums(ByteBuffer data, ByteBuffer checksums,
+ String fileName, long basePos)
+ throws ChecksumException {
+ if (type.size == 0) return;
+
+ if (data.hasArray() && checksums.hasArray()) {
+ verifyChunkedSums(
+ data.array(), data.arrayOffset() + data.position(), data.remaining(),
+ checksums.array(), checksums.arrayOffset() + checksums.position(),
+ fileName, basePos);
+ return;
+ }
+ if (NativeCrc32.isAvailable()) {
+ NativeCrc32.verifyChunkedSums(bytesPerChecksum, type.id, checksums, data,
+ fileName, basePos);
+ return;
+ }
+
+ int startDataPos = data.position();
+ data.mark();
+ checksums.mark();
+ try {
+ byte[] buf = new byte[bytesPerChecksum];
+ byte[] sum = new byte[type.size];
+ while (data.remaining() > 0) {
+ int n = Math.min(data.remaining(), bytesPerChecksum);
+ checksums.get(sum);
+ data.get(buf, 0, n);
+ summer.reset();
+ summer.update(buf, 0, n);
+ int calculated = (int)summer.getValue();
+ int stored = (sum[0] << 24 & 0xff000000) |
+ (sum[1] << 16 & 0xff0000) |
+ (sum[2] << 8 & 0xff00) |
+ sum[3] & 0xff;
+ if (calculated != stored) {
+ long errPos = basePos + data.position() - startDataPos - n;
+ throw new ChecksumException(
+ "Checksum error: "+ fileName + " at "+ errPos +
+ " exp: " + stored + " got: " + calculated, errPos);
+ }
+ }
+ } finally {
+ data.reset();
+ checksums.reset();
+ }
+ }
+
+ /**
+ * Implementation of chunked verification specifically on byte arrays. This
+ * is to avoid the copy when dealing with ByteBuffers that have array backing.
+ */
+ private void verifyChunkedSums(
+ byte[] data, int dataOff, int dataLen,
+ byte[] checksums, int checksumsOff, String fileName,
+ long basePos) throws ChecksumException {
+
+ int remaining = dataLen;
+ int dataPos = 0;
+ while (remaining > 0) {
+ int n = Math.min(remaining, bytesPerChecksum);
+
+ summer.reset();
+ summer.update(data, dataOff + dataPos, n);
+ dataPos += n;
+ remaining -= n;
+
+ int calculated = (int)summer.getValue();
+ int stored = (checksums[checksumsOff] << 24 & 0xff000000) |
+ (checksums[checksumsOff + 1] << 16 & 0xff0000) |
+ (checksums[checksumsOff + 2] << 8 & 0xff00) |
+ checksums[checksumsOff + 3] & 0xff;
+ checksumsOff += 4;
+ if (calculated != stored) {
+ long errPos = basePos + dataPos - n;
+ throw new ChecksumException(
+ "Checksum error: "+ fileName + " at "+ errPos +
+ " exp: " + stored + " got: " + calculated, errPos);
+ }
+ }
+ }
+
+ /**
+ * Calculate checksums for the given data.
+ *
+ * The 'mark' of the ByteBuffer parameters may be modified by this function,
+ * but the position is maintained.
+ *
+ * @param data the DirectByteBuffer pointing to the data to checksum.
+ * @param checksums the DirectByteBuffer into which checksums will be
+ * stored. Enough space must be available in this
+ * buffer to put the checksums.
+ */
+ public void calculateChunkedSums(ByteBuffer data, ByteBuffer checksums) {
+ if (type.size == 0) return;
+
+ if (data.hasArray() && checksums.hasArray()) {
+ calculateChunkedSums(data.array(), data.arrayOffset() + data.position(), data.remaining(),
+ checksums.array(), checksums.arrayOffset() + checksums.position());
+ return;
+ }
+
+ data.mark();
+ checksums.mark();
+ try {
+ byte[] buf = new byte[bytesPerChecksum];
+ while (data.remaining() > 0) {
+ int n = Math.min(data.remaining(), bytesPerChecksum);
+ data.get(buf, 0, n);
+ summer.reset();
+ summer.update(buf, 0, n);
+ checksums.putInt((int)summer.getValue());
+ }
+ } finally {
+ data.reset();
+ checksums.reset();
+ }
+ }
+
+ /**
+ * Implementation of chunked calculation specifically on byte arrays. This
+ * is to avoid the copy when dealing with ByteBuffers that have array backing.
+ */
+ private void calculateChunkedSums(
+ byte[] data, int dataOffset, int dataLength,
+ byte[] sums, int sumsOffset) {
+
+ int remaining = dataLength;
+ while (remaining > 0) {
+ int n = Math.min(remaining, bytesPerChecksum);
+ summer.reset();
+ summer.update(data, dataOffset, n);
+ dataOffset += n;
+ remaining -= n;
+ long calculated = summer.getValue();
+ sums[sumsOffset++] = (byte) (calculated >> 24);
+ sums[sumsOffset++] = (byte) (calculated >> 16);
+ sums[sumsOffset++] = (byte) (calculated >> 8);
+ sums[sumsOffset++] = (byte) (calculated);
+ }
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof DataChecksum)) {
+ return false;
+ }
+ DataChecksum o = (DataChecksum)other;
+ return o.bytesPerChecksum == this.bytesPerChecksum &&
+ o.type == this.type;
+ }
+
+ @Override
+ public int hashCode() {
+ return (this.type.id + 31) * this.bytesPerChecksum;
+ }
+
+ @Override
+ public String toString() {
+ return "DataChecksum(type=" + type +
+ ", chunkSize=" + bytesPerChecksum + ")";
+ }
+
+ /**
+ * This just provides a dummy implimentation for Checksum class
+ * This is used when there is no checksum available or required for
+ * data
+ */
+ static class ChecksumNull implements Checksum {
+
+ public ChecksumNull() {}
+
+ //Dummy interface
+ @Override
+ public long getValue() { return 0; }
+ @Override
+ public void reset() {}
+ @Override
+ public void update(byte[] b, int off, int len) {}
+ @Override
+ public void update(int b) {}
+ };
+}