You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2012/10/22 22:43:25 UTC
svn commit: r1401071 [4/4] - in
/hadoop/common/branches/branch-trunk-win/hadoop-common-project:
hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/
hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/
hado...
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java Mon Oct 22 20:43:16 2012
@@ -1,460 +1,460 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.util;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.zip.Checksum;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.ChecksumException;
-
-/**
- * This class provides inteface and utilities for processing checksums for
- * DFS data transfers.
- */
-@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
-@InterfaceStability.Evolving
-public class DataChecksum implements Checksum {
-
- // Misc constants
- public static final int HEADER_LEN = 5; /// 1 byte type and 4 byte len
-
- // checksum types
- public static final int CHECKSUM_NULL = 0;
- public static final int CHECKSUM_CRC32 = 1;
- public static final int CHECKSUM_CRC32C = 2;
- public static final int CHECKSUM_DEFAULT = 3;
- public static final int CHECKSUM_MIXED = 4;
-
- /** The checksum types */
- public static enum Type {
- NULL (CHECKSUM_NULL, 0),
- CRC32 (CHECKSUM_CRC32, 4),
- CRC32C(CHECKSUM_CRC32C, 4),
- DEFAULT(CHECKSUM_DEFAULT, 0), // This cannot be used to create DataChecksum
- MIXED (CHECKSUM_MIXED, 0); // This cannot be used to create DataChecksum
-
- public final int id;
- public final int size;
-
- private Type(int id, int size) {
- this.id = id;
- this.size = size;
- }
-
- /** @return the type corresponding to the id. */
- public static Type valueOf(int id) {
- if (id < 0 || id >= values().length) {
- throw new IllegalArgumentException("id=" + id
- + " out of range [0, " + values().length + ")");
- }
- return values()[id];
- }
- }
-
-
- public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) {
- if ( bytesPerChecksum <= 0 ) {
- return null;
- }
-
- switch ( type ) {
- case NULL :
- return new DataChecksum(type, new ChecksumNull(), bytesPerChecksum );
- case CRC32 :
- return new DataChecksum(type, new PureJavaCrc32(), bytesPerChecksum );
- case CRC32C:
- return new DataChecksum(type, new PureJavaCrc32C(), bytesPerChecksum);
- default:
- return null;
- }
- }
-
- /**
- * Creates a DataChecksum from HEADER_LEN bytes from arr[offset].
- * @return DataChecksum of the type in the array or null in case of an error.
- */
- public static DataChecksum newDataChecksum( byte bytes[], int offset ) {
- if ( offset < 0 || bytes.length < offset + HEADER_LEN ) {
- return null;
- }
-
- // like readInt():
- int bytesPerChecksum = ( (bytes[offset+1] & 0xff) << 24 ) |
- ( (bytes[offset+2] & 0xff) << 16 ) |
- ( (bytes[offset+3] & 0xff) << 8 ) |
- ( (bytes[offset+4] & 0xff) );
- return newDataChecksum( Type.valueOf(bytes[0]), bytesPerChecksum );
- }
-
- /**
- * This constructucts a DataChecksum by reading HEADER_LEN bytes from
- * input stream <i>in</i>
- */
- public static DataChecksum newDataChecksum( DataInputStream in )
- throws IOException {
- int type = in.readByte();
- int bpc = in.readInt();
- DataChecksum summer = newDataChecksum(Type.valueOf(type), bpc );
- if ( summer == null ) {
- throw new IOException( "Could not create DataChecksum of type " +
- type + " with bytesPerChecksum " + bpc );
- }
- return summer;
- }
-
- /**
- * Writes the checksum header to the output stream <i>out</i>.
- */
- public void writeHeader( DataOutputStream out )
- throws IOException {
- out.writeByte( type.id );
- out.writeInt( bytesPerChecksum );
- }
-
- public byte[] getHeader() {
- byte[] header = new byte[DataChecksum.HEADER_LEN];
- header[0] = (byte) (type.id & 0xff);
- // Writing in buffer just like DataOutput.WriteInt()
- header[1+0] = (byte) ((bytesPerChecksum >>> 24) & 0xff);
- header[1+1] = (byte) ((bytesPerChecksum >>> 16) & 0xff);
- header[1+2] = (byte) ((bytesPerChecksum >>> 8) & 0xff);
- header[1+3] = (byte) (bytesPerChecksum & 0xff);
- return header;
- }
-
- /**
- * Writes the current checksum to the stream.
- * If <i>reset</i> is true, then resets the checksum.
- * @return number of bytes written. Will be equal to getChecksumSize();
- */
- public int writeValue( DataOutputStream out, boolean reset )
- throws IOException {
- if ( type.size <= 0 ) {
- return 0;
- }
-
- if ( type.size == 4 ) {
- out.writeInt( (int) summer.getValue() );
- } else {
- throw new IOException( "Unknown Checksum " + type );
- }
-
- if ( reset ) {
- reset();
- }
-
- return type.size;
- }
-
- /**
- * Writes the current checksum to a buffer.
- * If <i>reset</i> is true, then resets the checksum.
- * @return number of bytes written. Will be equal to getChecksumSize();
- */
- public int writeValue( byte[] buf, int offset, boolean reset )
- throws IOException {
- if ( type.size <= 0 ) {
- return 0;
- }
-
- if ( type.size == 4 ) {
- int checksum = (int) summer.getValue();
- buf[offset+0] = (byte) ((checksum >>> 24) & 0xff);
- buf[offset+1] = (byte) ((checksum >>> 16) & 0xff);
- buf[offset+2] = (byte) ((checksum >>> 8) & 0xff);
- buf[offset+3] = (byte) (checksum & 0xff);
- } else {
- throw new IOException( "Unknown Checksum " + type );
- }
-
- if ( reset ) {
- reset();
- }
-
- return type.size;
- }
-
- /**
- * Compares the checksum located at buf[offset] with the current checksum.
- * @return true if the checksum matches and false otherwise.
- */
- public boolean compare( byte buf[], int offset ) {
- if ( type.size == 4 ) {
- int checksum = ( (buf[offset+0] & 0xff) << 24 ) |
- ( (buf[offset+1] & 0xff) << 16 ) |
- ( (buf[offset+2] & 0xff) << 8 ) |
- ( (buf[offset+3] & 0xff) );
- return checksum == (int) summer.getValue();
- }
- return type.size == 0;
- }
-
- private final Type type;
- private final Checksum summer;
- private final int bytesPerChecksum;
- private int inSum = 0;
-
- private DataChecksum( Type type, Checksum checksum, int chunkSize ) {
- this.type = type;
- summer = checksum;
- bytesPerChecksum = chunkSize;
- }
-
- // Accessors
- public Type getChecksumType() {
- return type;
- }
- public int getChecksumSize() {
- return type.size;
- }
- public int getBytesPerChecksum() {
- return bytesPerChecksum;
- }
- public int getNumBytesInSum() {
- return inSum;
- }
-
- public static final int SIZE_OF_INTEGER = Integer.SIZE / Byte.SIZE;
- static public int getChecksumHeaderSize() {
- return 1 + SIZE_OF_INTEGER; // type byte, bytesPerChecksum int
- }
- //Checksum Interface. Just a wrapper around member summer.
- @Override
- public long getValue() {
- return summer.getValue();
- }
- @Override
- public void reset() {
- summer.reset();
- inSum = 0;
- }
- @Override
- public void update( byte[] b, int off, int len ) {
- if ( len > 0 ) {
- summer.update( b, off, len );
- inSum += len;
- }
- }
- @Override
- public void update( int b ) {
- summer.update( b );
- inSum += 1;
- }
-
- /**
- * Verify that the given checksums match the given data.
- *
- * The 'mark' of the ByteBuffer parameters may be modified by this function,.
- * but the position is maintained.
- *
- * @param data the DirectByteBuffer pointing to the data to verify.
- * @param checksums the DirectByteBuffer pointing to a series of stored
- * checksums
- * @param fileName the name of the file being read, for error-reporting
- * @param basePos the file position to which the start of 'data' corresponds
- * @throws ChecksumException if the checksums do not match
- */
- public void verifyChunkedSums(ByteBuffer data, ByteBuffer checksums,
- String fileName, long basePos)
- throws ChecksumException {
- if (type.size == 0) return;
-
- if (data.hasArray() && checksums.hasArray()) {
- verifyChunkedSums(
- data.array(), data.arrayOffset() + data.position(), data.remaining(),
- checksums.array(), checksums.arrayOffset() + checksums.position(),
- fileName, basePos);
- return;
- }
- if (NativeCrc32.isAvailable()) {
- NativeCrc32.verifyChunkedSums(bytesPerChecksum, type.id, checksums, data,
- fileName, basePos);
- return;
- }
-
- int startDataPos = data.position();
- data.mark();
- checksums.mark();
- try {
- byte[] buf = new byte[bytesPerChecksum];
- byte[] sum = new byte[type.size];
- while (data.remaining() > 0) {
- int n = Math.min(data.remaining(), bytesPerChecksum);
- checksums.get(sum);
- data.get(buf, 0, n);
- summer.reset();
- summer.update(buf, 0, n);
- int calculated = (int)summer.getValue();
- int stored = (sum[0] << 24 & 0xff000000) |
- (sum[1] << 16 & 0xff0000) |
- (sum[2] << 8 & 0xff00) |
- sum[3] & 0xff;
- if (calculated != stored) {
- long errPos = basePos + data.position() - startDataPos - n;
- throw new ChecksumException(
- "Checksum error: "+ fileName + " at "+ errPos +
- " exp: " + stored + " got: " + calculated, errPos);
- }
- }
- } finally {
- data.reset();
- checksums.reset();
- }
- }
-
- /**
- * Implementation of chunked verification specifically on byte arrays. This
- * is to avoid the copy when dealing with ByteBuffers that have array backing.
- */
- private void verifyChunkedSums(
- byte[] data, int dataOff, int dataLen,
- byte[] checksums, int checksumsOff, String fileName,
- long basePos) throws ChecksumException {
-
- int remaining = dataLen;
- int dataPos = 0;
- while (remaining > 0) {
- int n = Math.min(remaining, bytesPerChecksum);
-
- summer.reset();
- summer.update(data, dataOff + dataPos, n);
- dataPos += n;
- remaining -= n;
-
- int calculated = (int)summer.getValue();
- int stored = (checksums[checksumsOff] << 24 & 0xff000000) |
- (checksums[checksumsOff + 1] << 16 & 0xff0000) |
- (checksums[checksumsOff + 2] << 8 & 0xff00) |
- checksums[checksumsOff + 3] & 0xff;
- checksumsOff += 4;
- if (calculated != stored) {
- long errPos = basePos + dataPos - n;
- throw new ChecksumException(
- "Checksum error: "+ fileName + " at "+ errPos +
- " exp: " + stored + " got: " + calculated, errPos);
- }
- }
- }
-
- /**
- * Calculate checksums for the given data.
- *
- * The 'mark' of the ByteBuffer parameters may be modified by this function,
- * but the position is maintained.
- *
- * @param data the DirectByteBuffer pointing to the data to checksum.
- * @param checksums the DirectByteBuffer into which checksums will be
- * stored. Enough space must be available in this
- * buffer to put the checksums.
- */
- public void calculateChunkedSums(ByteBuffer data, ByteBuffer checksums) {
- if (type.size == 0) return;
-
- if (data.hasArray() && checksums.hasArray()) {
- calculateChunkedSums(data.array(), data.arrayOffset() + data.position(), data.remaining(),
- checksums.array(), checksums.arrayOffset() + checksums.position());
- return;
- }
-
- data.mark();
- checksums.mark();
- try {
- byte[] buf = new byte[bytesPerChecksum];
- while (data.remaining() > 0) {
- int n = Math.min(data.remaining(), bytesPerChecksum);
- data.get(buf, 0, n);
- summer.reset();
- summer.update(buf, 0, n);
- checksums.putInt((int)summer.getValue());
- }
- } finally {
- data.reset();
- checksums.reset();
- }
- }
-
- /**
- * Implementation of chunked calculation specifically on byte arrays. This
- * is to avoid the copy when dealing with ByteBuffers that have array backing.
- */
- private void calculateChunkedSums(
- byte[] data, int dataOffset, int dataLength,
- byte[] sums, int sumsOffset) {
-
- int remaining = dataLength;
- while (remaining > 0) {
- int n = Math.min(remaining, bytesPerChecksum);
- summer.reset();
- summer.update(data, dataOffset, n);
- dataOffset += n;
- remaining -= n;
- long calculated = summer.getValue();
- sums[sumsOffset++] = (byte) (calculated >> 24);
- sums[sumsOffset++] = (byte) (calculated >> 16);
- sums[sumsOffset++] = (byte) (calculated >> 8);
- sums[sumsOffset++] = (byte) (calculated);
- }
- }
-
- @Override
- public boolean equals(Object other) {
- if (!(other instanceof DataChecksum)) {
- return false;
- }
- DataChecksum o = (DataChecksum)other;
- return o.bytesPerChecksum == this.bytesPerChecksum &&
- o.type == this.type;
- }
-
- @Override
- public int hashCode() {
- return (this.type.id + 31) * this.bytesPerChecksum;
- }
-
- @Override
- public String toString() {
- return "DataChecksum(type=" + type +
- ", chunkSize=" + bytesPerChecksum + ")";
- }
-
- /**
- * This just provides a dummy implimentation for Checksum class
- * This is used when there is no checksum available or required for
- * data
- */
- static class ChecksumNull implements Checksum {
-
- public ChecksumNull() {}
-
- //Dummy interface
- @Override
- public long getValue() { return 0; }
- @Override
- public void reset() {}
- @Override
- public void update(byte[] b, int off, int len) {}
- @Override
- public void update(int b) {}
- };
-}
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.zip.Checksum;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.ChecksumException;
+
+/**
+ * This class provides inteface and utilities for processing checksums for
+ * DFS data transfers.
+ */
+@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+@InterfaceStability.Evolving
+public class DataChecksum implements Checksum {
+
+ // Misc constants
+ public static final int HEADER_LEN = 5; /// 1 byte type and 4 byte len
+
+ // checksum types
+ public static final int CHECKSUM_NULL = 0;
+ public static final int CHECKSUM_CRC32 = 1;
+ public static final int CHECKSUM_CRC32C = 2;
+ public static final int CHECKSUM_DEFAULT = 3;
+ public static final int CHECKSUM_MIXED = 4;
+
+ /** The checksum types */
+ public static enum Type {
+ NULL (CHECKSUM_NULL, 0),
+ CRC32 (CHECKSUM_CRC32, 4),
+ CRC32C(CHECKSUM_CRC32C, 4),
+ DEFAULT(CHECKSUM_DEFAULT, 0), // This cannot be used to create DataChecksum
+ MIXED (CHECKSUM_MIXED, 0); // This cannot be used to create DataChecksum
+
+ public final int id;
+ public final int size;
+
+ private Type(int id, int size) {
+ this.id = id;
+ this.size = size;
+ }
+
+ /** @return the type corresponding to the id. */
+ public static Type valueOf(int id) {
+ if (id < 0 || id >= values().length) {
+ throw new IllegalArgumentException("id=" + id
+ + " out of range [0, " + values().length + ")");
+ }
+ return values()[id];
+ }
+ }
+
+
+ public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) {
+ if ( bytesPerChecksum <= 0 ) {
+ return null;
+ }
+
+ switch ( type ) {
+ case NULL :
+ return new DataChecksum(type, new ChecksumNull(), bytesPerChecksum );
+ case CRC32 :
+ return new DataChecksum(type, new PureJavaCrc32(), bytesPerChecksum );
+ case CRC32C:
+ return new DataChecksum(type, new PureJavaCrc32C(), bytesPerChecksum);
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Creates a DataChecksum from HEADER_LEN bytes from arr[offset].
+ * @return DataChecksum of the type in the array or null in case of an error.
+ */
+ public static DataChecksum newDataChecksum( byte bytes[], int offset ) {
+ if ( offset < 0 || bytes.length < offset + HEADER_LEN ) {
+ return null;
+ }
+
+ // like readInt():
+ int bytesPerChecksum = ( (bytes[offset+1] & 0xff) << 24 ) |
+ ( (bytes[offset+2] & 0xff) << 16 ) |
+ ( (bytes[offset+3] & 0xff) << 8 ) |
+ ( (bytes[offset+4] & 0xff) );
+ return newDataChecksum( Type.valueOf(bytes[0]), bytesPerChecksum );
+ }
+
+ /**
+ * This constructucts a DataChecksum by reading HEADER_LEN bytes from
+ * input stream <i>in</i>
+ */
+ public static DataChecksum newDataChecksum( DataInputStream in )
+ throws IOException {
+ int type = in.readByte();
+ int bpc = in.readInt();
+ DataChecksum summer = newDataChecksum(Type.valueOf(type), bpc );
+ if ( summer == null ) {
+ throw new IOException( "Could not create DataChecksum of type " +
+ type + " with bytesPerChecksum " + bpc );
+ }
+ return summer;
+ }
+
+ /**
+ * Writes the checksum header to the output stream <i>out</i>.
+ */
+ public void writeHeader( DataOutputStream out )
+ throws IOException {
+ out.writeByte( type.id );
+ out.writeInt( bytesPerChecksum );
+ }
+
+ public byte[] getHeader() {
+ byte[] header = new byte[DataChecksum.HEADER_LEN];
+ header[0] = (byte) (type.id & 0xff);
+ // Writing in buffer just like DataOutput.WriteInt()
+ header[1+0] = (byte) ((bytesPerChecksum >>> 24) & 0xff);
+ header[1+1] = (byte) ((bytesPerChecksum >>> 16) & 0xff);
+ header[1+2] = (byte) ((bytesPerChecksum >>> 8) & 0xff);
+ header[1+3] = (byte) (bytesPerChecksum & 0xff);
+ return header;
+ }
+
+ /**
+ * Writes the current checksum to the stream.
+ * If <i>reset</i> is true, then resets the checksum.
+ * @return number of bytes written. Will be equal to getChecksumSize();
+ */
+ public int writeValue( DataOutputStream out, boolean reset )
+ throws IOException {
+ if ( type.size <= 0 ) {
+ return 0;
+ }
+
+ if ( type.size == 4 ) {
+ out.writeInt( (int) summer.getValue() );
+ } else {
+ throw new IOException( "Unknown Checksum " + type );
+ }
+
+ if ( reset ) {
+ reset();
+ }
+
+ return type.size;
+ }
+
+ /**
+ * Writes the current checksum to a buffer.
+ * If <i>reset</i> is true, then resets the checksum.
+ * @return number of bytes written. Will be equal to getChecksumSize();
+ */
+ public int writeValue( byte[] buf, int offset, boolean reset )
+ throws IOException {
+ if ( type.size <= 0 ) {
+ return 0;
+ }
+
+ if ( type.size == 4 ) {
+ int checksum = (int) summer.getValue();
+ buf[offset+0] = (byte) ((checksum >>> 24) & 0xff);
+ buf[offset+1] = (byte) ((checksum >>> 16) & 0xff);
+ buf[offset+2] = (byte) ((checksum >>> 8) & 0xff);
+ buf[offset+3] = (byte) (checksum & 0xff);
+ } else {
+ throw new IOException( "Unknown Checksum " + type );
+ }
+
+ if ( reset ) {
+ reset();
+ }
+
+ return type.size;
+ }
+
+ /**
+ * Compares the checksum located at buf[offset] with the current checksum.
+ * @return true if the checksum matches and false otherwise.
+ */
+ public boolean compare( byte buf[], int offset ) {
+ if ( type.size == 4 ) {
+ int checksum = ( (buf[offset+0] & 0xff) << 24 ) |
+ ( (buf[offset+1] & 0xff) << 16 ) |
+ ( (buf[offset+2] & 0xff) << 8 ) |
+ ( (buf[offset+3] & 0xff) );
+ return checksum == (int) summer.getValue();
+ }
+ return type.size == 0;
+ }
+
+ private final Type type;
+ private final Checksum summer;
+ private final int bytesPerChecksum;
+ private int inSum = 0;
+
+ private DataChecksum( Type type, Checksum checksum, int chunkSize ) {
+ this.type = type;
+ summer = checksum;
+ bytesPerChecksum = chunkSize;
+ }
+
+ // Accessors
+ public Type getChecksumType() {
+ return type;
+ }
+ public int getChecksumSize() {
+ return type.size;
+ }
+ public int getBytesPerChecksum() {
+ return bytesPerChecksum;
+ }
+ public int getNumBytesInSum() {
+ return inSum;
+ }
+
+ public static final int SIZE_OF_INTEGER = Integer.SIZE / Byte.SIZE;
+ static public int getChecksumHeaderSize() {
+ return 1 + SIZE_OF_INTEGER; // type byte, bytesPerChecksum int
+ }
+ //Checksum Interface. Just a wrapper around member summer.
+ @Override
+ public long getValue() {
+ return summer.getValue();
+ }
+ @Override
+ public void reset() {
+ summer.reset();
+ inSum = 0;
+ }
+ @Override
+ public void update( byte[] b, int off, int len ) {
+ if ( len > 0 ) {
+ summer.update( b, off, len );
+ inSum += len;
+ }
+ }
+ @Override
+ public void update( int b ) {
+ summer.update( b );
+ inSum += 1;
+ }
+
+ /**
+ * Verify that the given checksums match the given data.
+ *
+ * The 'mark' of the ByteBuffer parameters may be modified by this function,.
+ * but the position is maintained.
+ *
+ * @param data the DirectByteBuffer pointing to the data to verify.
+ * @param checksums the DirectByteBuffer pointing to a series of stored
+ * checksums
+ * @param fileName the name of the file being read, for error-reporting
+ * @param basePos the file position to which the start of 'data' corresponds
+ * @throws ChecksumException if the checksums do not match
+ */
+ public void verifyChunkedSums(ByteBuffer data, ByteBuffer checksums,
+ String fileName, long basePos)
+ throws ChecksumException {
+ if (type.size == 0) return;
+
+ if (data.hasArray() && checksums.hasArray()) {
+ verifyChunkedSums(
+ data.array(), data.arrayOffset() + data.position(), data.remaining(),
+ checksums.array(), checksums.arrayOffset() + checksums.position(),
+ fileName, basePos);
+ return;
+ }
+ if (NativeCrc32.isAvailable()) {
+ NativeCrc32.verifyChunkedSums(bytesPerChecksum, type.id, checksums, data,
+ fileName, basePos);
+ return;
+ }
+
+ int startDataPos = data.position();
+ data.mark();
+ checksums.mark();
+ try {
+ byte[] buf = new byte[bytesPerChecksum];
+ byte[] sum = new byte[type.size];
+ while (data.remaining() > 0) {
+ int n = Math.min(data.remaining(), bytesPerChecksum);
+ checksums.get(sum);
+ data.get(buf, 0, n);
+ summer.reset();
+ summer.update(buf, 0, n);
+ int calculated = (int)summer.getValue();
+ int stored = (sum[0] << 24 & 0xff000000) |
+ (sum[1] << 16 & 0xff0000) |
+ (sum[2] << 8 & 0xff00) |
+ sum[3] & 0xff;
+ if (calculated != stored) {
+ long errPos = basePos + data.position() - startDataPos - n;
+ throw new ChecksumException(
+ "Checksum error: "+ fileName + " at "+ errPos +
+ " exp: " + stored + " got: " + calculated, errPos);
+ }
+ }
+ } finally {
+ data.reset();
+ checksums.reset();
+ }
+ }
+
+ /**
+ * Implementation of chunked verification specifically on byte arrays. This
+ * is to avoid the copy when dealing with ByteBuffers that have array backing.
+ */
+ private void verifyChunkedSums(
+ byte[] data, int dataOff, int dataLen,
+ byte[] checksums, int checksumsOff, String fileName,
+ long basePos) throws ChecksumException {
+
+ int remaining = dataLen;
+ int dataPos = 0;
+ while (remaining > 0) {
+ int n = Math.min(remaining, bytesPerChecksum);
+
+ summer.reset();
+ summer.update(data, dataOff + dataPos, n);
+ dataPos += n;
+ remaining -= n;
+
+ int calculated = (int)summer.getValue();
+ int stored = (checksums[checksumsOff] << 24 & 0xff000000) |
+ (checksums[checksumsOff + 1] << 16 & 0xff0000) |
+ (checksums[checksumsOff + 2] << 8 & 0xff00) |
+ checksums[checksumsOff + 3] & 0xff;
+ checksumsOff += 4;
+ if (calculated != stored) {
+ long errPos = basePos + dataPos - n;
+ throw new ChecksumException(
+ "Checksum error: "+ fileName + " at "+ errPos +
+ " exp: " + stored + " got: " + calculated, errPos);
+ }
+ }
+ }
+
+ /**
+ * Calculate checksums for the given data.
+ *
+ * The 'mark' of the ByteBuffer parameters may be modified by this function,
+ * but the position is maintained.
+ *
+ * @param data the DirectByteBuffer pointing to the data to checksum.
+ * @param checksums the DirectByteBuffer into which checksums will be
+ * stored. Enough space must be available in this
+ * buffer to put the checksums.
+ */
+ public void calculateChunkedSums(ByteBuffer data, ByteBuffer checksums) {
+ if (type.size == 0) return;
+
+ if (data.hasArray() && checksums.hasArray()) {
+ calculateChunkedSums(data.array(), data.arrayOffset() + data.position(), data.remaining(),
+ checksums.array(), checksums.arrayOffset() + checksums.position());
+ return;
+ }
+
+ data.mark();
+ checksums.mark();
+ try {
+ byte[] buf = new byte[bytesPerChecksum];
+ while (data.remaining() > 0) {
+ int n = Math.min(data.remaining(), bytesPerChecksum);
+ data.get(buf, 0, n);
+ summer.reset();
+ summer.update(buf, 0, n);
+ checksums.putInt((int)summer.getValue());
+ }
+ } finally {
+ data.reset();
+ checksums.reset();
+ }
+ }
+
+ /**
+ * Implementation of chunked calculation specifically on byte arrays. This
+ * is to avoid the copy when dealing with ByteBuffers that have array backing.
+ */
+ private void calculateChunkedSums(
+ byte[] data, int dataOffset, int dataLength,
+ byte[] sums, int sumsOffset) {
+
+ int remaining = dataLength;
+ while (remaining > 0) {
+ int n = Math.min(remaining, bytesPerChecksum);
+ summer.reset();
+ summer.update(data, dataOffset, n);
+ dataOffset += n;
+ remaining -= n;
+ long calculated = summer.getValue();
+ sums[sumsOffset++] = (byte) (calculated >> 24);
+ sums[sumsOffset++] = (byte) (calculated >> 16);
+ sums[sumsOffset++] = (byte) (calculated >> 8);
+ sums[sumsOffset++] = (byte) (calculated);
+ }
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof DataChecksum)) {
+ return false;
+ }
+ DataChecksum o = (DataChecksum)other;
+ return o.bytesPerChecksum == this.bytesPerChecksum &&
+ o.type == this.type;
+ }
+
+ @Override
+ public int hashCode() {
+ return (this.type.id + 31) * this.bytesPerChecksum;
+ }
+
+ @Override
+ public String toString() {
+ return "DataChecksum(type=" + type +
+ ", chunkSize=" + bytesPerChecksum + ")";
+ }
+
+ /**
+ * This just provides a dummy implimentation for Checksum class
+ * This is used when there is no checksum available or required for
+ * data
+ */
+ static class ChecksumNull implements Checksum {
+
+ public ChecksumNull() {}
+
+ //Dummy interface
+ @Override
+ public long getValue() { return 0; }
+ @Override
+ public void reset() {}
+ @Override
+ public void update(byte[] b, int off, int len) {}
+ @Override
+ public void update(int b) {}
+ };
+}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java Mon Oct 22 20:43:16 2012
@@ -59,22 +59,38 @@ public class PureJavaCrc32 implements Ch
@Override
public void update(byte[] b, int off, int len) {
int localCrc = crc;
+
while(len > 7) {
- int c0 = b[off++] ^ localCrc;
- int c1 = b[off++] ^ (localCrc >>>= 8);
- int c2 = b[off++] ^ (localCrc >>>= 8);
- int c3 = b[off++] ^ (localCrc >>>= 8);
- localCrc = (T8_7[c0 & 0xff] ^ T8_6[c1 & 0xff])
- ^ (T8_5[c2 & 0xff] ^ T8_4[c3 & 0xff]);
+ final int c0 =(b[off+0] ^ localCrc) & 0xff;
+ final int c1 =(b[off+1] ^ (localCrc >>>= 8)) & 0xff;
+ final int c2 =(b[off+2] ^ (localCrc >>>= 8)) & 0xff;
+ final int c3 =(b[off+3] ^ (localCrc >>>= 8)) & 0xff;
+ localCrc = (T[T8_7_start + c0] ^ T[T8_6_start + c1])
+ ^ (T[T8_5_start + c2] ^ T[T8_4_start + c3]);
+
+ final int c4 = b[off+4] & 0xff;
+ final int c5 = b[off+5] & 0xff;
+ final int c6 = b[off+6] & 0xff;
+ final int c7 = b[off+7] & 0xff;
- localCrc ^= (T8_3[b[off++] & 0xff] ^ T8_2[b[off++] & 0xff])
- ^ (T8_1[b[off++] & 0xff] ^ T8_0[b[off++] & 0xff]);
+ localCrc ^= (T[T8_3_start + c4] ^ T[T8_2_start + c5])
+ ^ (T[T8_1_start + c6] ^ T[T8_0_start + c7]);
+ off += 8;
len -= 8;
}
- while(len > 0) {
- localCrc = (localCrc >>> 8) ^ T8_0[(localCrc ^ b[off++]) & 0xff];
- len--;
+
+ /* loop unroll - duff's device style */
+ switch(len) {
+ case 7: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 6: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 5: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 4: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 3: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 2: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 1: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ default:
+ /* nothing */
}
// Publish crc out to object
@@ -83,14 +99,24 @@ public class PureJavaCrc32 implements Ch
@Override
final public void update(int b) {
- crc = (crc >>> 8) ^ T8_0[(crc ^ b) & 0xff];
+ crc = (crc >>> 8) ^ T[T8_0_start + ((crc ^ b) & 0xff)];
}
/*
* CRC-32 lookup tables generated by the polynomial 0xEDB88320.
* See also TestPureJavaCrc32.Table.
*/
- private static final int[] T8_0 = new int[] {
+ private static final int T8_0_start = 0*256;
+ private static final int T8_1_start = 1*256;
+ private static final int T8_2_start = 2*256;
+ private static final int T8_3_start = 3*256;
+ private static final int T8_4_start = 4*256;
+ private static final int T8_5_start = 5*256;
+ private static final int T8_6_start = 6*256;
+ private static final int T8_7_start = 7*256;
+
+ private static final int[] T = new int[] {
+ /* T8_0 */
0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA,
0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3,
0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988,
@@ -154,9 +180,8 @@ public class PureJavaCrc32 implements Ch
0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6,
0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF,
0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94,
- 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D
- };
- private static final int[] T8_1 = new int[] {
+ 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D,
+ /* T8_1 */
0x00000000, 0x191B3141, 0x32366282, 0x2B2D53C3,
0x646CC504, 0x7D77F445, 0x565AA786, 0x4F4196C7,
0xC8D98A08, 0xD1C2BB49, 0xFAEFE88A, 0xE3F4D9CB,
@@ -220,9 +245,8 @@ public class PureJavaCrc32 implements Ch
0x14BCE1BD, 0x0DA7D0FC, 0x268A833F, 0x3F91B27E,
0x70D024B9, 0x69CB15F8, 0x42E6463B, 0x5BFD777A,
0xDC656BB5, 0xC57E5AF4, 0xEE530937, 0xF7483876,
- 0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72
- };
- private static final int[] T8_2 = new int[] {
+ 0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72,
+ /* T8_2 */
0x00000000, 0x01C26A37, 0x0384D46E, 0x0246BE59,
0x0709A8DC, 0x06CBC2EB, 0x048D7CB2, 0x054F1685,
0x0E1351B8, 0x0FD13B8F, 0x0D9785D6, 0x0C55EFE1,
@@ -286,9 +310,8 @@ public class PureJavaCrc32 implements Ch
0xB5C473D0, 0xB40619E7, 0xB640A7BE, 0xB782CD89,
0xB2CDDB0C, 0xB30FB13B, 0xB1490F62, 0xB08B6555,
0xBBD72268, 0xBA15485F, 0xB853F606, 0xB9919C31,
- 0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED
- };
- private static final int[] T8_3 = new int[] {
+ 0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED,
+ /* T8_3 */
0x00000000, 0xB8BC6765, 0xAA09C88B, 0x12B5AFEE,
0x8F629757, 0x37DEF032, 0x256B5FDC, 0x9DD738B9,
0xC5B428EF, 0x7D084F8A, 0x6FBDE064, 0xD7018701,
@@ -352,9 +375,8 @@ public class PureJavaCrc32 implements Ch
0x866616A7, 0x3EDA71C2, 0x2C6FDE2C, 0x94D3B949,
0x090481F0, 0xB1B8E695, 0xA30D497B, 0x1BB12E1E,
0x43D23E48, 0xFB6E592D, 0xE9DBF6C3, 0x516791A6,
- 0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1
- };
- private static final int[] T8_4 = new int[] {
+ 0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1,
+ /* T8_4 */
0x00000000, 0x3D6029B0, 0x7AC05360, 0x47A07AD0,
0xF580A6C0, 0xC8E08F70, 0x8F40F5A0, 0xB220DC10,
0x30704BC1, 0x0D106271, 0x4AB018A1, 0x77D03111,
@@ -418,9 +440,8 @@ public class PureJavaCrc32 implements Ch
0x4834505D, 0x755479ED, 0x32F4033D, 0x0F942A8D,
0xBDB4F69D, 0x80D4DF2D, 0xC774A5FD, 0xFA148C4D,
0x78441B9C, 0x4524322C, 0x028448FC, 0x3FE4614C,
- 0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C
- };
- private static final int[] T8_5 = new int[] {
+ 0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C,
+ /* T8_5 */
0x00000000, 0xCB5CD3A5, 0x4DC8A10B, 0x869472AE,
0x9B914216, 0x50CD91B3, 0xD659E31D, 0x1D0530B8,
0xEC53826D, 0x270F51C8, 0xA19B2366, 0x6AC7F0C3,
@@ -484,9 +505,8 @@ public class PureJavaCrc32 implements Ch
0x15921919, 0xDECECABC, 0x585AB812, 0x93066BB7,
0x8E035B0F, 0x455F88AA, 0xC3CBFA04, 0x089729A1,
0xF9C19B74, 0x329D48D1, 0xB4093A7F, 0x7F55E9DA,
- 0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC
- };
- private static final int[] T8_6 = new int[] {
+ 0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC,
+ /* T8_6 */
0x00000000, 0xA6770BB4, 0x979F1129, 0x31E81A9D,
0xF44F2413, 0x52382FA7, 0x63D0353A, 0xC5A73E8E,
0x33EF4E67, 0x959845D3, 0xA4705F4E, 0x020754FA,
@@ -550,9 +570,8 @@ public class PureJavaCrc32 implements Ch
0x647E3AD9, 0xC209316D, 0xF3E12BF0, 0x55962044,
0x90311ECA, 0x3646157E, 0x07AE0FE3, 0xA1D90457,
0x579174BE, 0xF1E67F0A, 0xC00E6597, 0x66796E23,
- 0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30
- };
- private static final int[] T8_7 = new int[] {
+ 0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30,
+ /* T8_7 */
0x00000000, 0xCCAA009E, 0x4225077D, 0x8E8F07E3,
0x844A0EFA, 0x48E00E64, 0xC66F0987, 0x0AC50919,
0xD3E51BB5, 0x1F4F1B2B, 0x91C01CC8, 0x5D6A1C56,
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java Mon Oct 22 20:43:16 2012
@@ -56,22 +56,38 @@ public class PureJavaCrc32C implements C
@Override
public void update(byte[] b, int off, int len) {
int localCrc = crc;
+
while(len > 7) {
- int c0 = b[off++] ^ localCrc;
- int c1 = b[off++] ^ (localCrc >>>= 8);
- int c2 = b[off++] ^ (localCrc >>>= 8);
- int c3 = b[off++] ^ (localCrc >>>= 8);
- localCrc = (T8_7[c0 & 0xff] ^ T8_6[c1 & 0xff])
- ^ (T8_5[c2 & 0xff] ^ T8_4[c3 & 0xff]);
+ final int c0 =(b[off+0] ^ localCrc) & 0xff;
+ final int c1 =(b[off+1] ^ (localCrc >>>= 8)) & 0xff;
+ final int c2 =(b[off+2] ^ (localCrc >>>= 8)) & 0xff;
+ final int c3 =(b[off+3] ^ (localCrc >>>= 8)) & 0xff;
+ localCrc = (T[T8_7_start + c0] ^ T[T8_6_start + c1])
+ ^ (T[T8_5_start + c2] ^ T[T8_4_start + c3]);
+
+ final int c4 = b[off+4] & 0xff;
+ final int c5 = b[off+5] & 0xff;
+ final int c6 = b[off+6] & 0xff;
+ final int c7 = b[off+7] & 0xff;
- localCrc ^= (T8_3[b[off++] & 0xff] ^ T8_2[b[off++] & 0xff])
- ^ (T8_1[b[off++] & 0xff] ^ T8_0[b[off++] & 0xff]);
+ localCrc ^= (T[T8_3_start + c4] ^ T[T8_2_start + c5])
+ ^ (T[T8_1_start + c6] ^ T[T8_0_start + c7]);
+ off += 8;
len -= 8;
}
- while(len > 0) {
- localCrc = (localCrc >>> 8) ^ T8_0[(localCrc ^ b[off++]) & 0xff];
- len--;
+
+ /* loop unroll - duff's device style */
+ switch(len) {
+ case 7: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 6: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 5: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 4: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 3: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 2: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 1: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ default:
+ /* nothing */
}
// Publish crc out to object
@@ -80,14 +96,24 @@ public class PureJavaCrc32C implements C
@Override
final public void update(int b) {
- crc = (crc >>> 8) ^ T8_0[(crc ^ b) & 0xff];
+ crc = (crc >>> 8) ^ T[T8_0_start + ((crc ^ b) & 0xff)];
}
// CRC polynomial tables generated by:
// java -cp build/test/classes/:build/classes/ \
// org.apache.hadoop.util.TestPureJavaCrc32\$Table 82F63B78
- static final int[] T8_0 = new int[] {
+ private static final int T8_0_start = 0*256;
+ private static final int T8_1_start = 1*256;
+ private static final int T8_2_start = 2*256;
+ private static final int T8_3_start = 3*256;
+ private static final int T8_4_start = 4*256;
+ private static final int T8_5_start = 5*256;
+ private static final int T8_6_start = 6*256;
+ private static final int T8_7_start = 7*256;
+
+ private static final int[] T = new int[] {
+ /* T8_0 */
0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4,
0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB,
0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B,
@@ -151,9 +177,8 @@ public class PureJavaCrc32C implements C
0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81,
0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E,
0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E,
- 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351
- };
- static final int[] T8_1 = new int[] {
+ 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351,
+ /* T8_1 */
0x00000000, 0x13A29877, 0x274530EE, 0x34E7A899,
0x4E8A61DC, 0x5D28F9AB, 0x69CF5132, 0x7A6DC945,
0x9D14C3B8, 0x8EB65BCF, 0xBA51F356, 0xA9F36B21,
@@ -217,9 +242,8 @@ public class PureJavaCrc32C implements C
0x449A2E7E, 0x5738B609, 0x63DF1E90, 0x707D86E7,
0x0A104FA2, 0x19B2D7D5, 0x2D557F4C, 0x3EF7E73B,
0xD98EEDC6, 0xCA2C75B1, 0xFECBDD28, 0xED69455F,
- 0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483
- };
- static final int[] T8_2 = new int[] {
+ 0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483,
+ /* T8_2 */
0x00000000, 0xA541927E, 0x4F6F520D, 0xEA2EC073,
0x9EDEA41A, 0x3B9F3664, 0xD1B1F617, 0x74F06469,
0x38513EC5, 0x9D10ACBB, 0x773E6CC8, 0xD27FFEB6,
@@ -283,9 +307,8 @@ public class PureJavaCrc32C implements C
0xDDA47104, 0x78E5E37A, 0x92CB2309, 0x378AB177,
0x437AD51E, 0xE63B4760, 0x0C158713, 0xA954156D,
0xE5F54FC1, 0x40B4DDBF, 0xAA9A1DCC, 0x0FDB8FB2,
- 0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8
- };
- static final int[] T8_3 = new int[] {
+ 0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8,
+ /* T8_3 */
0x00000000, 0xDD45AAB8, 0xBF672381, 0x62228939,
0x7B2231F3, 0xA6679B4B, 0xC4451272, 0x1900B8CA,
0xF64463E6, 0x2B01C95E, 0x49234067, 0x9466EADF,
@@ -349,9 +372,8 @@ public class PureJavaCrc32C implements C
0xC747336E, 0x1A0299D6, 0x782010EF, 0xA565BA57,
0xBC65029D, 0x6120A825, 0x0302211C, 0xDE478BA4,
0x31035088, 0xEC46FA30, 0x8E647309, 0x5321D9B1,
- 0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842
- };
- static final int[] T8_4 = new int[] {
+ 0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842,
+ /* T8_4 */
0x00000000, 0x38116FAC, 0x7022DF58, 0x4833B0F4,
0xE045BEB0, 0xD854D11C, 0x906761E8, 0xA8760E44,
0xC5670B91, 0xFD76643D, 0xB545D4C9, 0x8D54BB65,
@@ -415,9 +437,8 @@ public class PureJavaCrc32C implements C
0xCD796B76, 0xF56804DA, 0xBD5BB42E, 0x854ADB82,
0x2D3CD5C6, 0x152DBA6A, 0x5D1E0A9E, 0x650F6532,
0x081E60E7, 0x300F0F4B, 0x783CBFBF, 0x402DD013,
- 0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3
- };
- static final int[] T8_5 = new int[] {
+ 0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3,
+ /* T8_5 */
0x00000000, 0xEF306B19, 0xDB8CA0C3, 0x34BCCBDA,
0xB2F53777, 0x5DC55C6E, 0x697997B4, 0x8649FCAD,
0x6006181F, 0x8F367306, 0xBB8AB8DC, 0x54BAD3C5,
@@ -481,9 +502,8 @@ public class PureJavaCrc32C implements C
0x57F4CA8E, 0xB8C4A197, 0x8C786A4D, 0x63480154,
0xE501FDF9, 0x0A3196E0, 0x3E8D5D3A, 0xD1BD3623,
0x37F2D291, 0xD8C2B988, 0xEC7E7252, 0x034E194B,
- 0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C
- };
- static final int[] T8_6 = new int[] {
+ 0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C,
+ /* T8_6 */
0x00000000, 0x68032CC8, 0xD0065990, 0xB8057558,
0xA5E0C5D1, 0xCDE3E919, 0x75E69C41, 0x1DE5B089,
0x4E2DFD53, 0x262ED19B, 0x9E2BA4C3, 0xF628880B,
@@ -547,9 +567,8 @@ public class PureJavaCrc32C implements C
0x2ED97095, 0x46DA5C5D, 0xFEDF2905, 0x96DC05CD,
0x8B39B544, 0xE33A998C, 0x5B3FECD4, 0x333CC01C,
0x60F48DC6, 0x08F7A10E, 0xB0F2D456, 0xD8F1F89E,
- 0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F
- };
- static final int[] T8_7 = new int[] {
+ 0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F,
+ /* T8_7 */
0x00000000, 0x493C7D27, 0x9278FA4E, 0xDB448769,
0x211D826D, 0x6821FF4A, 0xB3657823, 0xFA590504,
0x423B04DA, 0x0B0779FD, 0xD043FE94, 0x997F83B3,
@@ -613,6 +632,6 @@ public class PureJavaCrc32C implements C
0xA777317B, 0xEE4B4C5C, 0x350FCB35, 0x7C33B612,
0x866AB316, 0xCF56CE31, 0x14124958, 0x5D2E347F,
0xE54C35A1, 0xAC704886, 0x7734CFEF, 0x3E08B2C8,
- 0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5
+ 0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5
};
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java Mon Oct 22 20:43:16 2012
@@ -126,6 +126,10 @@ public class RunJar {
int firstArg = 0;
String fileName = args[firstArg++];
File file = new File(fileName);
+ if (!file.exists() || !file.isFile()) {
+ System.err.println("Not a valid JAR: " + file.getCanonicalPath());
+ System.exit(-1);
+ }
String mainClassName = null;
JarFile jarFile;
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java Mon Oct 22 20:43:16 2012
@@ -608,7 +608,8 @@ public class StringUtils {
" build = " + VersionInfo.getUrl() + " -r "
+ VersionInfo.getRevision()
+ "; compiled by '" + VersionInfo.getUser()
- + "' on " + VersionInfo.getDate()}
+ + "' on " + VersionInfo.getDate(),
+ " java = " + System.getProperty("java.version") }
)
);
Propchange: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1397381-1401062
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java Mon Oct 22 20:43:16 2012
@@ -472,7 +472,9 @@ public class TestFileUtil {
setupDirs();
long du = FileUtil.getDU(TEST_DIR);
- //Only two files (in partitioned) have 4 bytes each
- Assert.assertEquals(du, 8);
+ // Only two files (in partitioned). Each has 3 characters + system-specific
+ // line separator.
+ long expected = 2 * (3 + System.getProperty("line.separator").length());
+ Assert.assertEquals(expected, du);
}
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java Mon Oct 22 20:43:16 2012
@@ -730,6 +730,55 @@ public class TestCodec {
}
}
+ @Test
+ public void testGzipLongOverflow() throws IOException {
+ LOG.info("testGzipLongOverflow");
+
+ // Don't use native libs for this test.
+ Configuration conf = new Configuration();
+ conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, false);
+ assertFalse("ZlibFactory is using native libs against request",
+ ZlibFactory.isNativeZlibLoaded(conf));
+
+ // Ensure that the CodecPool has a BuiltInZlibInflater in it.
+ Decompressor zlibDecompressor = ZlibFactory.getZlibDecompressor(conf);
+ assertNotNull("zlibDecompressor is null!", zlibDecompressor);
+ assertTrue("ZlibFactory returned unexpected inflator",
+ zlibDecompressor instanceof BuiltInZlibInflater);
+ CodecPool.returnDecompressor(zlibDecompressor);
+
+ // Now create a GZip text file.
+ String tmpDir = System.getProperty("test.build.data", "/tmp/");
+ Path f = new Path(new Path(tmpDir), "testGzipLongOverflow.bin.gz");
+ BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
+ new GZIPOutputStream(new FileOutputStream(f.toString()))));
+
+ final int NBUF = 1024 * 4 + 1;
+ final char[] buf = new char[1024 * 1024];
+ for (int i = 0; i < buf.length; i++) buf[i] = '\0';
+ for (int i = 0; i < NBUF; i++) {
+ bw.write(buf);
+ }
+ bw.close();
+
+ // Now read it back, using the CodecPool to establish the
+ // decompressor to use.
+ CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
+ CompressionCodec codec = ccf.getCodec(f);
+ Decompressor decompressor = CodecPool.getDecompressor(codec);
+ FileSystem fs = FileSystem.getLocal(conf);
+ InputStream is = fs.open(f);
+ is = codec.createInputStream(is, decompressor);
+ BufferedReader br = new BufferedReader(new InputStreamReader(is));
+ for (int j = 0; j < NBUF; j++) {
+ int n = br.read(buf);
+ assertEquals("got wrong read length!", n, buf.length);
+ for (int i = 0; i < buf.length; i++)
+ assertEquals("got wrong byte!", buf[i], '\0');
+ }
+ br.close();
+ }
+
public void testGzipCodecWrite(boolean useNative) throws IOException {
// Create a gzipped file using a compressor from the CodecPool,
// and try to read it back via the regular GZIPInputStream.
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestVLong.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestVLong.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestVLong.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestVLong.java Mon Oct 22 20:43:16 2012
@@ -141,7 +141,7 @@ public class TestVLong extends TestCase
int shift = rng.nextInt(Long.SIZE) + 1;
long mask = (1L << shift) - 1;
long a = ((long) rng.nextInt()) << 32;
- long b = ((long) rng.nextInt()) & 0xffffffff;
+ long b = ((long) rng.nextInt()) & 0xffffffffL;
data[i] = (a + b) & mask;
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java Mon Oct 22 20:43:16 2012
@@ -19,10 +19,7 @@
package org.apache.hadoop.ipc;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
+import static org.junit.Assert.*;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
@@ -44,6 +41,7 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.Client.ConnectionId;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.security.SaslInputStream;
import org.apache.hadoop.security.SaslRpcClient;
@@ -449,23 +447,100 @@ public class TestSaslRPC {
}
System.out.println("Test is successful.");
}
-
+
+ // insecure -> insecure
+ @Test
+ public void testInsecureClientInsecureServer() throws Exception {
+ assertEquals(AuthenticationMethod.SIMPLE,
+ getAuthMethod(false, false, false));
+ }
+
+ @Test
+ public void testInsecureClientInsecureServerWithToken() throws Exception {
+ assertEquals(AuthenticationMethod.TOKEN,
+ getAuthMethod(false, false, true));
+ }
+
+ // insecure -> secure
+ @Test
+ public void testInsecureClientSecureServer() throws Exception {
+ RemoteException e = null;
+ try {
+ getAuthMethod(false, true, false);
+ } catch (RemoteException re) {
+ e = re;
+ }
+ assertNotNull(e);
+ assertEquals(AccessControlException.class.getName(), e.getClassName());
+ }
+
+ @Test
+ public void testInsecureClientSecureServerWithToken() throws Exception {
+ assertEquals(AuthenticationMethod.TOKEN,
+ getAuthMethod(false, true, true));
+ }
+
+ // secure -> secure
+ @Test
+ public void testSecureClientSecureServer() throws Exception {
+ /* Should be this when multiple secure auths are supported and we can
+ * dummy one out:
+ * assertEquals(AuthenticationMethod.SECURE_AUTH_METHOD,
+ * getAuthMethod(true, true, false));
+ */
+ try {
+ getAuthMethod(true, true, false);
+ } catch (IOException ioe) {
+ // can't actually test kerberos w/o kerberos...
+ String expectedError = "Failed to specify server's Kerberos principal";
+ String actualError = ioe.getMessage();
+ assertTrue("["+actualError+"] doesn't start with ["+expectedError+"]",
+ actualError.contains(expectedError));
+ }
+ }
+
@Test
- public void testDigestAuthMethodSecureServer() throws Exception {
- checkDigestAuthMethod(true);
+ public void testSecureClientSecureServerWithToken() throws Exception {
+ assertEquals(AuthenticationMethod.TOKEN,
+ getAuthMethod(true, true, true));
}
+ // secure -> insecure
@Test
- public void testDigestAuthMethodInsecureServer() throws Exception {
- checkDigestAuthMethod(false);
+ public void testSecureClientInsecureServerWithToken() throws Exception {
+ assertEquals(AuthenticationMethod.TOKEN,
+ getAuthMethod(true, false, true));
}
- private void checkDigestAuthMethod(boolean secureServer) throws Exception {
+ @Test
+ public void testSecureClientInsecureServer() throws Exception {
+ /* Should be this when multiple secure auths are supported and we can
+ * dummy one out:
+ * assertEquals(AuthenticationMethod.SIMPLE
+ * getAuthMethod(true, false, false));
+ */
+ try {
+ getAuthMethod(true, false, false);
+ } catch (IOException ioe) {
+ // can't actually test kerberos w/o kerberos...
+ String expectedError = "Failed to specify server's Kerberos principal";
+ String actualError = ioe.getMessage();
+ assertTrue("["+actualError+"] doesn't start with ["+expectedError+"]",
+ actualError.contains(expectedError));
+ }
+ }
+
+
+ private AuthenticationMethod getAuthMethod(final boolean isSecureClient,
+ final boolean isSecureServer,
+ final boolean useToken
+
+ ) throws Exception {
TestTokenSecretManager sm = new TestTokenSecretManager();
Server server = new RPC.Builder(conf).setProtocol(TestSaslProtocol.class)
.setInstance(new TestSaslImpl()).setBindAddress(ADDRESS).setPort(0)
.setNumHandlers(5).setVerbose(true).setSecretManager(sm).build();
- if (secureServer) {
+ if (isSecureServer) {
server.enableSecurity();
} else {
server.disableSecurity();
@@ -474,30 +549,36 @@ public class TestSaslRPC {
final UserGroupInformation current = UserGroupInformation.getCurrentUser();
final InetSocketAddress addr = NetUtils.getConnectAddress(server);
- TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
- .getUserName()));
- Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(tokenId,
- sm);
- SecurityUtil.setTokenService(token, addr);
- current.addToken(token);
+ if (useToken) {
+ TestTokenIdentifier tokenId = new TestTokenIdentifier(
+ new Text(current.getUserName()));
+ Token<TestTokenIdentifier> token =
+ new Token<TestTokenIdentifier>(tokenId, sm);
+ SecurityUtil.setTokenService(token, addr);
+ current.addToken(token);
+ }
- current.doAs(new PrivilegedExceptionAction<Object>() {
- @Override
- public Object run() throws IOException {
- TestSaslProtocol proxy = null;
- try {
- proxy = (TestSaslProtocol) RPC.getProxy(TestSaslProtocol.class,
- TestSaslProtocol.versionID, addr, conf);
- Assert.assertEquals(AuthenticationMethod.TOKEN, proxy.getAuthMethod());
- } finally {
- if (proxy != null) {
- RPC.stopProxy(proxy);
+ conf.set(HADOOP_SECURITY_AUTHENTICATION, isSecureClient ? "kerberos" : "simple");
+ UserGroupInformation.setConfiguration(conf);
+ try {
+ return current.doAs(new PrivilegedExceptionAction<AuthenticationMethod>() {
+ @Override
+ public AuthenticationMethod run() throws IOException {
+ TestSaslProtocol proxy = null;
+ try {
+ proxy = (TestSaslProtocol) RPC.getProxy(TestSaslProtocol.class,
+ TestSaslProtocol.versionID, addr, conf);
+ return proxy.getAuthMethod();
+ } finally {
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
}
}
- return null;
- }
- });
- server.stop();
+ });
+ } finally {
+ server.stop();
+ }
}
public static void main(String[] args) throws Exception {
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java Mon Oct 22 20:43:16 2012
@@ -78,5 +78,29 @@ public class TestJMXJsonServlet extends
"/jmx?get=java.lang:type=Memory::"));
LOG.info("/jmx RESULT: "+result);
assertReFind("\"ERROR\"", result);
+
+ // test to get JSONP result
+ result = readOutput(new URL(baseUrl, "/jmx?qry=java.lang:type=Memory&callback=mycallback1"));
+ LOG.info("/jmx?qry=java.lang:type=Memory&callback=mycallback RESULT: "+result);
+ assertReFind("^mycallback1\\(\\{", result);
+ assertReFind("\\}\\);$", result);
+
+ // negative test to get an attribute of a mbean as JSONP
+ result = readOutput(new URL(baseUrl,
+ "/jmx?get=java.lang:type=Memory::&callback=mycallback2"));
+ LOG.info("/jmx RESULT: "+result);
+ assertReFind("^mycallback2\\(\\{", result);
+ assertReFind("\"ERROR\"", result);
+ assertReFind("\\}\\);$", result);
+
+ // test to get an attribute of a mbean as JSONP
+ result = readOutput(new URL(baseUrl,
+ "/jmx?get=java.lang:type=Memory::HeapMemoryUsage&callback=mycallback3"));
+ LOG.info("/jmx RESULT: "+result);
+ assertReFind("^mycallback3\\(\\{", result);
+ assertReFind("\"name\"\\s*:\\s*\"java.lang:type=Memory\"", result);
+ assertReFind("\"committed\"\\s*:", result);
+ assertReFind("\\}\\);$", result);
+
}
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java Mon Oct 22 20:43:16 2012
@@ -18,9 +18,7 @@
package org.apache.hadoop.metrics2.util;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.*;
import java.io.IOException;
import java.util.Arrays;
@@ -28,7 +26,6 @@ import java.util.Collections;
import java.util.Map;
import java.util.Random;
-import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Before;
import org.junit.Test;
@@ -54,18 +51,22 @@ public class TestSampleQuantiles {
// Counts start off zero
assertEquals(estimator.getCount(), 0);
assertEquals(estimator.getSampleCount(), 0);
- try {
- estimator.snapshot();
- fail("Expected IOException from empty window");
- } catch (IOException e) {
- GenericTestUtils.assertExceptionContains("No samples", e);
- }
+
+ // Snapshot should be null if there are no entries.
+ assertNull(estimator.snapshot());
// Count increment correctly by 1
estimator.insert(1337);
assertEquals(estimator.getCount(), 1);
estimator.snapshot();
assertEquals(estimator.getSampleCount(), 1);
+
+ assertEquals(
+ "50.00 %ile +/- 5.00%: 1337\n" +
+ "75.00 %ile +/- 2.50%: 1337\n" +
+ "90.00 %ile +/- 1.00%: 1337\n" +
+ "95.00 %ile +/- 0.50%: 1337\n" +
+ "99.00 %ile +/- 0.10%: 1337", estimator.toString());
}
/**
@@ -80,12 +81,7 @@ public class TestSampleQuantiles {
estimator.clear();
assertEquals(estimator.getCount(), 0);
assertEquals(estimator.getSampleCount(), 0);
- try {
- estimator.snapshot();
- fail("Expected IOException for an empty window.");
- } catch (IOException e) {
- GenericTestUtils.assertExceptionContains("No samples", e);
- }
+ assertNull(estimator.snapshot());
}
/**
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java?rev=1401071&r1=1401070&r2=1401071&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java Mon Oct 22 20:43:16 2012
@@ -142,16 +142,14 @@ public class TestPureJavaCrc32 {
for (int j = 0; j < tables.length; j++) {
final int[] t = tables[j];
final StringBuilder b = new StringBuilder();
- b.append(String.format(" static final int[] " + nameformat
- + " = new int[] {", j));
+ b.append(String.format(" /* "+ nameformat +" */", j));
for (int i = 0; i < t.length;) {
b.append("\n ");
for(int k = 0; k < 4; k++) {
b.append(String.format("0x%08X, ", t[i++]));
}
}
- b.setCharAt(b.length() - 2, '\n');
- s[j] = b.toString() + " };\n";
+ s[j] = b.toString();
}
return s;
}
@@ -159,10 +157,23 @@ public class TestPureJavaCrc32 {
@Override
public String toString() {
final StringBuilder b = new StringBuilder();
- for(String s : toStrings(String.format("T%d_",
- Integer.numberOfTrailingZeros(tables[0].length)) + "%d")) {
+
+ final String tableFormat = String.format("T%d_",
+ Integer.numberOfTrailingZeros(tables[0].length)) + "%d";
+ final String startFormat = " private static final int "+tableFormat+"_start = %d*256;";
+
+ for (int j = 0; j < tables.length; j++) {
+ b.append(String.format(startFormat, j, j));
+ b.append("\n");
+ }
+
+ b.append(" private static final int[] T = new int[] {");
+ for(String s : toStrings(tableFormat)) {
+ b.append("\n");
b.append(s);
}
+ b.setCharAt(b.length() - 2, '\n');
+ b.append(" };\n");
return b.toString();
}