You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2012/03/08 23:55:50 UTC
svn commit: r1298641 [3/4] - in /hbase/trunk/src:
main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/fs/
main/java/org/apache/hadoop/hbase/io/hfile/
main/java/org/apache/hadoop/hbase/mapreduce/
main/java/org/apache/hadoop/hbase/region...
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Thu Mar 8 22:55:49 2012
@@ -70,6 +70,7 @@ import org.apache.hadoop.hbase.regionser
import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.hbase.util.CollectionBackedScanner;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -157,6 +158,10 @@ public class Store extends SchemaConfigu
private final Compression.Algorithm compactionCompression;
private HFileDataBlockEncoder dataBlockEncoder;
+ /** Checksum configuration */
+ private ChecksumType checksumType;
+ private int bytesPerChecksum;
+
// Comparing KeyValues
final KeyValue.KVComparator comparator;
@@ -246,6 +251,35 @@ public class Store extends SchemaConfigu
"hbase.hstore.close.check.interval", 10*1000*1000 /* 10 MB */);
}
this.storefiles = sortAndClone(loadStoreFiles());
+
+ // Initialize checksum type from name. The names are CRC32, CRC32C, etc.
+ this.checksumType = getChecksumType(conf);
+ // initilize bytes per checksum
+ this.bytesPerChecksum = getBytesPerChecksum(conf);
+ }
+
+ /**
+ * Returns the configured bytesPerChecksum value.
+ * @param conf The configuration
+ * @return The bytesPerChecksum that is set in the configuration
+ */
+ public static int getBytesPerChecksum(Configuration conf) {
+ return conf.getInt(HConstants.BYTES_PER_CHECKSUM,
+ HFile.DEFAULT_BYTES_PER_CHECKSUM);
+ }
+
+ /**
+ * Returns the configured checksum algorithm.
+ * @param conf The configuration
+ * @return The checksum algorithm that is set in the configuration
+ */
+ public static ChecksumType getChecksumType(Configuration conf) {
+ String checksumName = conf.get(HConstants.CHECKSUM_TYPE_NAME);
+ if (checksumName == null) {
+ return HFile.DEFAULT_CHECKSUM_TYPE;
+ } else {
+ return ChecksumType.nameToType(checksumName);
+ }
}
public HColumnDescriptor getFamily() {
@@ -799,6 +833,8 @@ public class Store extends SchemaConfigu
.withComparator(comparator)
.withBloomType(family.getBloomFilterType())
.withMaxKeyCount(maxKeyCount)
+ .withChecksumType(checksumType)
+ .withBytesPerChecksum(bytesPerChecksum)
.build();
// The store file writer's path does not include the CF name, so we need
// to configure the HFile writer directly.
@@ -2192,8 +2228,8 @@ public class Store extends SchemaConfigu
public static final long FIXED_OVERHEAD =
ClassSize.align(SchemaConfigured.SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE +
- + (19 * ClassSize.REFERENCE) + (6 * Bytes.SIZEOF_LONG)
- + (5 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN);
+ + (20 * ClassSize.REFERENCE) + (6 * Bytes.SIZEOF_LONG)
+ + (6 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN);
public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD
+ ClassSize.OBJECT + ClassSize.REENTRANT_LOCK
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Thu Mar 8 22:55:49 2012
@@ -60,6 +60,7 @@ import org.apache.hadoop.hbase.regionser
import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
+import org.apache.hadoop.hbase.util.ChecksumType;
import org.apache.hadoop.hbase.util.BloomFilter;
import org.apache.hadoop.hbase.util.BloomFilterFactory;
import org.apache.hadoop.hbase.util.BloomFilterWriter;
@@ -699,6 +700,8 @@ public class StoreFile extends SchemaCon
private long maxKeyCount = 0;
private Path dir;
private Path filePath;
+ private ChecksumType checksumType = HFile.DEFAULT_CHECKSUM_TYPE;
+ private int bytesPerChecksum = HFile.DEFAULT_BYTES_PER_CHECKSUM;
public WriterBuilder(Configuration conf, CacheConfig cacheConf,
FileSystem fs, int blockSize) {
@@ -766,6 +769,24 @@ public class StoreFile extends SchemaCon
}
/**
+ * @param checksumType the type of checksum
+ * @return this (for chained invocation)
+ */
+ public WriterBuilder withChecksumType(ChecksumType checksumType) {
+ this.checksumType = checksumType;
+ return this;
+ }
+
+ /**
+ * @param bytesPerChecksum the number of bytes per checksum chunk
+ * @return this (for chained invocation)
+ */
+ public WriterBuilder withBytesPerChecksum(int bytesPerChecksum) {
+ this.bytesPerChecksum = bytesPerChecksum;
+ return this;
+ }
+
+ /**
* Create a store file writer. Client is responsible for closing file when
* done. If metadata, add BEFORE closing using
* {@link Writer#appendMetadata}.
@@ -798,7 +819,8 @@ public class StoreFile extends SchemaCon
comparator = KeyValue.COMPARATOR;
}
return new Writer(fs, filePath, blockSize, compressAlgo, dataBlockEncoder,
- conf, cacheConf, comparator, bloomType, maxKeyCount);
+ conf, cacheConf, comparator, bloomType, maxKeyCount, checksumType,
+ bytesPerChecksum);
}
}
@@ -896,6 +918,12 @@ public class StoreFile extends SchemaCon
protected HFileDataBlockEncoder dataBlockEncoder;
+ /** Checksum type */
+ protected ChecksumType checksumType;
+
+ /** Bytes per Checksum */
+ protected int bytesPerChecksum;
+
TimeRangeTracker timeRangeTracker = new TimeRangeTracker();
/* isTimeRangeTrackerSet keeps track if the timeRange has already been set
* When flushing a memstore, we set TimeRange and use this variable to
@@ -918,13 +946,16 @@ public class StoreFile extends SchemaCon
* @param bloomType bloom filter setting
* @param maxKeys the expected maximum number of keys to be added. Was used
* for Bloom filter size in {@link HFile} format version 1.
+ * @param checksumType the checksum type
+ * @param bytesPerChecksum the number of bytes per checksum value
* @throws IOException problem writing to FS
*/
private Writer(FileSystem fs, Path path, int blocksize,
Compression.Algorithm compress,
HFileDataBlockEncoder dataBlockEncoder, final Configuration conf,
CacheConfig cacheConf,
- final KVComparator comparator, BloomType bloomType, long maxKeys)
+ final KVComparator comparator, BloomType bloomType, long maxKeys,
+ final ChecksumType checksumType, final int bytesPerChecksum)
throws IOException {
this.dataBlockEncoder = dataBlockEncoder != null ?
dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
@@ -934,6 +965,8 @@ public class StoreFile extends SchemaCon
.withCompression(compress)
.withDataBlockEncoder(dataBlockEncoder)
.withComparator(comparator.getRawComparator())
+ .withChecksumType(checksumType)
+ .withBytesPerChecksum(bytesPerChecksum)
.create();
this.kvComparator = comparator;
@@ -964,6 +997,8 @@ public class StoreFile extends SchemaCon
LOG.info("Delete Family Bloom filter type for " + path + ": "
+ deleteFamilyBloomFilterWriter.getClass().getSimpleName());
}
+ this.checksumType = checksumType;
+ this.bytesPerChecksum = bytesPerChecksum;
}
/**
@@ -1660,7 +1695,7 @@ public class StoreFile extends SchemaCon
}
public int getHFileVersion() {
- return reader.getTrailer().getVersion();
+ return reader.getTrailer().getMajorVersion();
}
HFile.Reader getHFileReader() {
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java Thu Mar 8 22:55:49 2012
@@ -248,6 +248,12 @@ public class RegionServerMetrics impleme
public final MetricsTimeVaryingLong regionSplitFailureCount =
new MetricsTimeVaryingLong("regionSplitFailureCount", registry);
+ /**
+ * Number of times checksum verification failed.
+ */
+ public final MetricsLongValue checksumFailuresCount =
+ new MetricsLongValue("checksumFailuresCount", registry);
+
public RegionServerMetrics() {
MetricsContext context = MetricsUtil.getContext("hbase");
metricsRecord = MetricsUtil.createRecord(context, "regionserver");
@@ -346,6 +352,8 @@ public class RegionServerMetrics impleme
// HFile metrics, positional reads
ops = HFile.getPreadOps();
if (ops != 0) this.fsPreadLatency.inc(ops, HFile.getPreadTimeMs());
+ this.checksumFailuresCount.set(HFile.getChecksumFailuresCount());
+
/* NOTE: removed HFile write latency. 2 reasons:
* 1) Mixing HLog latencies are far higher priority since they're
* on-demand and HFile is used in background (compact/flush)
@@ -366,6 +374,7 @@ public class RegionServerMetrics impleme
this.slowHLogAppendCount.pushMetric(this.metricsRecord);
this.regionSplitSuccessCount.pushMetric(this.metricsRecord);
this.regionSplitFailureCount.pushMetric(this.metricsRecord);
+ this.checksumFailuresCount.pushMetric(this.metricsRecord);
}
this.metricsRecord.update();
}
Added: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/ChecksumFactory.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/ChecksumFactory.java?rev=1298641&view=auto
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/ChecksumFactory.java (added)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/ChecksumFactory.java Thu Mar 8 22:55:49 2012
@@ -0,0 +1,99 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.lang.ClassNotFoundException;
+import java.util.zip.Checksum;
+import java.lang.reflect.Constructor;
+
+/**
+ * Utility class that is used to generate a Checksum object.
+ * The Checksum implementation is pluggable and an application
+ * can specify their own class that implements their own
+ * Checksum algorithm.
+ */
+public class ChecksumFactory {
+
+ static private final Class<?>[] EMPTY_ARRAY = new Class[]{};
+
+ /**
+ * Create a new instance of a Checksum object.
+ * @return The newly created Checksum object
+ */
+ static public Checksum newInstance(String className) throws IOException {
+ try {
+ Class<?> clazz = getClassByName(className);
+ return (Checksum)newInstance(clazz);
+ } catch (ClassNotFoundException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Returns a Constructor that can be used to create a Checksum object.
+ * @return The Constructor that can be used to create a
+ * new Checksum object.
+ * @param theClass classname for which an constructor is created
+ * @return a new Constructor object
+ */
+ static public Constructor<?> newConstructor(String className)
+ throws IOException {
+ try {
+ Class<?> clazz = getClassByName(className);
+ Constructor<?> ctor = clazz.getDeclaredConstructor(EMPTY_ARRAY);
+ ctor.setAccessible(true);
+ return ctor;
+ } catch (ClassNotFoundException e) {
+ throw new IOException(e);
+ } catch (java.lang.NoSuchMethodException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /** Create an object for the given class and initialize it from conf
+ *
+ * @param theClass class of which an object is created
+ * @return a new object
+ */
+ static private <T> T newInstance(Class<T> theClass) {
+ T result;
+ try {
+ Constructor<T> ctor = theClass.getDeclaredConstructor(EMPTY_ARRAY);
+ ctor.setAccessible(true);
+ result = ctor.newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ return result;
+ }
+
+ /**
+ * Load a class by name.
+ * @param name the class name.
+ * @return the class object.
+ * @throws ClassNotFoundException if the class is not found.
+ */
+ static private Class<?> getClassByName(String name)
+ throws ClassNotFoundException {
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ return Class.forName(name, true, classLoader);
+ }
+}
Added: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java?rev=1298641&view=auto
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java (added)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java Thu Mar 8 22:55:49 2012
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.util.zip.Checksum;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.util.ChecksumFactory;
+
+/**
+ * Checksum types. The Checksum type is a one byte number
+ * that stores a representation of the checksum algorithm
+ * used to encode a hfile. The ordinal of these cannot
+ * change or else you risk breaking all existing HFiles out there.
+ */
+public enum ChecksumType {
+
+ NULL((byte)0) {
+ @Override
+ public String getName() {
+ return "NULL";
+ }
+ @Override
+ public void initialize() {
+ // do nothing
+ }
+ @Override
+ public Checksum getChecksumObject() throws IOException {
+ return null; // checksums not used
+ }
+ },
+
+ CRC32((byte)1) {
+ private volatile Constructor<?> ctor;
+
+ @Override
+ public String getName() {
+ return "CRC32";
+ }
+
+ @Override
+ public void initialize() {
+ final String PURECRC32 = "org.apache.hadoop.util.PureJavaCrc32";
+ final String JDKCRC = "java.util.zip.CRC32";
+ LOG = LogFactory.getLog(ChecksumType.class);
+
+ // check if hadoop library is available
+ try {
+ ctor = ChecksumFactory.newConstructor(PURECRC32);
+ LOG.info("Checksum using " + PURECRC32);
+ } catch (Exception e) {
+ LOG.info(PURECRC32 + " not available.");
+ }
+ try {
+ // The default checksum class name is java.util.zip.CRC32.
+ // This is available on all JVMs.
+ if (ctor == null) {
+ ctor = ChecksumFactory.newConstructor(JDKCRC);
+ LOG.info("Checksum can use " + JDKCRC);
+ }
+ } catch (Exception e) {
+ LOG.warn(JDKCRC + " not available. ", e);
+ }
+ }
+
+ @Override
+ public Checksum getChecksumObject() throws IOException {
+ if (ctor == null) {
+ throw new IOException("Bad constructor for " + getName());
+ }
+ try {
+ return (Checksum)ctor.newInstance();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+ },
+
+ CRC32C((byte)2) {
+ private transient Constructor<?> ctor;
+
+ @Override
+ public String getName() {
+ return "CRC32C";
+ }
+
+ @Override
+ public void initialize() {
+ final String PURECRC32C = "org.apache.hadoop.util.PureJavaCrc32C";
+ LOG = LogFactory.getLog(ChecksumType.class);
+ try {
+ ctor = ChecksumFactory.newConstructor(PURECRC32C);
+ LOG.info("Checksum can use " + PURECRC32C);
+ } catch (Exception e) {
+ LOG.info(PURECRC32C + " not available. ");
+ }
+ }
+
+ @Override
+ public Checksum getChecksumObject() throws IOException {
+ if (ctor == null) {
+ throw new IOException("Bad constructor for " + getName());
+ }
+ try {
+ return (Checksum)ctor.newInstance();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+ };
+
+ private final byte code;
+ protected Log LOG;
+
+ /** initializes the relevant checksum class object */
+ abstract void initialize();
+
+ /** returns the name of this checksum type */
+ public abstract String getName();
+
+ private ChecksumType(final byte c) {
+ this.code = c;
+ initialize();
+ }
+
+ /** returns a object that can be used to generate/validate checksums */
+ public abstract Checksum getChecksumObject() throws IOException;
+
+ public byte getCode() {
+ return this.code;
+ }
+
+ /**
+ * Cannot rely on enum ordinals . They change if item is removed or moved.
+ * Do our own codes.
+ * @param b
+ * @return Type associated with passed code.
+ */
+ public static ChecksumType codeToType(final byte b) {
+ for (ChecksumType t : ChecksumType.values()) {
+ if (t.getCode() == b) {
+ return t;
+ }
+ }
+ throw new RuntimeException("Unknown checksum type code " + b);
+ }
+
+ /**
+ * Map a checksum name to a specific type.
+ * Do our own names.
+ * @param b
+ * @return Type associated with passed code.
+ */
+ public static ChecksumType nameToType(final String name) {
+ for (ChecksumType t : ChecksumType.values()) {
+ if (t.getName().equals(name)) {
+ return t;
+ }
+ }
+ throw new RuntimeException("Unknown checksum type name " + name);
+ }
+}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java Thu Mar 8 22:55:49 2012
@@ -109,7 +109,7 @@ public class CompoundBloomFilter extends
ByteBuffer bloomBuf = bloomBlock.getBufferReadOnly();
result = ByteBloomFilter.contains(key, keyOffset, keyLength,
- bloomBuf.array(), bloomBuf.arrayOffset() + HFileBlock.HEADER_SIZE,
+ bloomBuf.array(), bloomBuf.arrayOffset() + bloomBlock.headerSize(),
bloomBlock.getUncompressedSizeWithoutHeader(), hash, hashCount);
}
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Thu Mar 8 22:55:49 2012
@@ -58,7 +58,9 @@ import org.apache.hadoop.hbase.client.Pu
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.ChecksumUtil;
import org.apache.hadoop.hbase.io.hfile.Compression;
import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
import org.apache.hadoop.hbase.master.HMaster;
@@ -189,6 +191,9 @@ public class HBaseTestingUtility {
public HBaseTestingUtility(Configuration conf) {
this.conf = conf;
+
+ // a hbase checksum verification failure will cause unit tests to fail
+ ChecksumUtil.generateExceptionForChecksumFailureForTest(true);
}
/**
@@ -1437,7 +1442,7 @@ public class HBaseTestingUtility {
}
public FileSystem getTestFileSystem() throws IOException {
- return FileSystem.get(conf);
+ return HFileSystem.get(conf);
}
/**
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java Thu Mar 8 22:55:49 2012
@@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hbase.MultithreadedTestUtil;
import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.util.ChecksumType;
import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
public class CacheTestUtils {
@@ -323,7 +324,9 @@ public class CacheTestUtils {
HFileBlock generated = new HFileBlock(BlockType.DATA,
onDiskSizeWithoutHeader, uncompressedSizeWithoutHeader,
prevBlockOffset, cachedBuffer, HFileBlock.DONT_FILL_HEADER,
- blockSize, includesMemstoreTS);
+ blockSize, includesMemstoreTS, HFileBlock.MINOR_VERSION_NO_CHECKSUM,
+ 0, ChecksumType.NULL.getCode(),
+ onDiskSizeWithoutHeader + HFileBlock.HEADER_SIZE);
String strKey;
/* No conflicting keys */
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java Thu Mar 8 22:55:49 2012
@@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.HColumnDe
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MediumTests;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.StoreFile;
@@ -49,6 +50,7 @@ import org.apache.hadoop.hbase.regionser
import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
import org.apache.hadoop.hbase.util.BloomFilterFactory;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.junit.After;
import org.junit.Before;
@@ -88,6 +90,8 @@ public class TestCacheOnWrite {
private static final int INDEX_BLOCK_SIZE = 512;
private static final int BLOOM_BLOCK_SIZE = 4096;
private static final BloomType BLOOM_TYPE = StoreFile.BloomType.ROWCOL;
+ private static final ChecksumType CKTYPE = ChecksumType.CRC32;
+ private static final int CKBYTES = 512;
/** The number of valid key types possible in a store file */
private static final int NUM_VALID_KEY_TYPES =
@@ -192,7 +196,7 @@ public class TestCacheOnWrite {
conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
cowType.shouldBeCached(BlockType.BLOOM_CHUNK));
cowType.modifyConf(conf);
- fs = FileSystem.get(conf);
+ fs = HFileSystem.get(conf);
cacheConf = new CacheConfig(conf);
blockCache = cacheConf.getBlockCache();
}
@@ -292,6 +296,8 @@ public class TestCacheOnWrite {
.withComparator(KeyValue.COMPARATOR)
.withBloomType(BLOOM_TYPE)
.withMaxKeyCount(NUM_KV)
+ .withChecksumType(CKTYPE)
+ .withBytesPerChecksum(CKBYTES)
.build();
final int rowLen = 32;
Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java?rev=1298641&view=auto
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java (added)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java Thu Mar 8 22:55:49 2012
@@ -0,0 +1,290 @@
+/*
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import static org.junit.Assert.*;
+
+import java.io.ByteArrayOutputStream;
+import java.io.ByteArrayInputStream;
+import java.io.DataOutputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.zip.Checksum;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.io.compress.Compressor;
+
+import static org.apache.hadoop.hbase.io.hfile.Compression.Algorithm.*;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(MediumTests.class)
+public class TestChecksum {
+ // change this value to activate more logs
+ private static final boolean detailedLogging = true;
+ private static final boolean[] BOOLEAN_VALUES = new boolean[] { false, true };
+
+ private static final Log LOG = LogFactory.getLog(TestHFileBlock.class);
+
+ static final Compression.Algorithm[] COMPRESSION_ALGORITHMS = {
+ NONE, GZ };
+
+ static final int[] BYTES_PER_CHECKSUM = {
+ 50, 500, 688, 16*1024, (16*1024+980), 64 * 1024};
+
+ private static final HBaseTestingUtility TEST_UTIL =
+ new HBaseTestingUtility();
+ private FileSystem fs;
+ private HFileSystem hfs;
+
+ @Before
+ public void setUp() throws Exception {
+ fs = HFileSystem.get(TEST_UTIL.getConfiguration());
+ hfs = (HFileSystem)fs;
+ }
+
+ /**
+ * Introduce checksum failures and check that we can still read
+ * the data
+ */
+ @Test
+ public void testChecksumCorruption() throws IOException {
+ for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
+ for (boolean pread : new boolean[] { false, true }) {
+ LOG.info("testChecksumCorruption: Compression algorithm: " + algo +
+ ", pread=" + pread);
+ Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
+ + algo);
+ FSDataOutputStream os = fs.create(path);
+ HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
+ true, HFile.DEFAULT_CHECKSUM_TYPE,
+ HFile.DEFAULT_BYTES_PER_CHECKSUM);
+ long totalSize = 0;
+ for (int blockId = 0; blockId < 2; ++blockId) {
+ DataOutputStream dos = hbw.startWriting(BlockType.DATA);
+ for (int i = 0; i < 1234; ++i)
+ dos.writeInt(i);
+ hbw.writeHeaderAndData(os);
+ totalSize += hbw.getOnDiskSizeWithHeader();
+ }
+ os.close();
+
+ // Use hbase checksums.
+ assertEquals(true, hfs.useHBaseChecksum());
+ assertEquals(true, hfs.getNoChecksumFs() != hfs.getBackingFs());
+
+ // Do a read that purposely introduces checksum verification failures.
+ FSDataInputStream is = fs.open(path);
+ HFileBlock.FSReader hbr = new FSReaderV2Test(is, algo,
+ totalSize, HFile.MAX_FORMAT_VERSION, fs, path);
+ HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
+ b.sanityCheck();
+ assertEquals(4936, b.getUncompressedSizeWithoutHeader());
+ assertEquals(algo == GZ ? 2173 : 4936,
+ b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
+ // read data back from the hfile, exclude header and checksum
+ ByteBuffer bb = b.getBufferWithoutHeader(); // read back data
+ DataInputStream in = new DataInputStream(
+ new ByteArrayInputStream(
+ bb.array(), bb.arrayOffset(), bb.limit()));
+
+ // assert that we encountered hbase checksum verification failures
+ // but still used hdfs checksums and read data successfully.
+ assertEquals(1, HFile.getChecksumFailuresCount());
+ validateData(in);
+
+ // A single instance of hbase checksum failure causes the reader to
+ // switch off hbase checksum verification for the next 100 read
+ // requests. Verify that this is correct.
+ for (int i = 0; i <
+ HFileBlock.CHECKSUM_VERIFICATION_NUM_IO_THRESHOLD + 1; i++) {
+ b = hbr.readBlockData(0, -1, -1, pread);
+ assertEquals(0, HFile.getChecksumFailuresCount());
+ }
+ // The next read should have hbase checksum verification reanabled,
+ // we verify this by assertng that there was a hbase-checksum failure.
+ b = hbr.readBlockData(0, -1, -1, pread);
+ assertEquals(1, HFile.getChecksumFailuresCount());
+
+ // Since the above encountered a checksum failure, we switch
+ // back to not checking hbase checksums.
+ b = hbr.readBlockData(0, -1, -1, pread);
+ assertEquals(0, HFile.getChecksumFailuresCount());
+ is.close();
+
+ // Now, use a completely new reader. Switch off hbase checksums in
+ // the configuration. In this case, we should not detect
+ // any retries within hbase.
+ HFileSystem newfs = new HFileSystem(TEST_UTIL.getConfiguration(), false);
+ assertEquals(false, newfs.useHBaseChecksum());
+ is = newfs.open(path);
+ hbr = new FSReaderV2Test(is, algo,
+ totalSize, HFile.MAX_FORMAT_VERSION, newfs, path);
+ b = hbr.readBlockData(0, -1, -1, pread);
+ is.close();
+ b.sanityCheck();
+ assertEquals(4936, b.getUncompressedSizeWithoutHeader());
+ assertEquals(algo == GZ ? 2173 : 4936,
+ b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
+ // read data back from the hfile, exclude header and checksum
+ bb = b.getBufferWithoutHeader(); // read back data
+ in = new DataInputStream(new ByteArrayInputStream(
+ bb.array(), bb.arrayOffset(), bb.limit()));
+
+ // assert that we did not encounter hbase checksum verification failures
+ // but still used hdfs checksums and read data successfully.
+ assertEquals(0, HFile.getChecksumFailuresCount());
+ validateData(in);
+ }
+ }
+ }
+
+ /**
+ * Test different values of bytesPerChecksum
+ */
+ @Test
+ public void testChecksumChunks() throws IOException {
+ Compression.Algorithm algo = NONE;
+ for (boolean pread : new boolean[] { false, true }) {
+ for (int bytesPerChecksum : BYTES_PER_CHECKSUM) {
+ Path path = new Path(TEST_UTIL.getDataTestDir(), "checksumChunk_" +
+ algo + bytesPerChecksum);
+ FSDataOutputStream os = fs.create(path);
+ HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
+ true, HFile.DEFAULT_CHECKSUM_TYPE, bytesPerChecksum);
+
+ // write one block. The block has data
+ // that is at least 6 times more than the checksum chunk size
+ long dataSize = 0;
+ DataOutputStream dos = hbw.startWriting(BlockType.DATA);
+ for (; dataSize < 6 * bytesPerChecksum;) {
+ for (int i = 0; i < 1234; ++i) {
+ dos.writeInt(i);
+ dataSize += 4;
+ }
+ }
+ hbw.writeHeaderAndData(os);
+ long totalSize = hbw.getOnDiskSizeWithHeader();
+ os.close();
+
+ long expectedChunks = ChecksumUtil.numChunks(
+ dataSize + HFileBlock.HEADER_SIZE,
+ bytesPerChecksum);
+ LOG.info("testChecksumChunks: pread=" + pread +
+ ", bytesPerChecksum=" + bytesPerChecksum +
+ ", fileSize=" + totalSize +
+ ", dataSize=" + dataSize +
+ ", expectedChunks=" + expectedChunks);
+
+ // Verify hbase checksums.
+ assertEquals(true, hfs.useHBaseChecksum());
+ assertEquals(true, hfs.getNoChecksumFs() != hfs.getBackingFs());
+
+ // Read data back from file.
+ FSDataInputStream is = fs.open(path);
+ FSDataInputStream nochecksum = hfs.getNoChecksumFs().open(path);
+ HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, nochecksum,
+ algo, totalSize, HFile.MAX_FORMAT_VERSION, hfs, path);
+ HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
+ is.close();
+ b.sanityCheck();
+ assertEquals(dataSize, b.getUncompressedSizeWithoutHeader());
+
+ // verify that we have the expected number of checksum chunks
+ assertEquals(totalSize, HFileBlock.HEADER_SIZE + dataSize +
+ expectedChunks * HFileBlock.CHECKSUM_SIZE);
+
+ // assert that we did not encounter hbase checksum verification failures
+ assertEquals(0, HFile.getChecksumFailuresCount());
+ }
+ }
+ }
+
+ /**
+ * Test to ensure that these is at least one valid checksum implementation
+ */
+ @Test
+ public void testChecksumAlgorithm() throws IOException {
+ ChecksumType type = ChecksumType.CRC32;
+ assertEquals(ChecksumType.nameToType(type.getName()), type);
+ assertEquals(ChecksumType.valueOf(type.toString()), type);
+ }
+
+ private void validateData(DataInputStream in) throws IOException {
+ // validate data
+ for (int i = 0; i < 1234; i++) {
+ int val = in.readInt();
+ if (val != i) {
+ String msg = "testChecksumCorruption: data mismatch at index " +
+ i + " expected " + i + " found " + val;
+ LOG.warn(msg);
+ assertEquals(i, val);
+ }
+ }
+ }
+
+ @org.junit.Rule
+ public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+ new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+
+ /**
+ * A class that introduces hbase-checksum failures while
+ * reading data from hfiles. This should trigger the hdfs level
+ * checksum validations.
+ */
+ static private class FSReaderV2Test extends HFileBlock.FSReaderV2 {
+
+ FSReaderV2Test(FSDataInputStream istream, Algorithm algo,
+ long fileSize, int minorVersion, FileSystem fs,
+ Path path) throws IOException {
+ super(istream, istream, algo, fileSize, minorVersion,
+ (HFileSystem)fs, path);
+ }
+
+ @Override
+ protected boolean validateBlockChecksum(HFileBlock block,
+ byte[] data, int hdrSize) throws IOException {
+ return false; // checksum validation failure
+ }
+ }
+}
+
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java Thu Mar 8 22:55:49 2012
@@ -52,7 +52,7 @@ public class TestFixedFileTrailer {
private static final Log LOG = LogFactory.getLog(TestFixedFileTrailer.class);
/** The number of used fields by version. Indexed by version minus one. */
- private static final int[] NUM_FIELDS_BY_VERSION = new int[] { 8, 13 };
+ private static final int[] NUM_FIELDS_BY_VERSION = new int[] { 9, 14 };
private HBaseTestingUtility util = new HBaseTestingUtility();
private FileSystem fs;
@@ -83,7 +83,8 @@ public class TestFixedFileTrailer {
@Test
public void testTrailer() throws IOException {
- FixedFileTrailer t = new FixedFileTrailer(version);
+ FixedFileTrailer t = new FixedFileTrailer(version,
+ HFileBlock.MINOR_VERSION_NO_CHECKSUM);
t.setDataIndexCount(3);
t.setEntryCount(((long) Integer.MAX_VALUE) + 1);
@@ -121,7 +122,8 @@ public class TestFixedFileTrailer {
// Finished writing, trying to read.
{
DataInputStream dis = new DataInputStream(bais);
- FixedFileTrailer t2 = new FixedFileTrailer(version);
+ FixedFileTrailer t2 = new FixedFileTrailer(version,
+ HFileBlock.MINOR_VERSION_NO_CHECKSUM);
t2.deserialize(dis);
assertEquals(-1, bais.read()); // Ensure we have read everything.
checkLoadedTrailer(version, t, t2);
@@ -191,7 +193,7 @@ public class TestFixedFileTrailer {
private void checkLoadedTrailer(int version, FixedFileTrailer expected,
FixedFileTrailer loaded) throws IOException {
- assertEquals(version, loaded.getVersion());
+ assertEquals(version, loaded.getMajorVersion());
assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount());
assertEquals(Math.min(expected.getEntryCount(),
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java Thu Mar 8 22:55:49 2012
@@ -48,9 +48,11 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.MediumTests;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.io.DoubleOutputStream;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.io.compress.CompressionOutputStream;
@@ -102,16 +104,16 @@ public class TestHFileBlock {
@Before
public void setUp() throws IOException {
- fs = FileSystem.get(TEST_UTIL.getConfiguration());
+ fs = HFileSystem.get(TEST_UTIL.getConfiguration());
}
- public void writeTestBlockContents(DataOutputStream dos) throws IOException {
+ static void writeTestBlockContents(DataOutputStream dos) throws IOException {
// This compresses really well.
for (int i = 0; i < 1000; ++i)
dos.writeInt(i / 100);
}
- private int writeTestKeyValues(OutputStream dos, int seed)
+ static int writeTestKeyValues(OutputStream dos, int seed, boolean includesMemstoreTS)
throws IOException {
List<KeyValue> keyValues = new ArrayList<KeyValue>();
Random randomizer = new Random(42l + seed); // just any fixed number
@@ -191,22 +193,24 @@ public class TestHFileBlock {
return baos.toByteArray();
}
- private byte[] createTestV2Block(Compression.Algorithm algo)
- throws IOException {
+ static HFileBlock.Writer createTestV2Block(Compression.Algorithm algo,
+ boolean includesMemstoreTS) throws IOException {
final BlockType blockType = BlockType.DATA;
HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
- includesMemstoreTS);
+ includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+ HFile.DEFAULT_BYTES_PER_CHECKSUM);
DataOutputStream dos = hbw.startWriting(blockType);
writeTestBlockContents(dos);
- byte[] headerAndData = hbw.getHeaderAndData();
+ byte[] headerAndData = hbw.getHeaderAndDataForTest();
assertEquals(1000 * 4, hbw.getUncompressedSizeWithoutHeader());
hbw.releaseCompressor();
- return headerAndData;
+ return hbw;
}
public String createTestBlockStr(Compression.Algorithm algo,
int correctLength) throws IOException {
- byte[] testV2Block = createTestV2Block(algo);
+ HFileBlock.Writer hbw = createTestV2Block(algo, includesMemstoreTS);
+ byte[] testV2Block = hbw.getHeaderAndDataForTest();
int osOffset = HFileBlock.HEADER_SIZE + 9;
if (testV2Block.length == correctLength) {
// Force-set the "OS" field of the gzip header to 3 (Unix) to avoid
@@ -221,14 +225,16 @@ public class TestHFileBlock {
@Test
public void testNoCompression() throws IOException {
- assertEquals(4000 + HFileBlock.HEADER_SIZE, createTestV2Block(NONE).length);
+ assertEquals(4000, createTestV2Block(NONE, includesMemstoreTS).
+ getBlockForCaching().getUncompressedSizeWithoutHeader());
}
@Test
public void testGzipCompression() throws IOException {
final String correctTestBlockStr =
- "DATABLK*\\x00\\x00\\x00:\\x00\\x00\\x0F\\xA0\\xFF\\xFF\\xFF\\xFF"
+ "DATABLK*\\x00\\x00\\x00>\\x00\\x00\\x0F\\xA0\\xFF\\xFF\\xFF\\xFF"
+ "\\xFF\\xFF\\xFF\\xFF"
+ + "\\x01\\x00\\x00@\\x00\\x00\\x00\\x00["
// gzip-compressed block: http://www.gzip.org/zlib/rfc-gzip.html
+ "\\x1F\\x8B" // gzip magic signature
+ "\\x08" // Compression method: 8 = "deflate"
@@ -240,8 +246,9 @@ public class TestHFileBlock {
+ "\\x03"
+ "\\xED\\xC3\\xC1\\x11\\x00 \\x08\\xC00DD\\xDD\\x7Fa"
+ "\\xD6\\xE8\\xA3\\xB9K\\x84`\\x96Q\\xD3\\xA8\\xDB\\xA8e\\xD4c"
- + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\s\\xA0\\x0F\\x00\\x00";
- final int correctGzipBlockLength = 82;
+ + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\s\\xA0\\x0F\\x00\\x00"
+ + "\\xAB\\x85g\\x91"; // 4 byte checksum
+ final int correctGzipBlockLength = 95;
assertEquals(correctTestBlockStr, createTestBlockStr(GZ,
correctGzipBlockLength));
}
@@ -285,11 +292,14 @@ public class TestHFileBlock {
public void testReaderV2() throws IOException {
for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
for (boolean pread : new boolean[] { false, true }) {
+ LOG.info("testReaderV2: Compression algorithm: " + algo +
+ ", pread=" + pread);
Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
+ algo);
FSDataOutputStream os = fs.create(path);
HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
- includesMemstoreTS);
+ includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+ HFile.DEFAULT_BYTES_PER_CHECKSUM);
long totalSize = 0;
for (int blockId = 0; blockId < 2; ++blockId) {
DataOutputStream dos = hbw.startWriting(BlockType.DATA);
@@ -305,16 +315,19 @@ public class TestHFileBlock {
totalSize);
HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
is.close();
+ assertEquals(0, HFile.getChecksumFailuresCount());
b.sanityCheck();
assertEquals(4936, b.getUncompressedSizeWithoutHeader());
- assertEquals(algo == GZ ? 2173 : 4936, b.getOnDiskSizeWithoutHeader());
+ assertEquals(algo == GZ ? 2173 : 4936,
+ b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
String blockStr = b.toString();
if (algo == GZ) {
is = fs.open(path);
hbr = new HFileBlock.FSReaderV2(is, algo, totalSize);
- b = hbr.readBlockData(0, 2173 + HFileBlock.HEADER_SIZE, -1, pread);
+ b = hbr.readBlockData(0, 2173 + HFileBlock.HEADER_SIZE +
+ b.totalChecksumBytes(), -1, pread);
assertEquals(blockStr, b.toString());
int wrongCompressedSize = 2172;
try {
@@ -351,13 +364,15 @@ public class TestHFileBlock {
HFileDataBlockEncoder dataBlockEncoder =
new HFileDataBlockEncoderImpl(encoding);
HFileBlock.Writer hbw = new HFileBlock.Writer(algo, dataBlockEncoder,
- includesMemstoreTS);
+ includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+ HFile.DEFAULT_BYTES_PER_CHECKSUM);
long totalSize = 0;
final List<Integer> encodedSizes = new ArrayList<Integer>();
final List<ByteBuffer> encodedBlocks = new ArrayList<ByteBuffer>();
for (int blockId = 0; blockId < numBlocks; ++blockId) {
- writeEncodedBlock(encoding, hbw, encodedSizes, encodedBlocks,
- blockId);
+ DataOutputStream dos = hbw.startWriting(BlockType.DATA);
+ writeEncodedBlock(encoding, dos, encodedSizes, encodedBlocks,
+ blockId, includesMemstoreTS);
hbw.writeHeaderAndData(os);
totalSize += hbw.getOnDiskSizeWithHeader();
@@ -374,6 +389,7 @@ public class TestHFileBlock {
int pos = 0;
for (int blockId = 0; blockId < numBlocks; ++blockId) {
b = hbr.readBlockData(pos, -1, -1, pread);
+ assertEquals(0, HFile.getChecksumFailuresCount());
b.sanityCheck();
pos += b.getOnDiskSizeWithHeader();
@@ -401,16 +417,16 @@ public class TestHFileBlock {
}
}
- private void writeEncodedBlock(DataBlockEncoding encoding,
- HFileBlock.Writer hbw, final List<Integer> encodedSizes,
- final List<ByteBuffer> encodedBlocks, int blockId) throws IOException {
- DataOutputStream dos = hbw.startWriting(BlockType.DATA);
+ static void writeEncodedBlock(DataBlockEncoding encoding,
+ DataOutputStream dos, final List<Integer> encodedSizes,
+ final List<ByteBuffer> encodedBlocks, int blockId,
+ boolean includesMemstoreTS) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DoubleOutputStream doubleOutputStream =
new DoubleOutputStream(dos, baos);
final int rawBlockSize = writeTestKeyValues(doubleOutputStream,
- blockId);
+ blockId, includesMemstoreTS);
ByteBuffer rawBuf = ByteBuffer.wrap(baos.toByteArray());
rawBuf.rewind();
@@ -434,7 +450,7 @@ public class TestHFileBlock {
encodedBlocks.add(encodedBuf);
}
- private void assertBuffersEqual(ByteBuffer expectedBuffer,
+ static void assertBuffersEqual(ByteBuffer expectedBuffer,
ByteBuffer actualBuffer, Compression.Algorithm compression,
DataBlockEncoding encoding, boolean pread) {
if (!actualBuffer.equals(expectedBuffer)) {
@@ -471,7 +487,9 @@ public class TestHFileBlock {
for (boolean pread : BOOLEAN_VALUES) {
for (boolean cacheOnWrite : BOOLEAN_VALUES) {
Random rand = defaultRandom();
- LOG.info("Compression algorithm: " + algo + ", pread=" + pread);
+ LOG.info("testPreviousOffset:Compression algorithm: " + algo +
+ ", pread=" + pread +
+ ", cacheOnWrite=" + cacheOnWrite);
Path path = new Path(TEST_UTIL.getDataTestDir(), "prev_offset");
List<Long> expectedOffsets = new ArrayList<Long>();
List<Long> expectedPrevOffsets = new ArrayList<Long>();
@@ -522,17 +540,23 @@ public class TestHFileBlock {
b2.getUncompressedSizeWithoutHeader());
assertEquals(b.getPrevBlockOffset(), b2.getPrevBlockOffset());
assertEquals(curOffset, b2.getOffset());
+ assertEquals(b.getBytesPerChecksum(), b2.getBytesPerChecksum());
+ assertEquals(b.getOnDiskDataSizeWithHeader(),
+ b2.getOnDiskDataSizeWithHeader());
+ assertEquals(0, HFile.getChecksumFailuresCount());
curOffset += b.getOnDiskSizeWithHeader();
if (cacheOnWrite) {
// In the cache-on-write mode we store uncompressed bytes so we
// can compare them to what was read by the block reader.
-
+ // b's buffer has header + data + checksum while
+ // expectedContents have header + data only
ByteBuffer bufRead = b.getBufferWithHeader();
ByteBuffer bufExpected = expectedContents.get(i);
boolean bytesAreCorrect = Bytes.compareTo(bufRead.array(),
- bufRead.arrayOffset(), bufRead.limit(),
+ bufRead.arrayOffset(),
+ bufRead.limit() - b.totalChecksumBytes(),
bufExpected.array(), bufExpected.arrayOffset(),
bufExpected.limit()) == 0;
String wrongBytesMsg = "";
@@ -541,15 +565,26 @@ public class TestHFileBlock {
// Optimization: only construct an error message in case we
// will need it.
wrongBytesMsg = "Expected bytes in block #" + i + " (algo="
- + algo + ", pread=" + pread + "):\n";
+ + algo + ", pread=" + pread
+ + ", cacheOnWrite=" + cacheOnWrite + "):\n";
wrongBytesMsg += Bytes.toStringBinary(bufExpected.array(),
bufExpected.arrayOffset(), Math.min(32,
bufExpected.limit()))
+ ", actual:\n"
+ Bytes.toStringBinary(bufRead.array(),
bufRead.arrayOffset(), Math.min(32, bufRead.limit()));
+ if (detailedLogging) {
+ LOG.warn("expected header" +
+ HFileBlock.toStringHeader(bufExpected) +
+ "\nfound header" +
+ HFileBlock.toStringHeader(bufRead));
+ LOG.warn("bufread offset " + bufRead.arrayOffset() +
+ " limit " + bufRead.limit() +
+ " expected offset " + bufExpected.arrayOffset() +
+ " limit " + bufExpected.limit());
+ LOG.warn(wrongBytesMsg);
+ }
}
-
assertTrue(wrongBytesMsg, bytesAreCorrect);
}
}
@@ -672,10 +707,12 @@ public class TestHFileBlock {
boolean cacheOnWrite = expectedContents != null;
FSDataOutputStream os = fs.create(path);
HFileBlock.Writer hbw = new HFileBlock.Writer(compressAlgo, null,
- includesMemstoreTS);
+ includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+ HFile.DEFAULT_BYTES_PER_CHECKSUM);
Map<BlockType, Long> prevOffsetByType = new HashMap<BlockType, Long>();
long totalSize = 0;
for (int i = 0; i < NUM_TEST_BLOCKS; ++i) {
+ long pos = os.getPos();
int blockTypeOrdinal = rand.nextInt(BlockType.values().length);
if (blockTypeOrdinal == BlockType.ENCODED_DATA.ordinal()) {
blockTypeOrdinal = BlockType.DATA.ordinal();
@@ -706,9 +743,9 @@ public class TestHFileBlock {
expectedContents.add(hbw.getUncompressedBufferWithHeader());
if (detailedLogging) {
- LOG.info("Writing block #" + i + " of type " + bt
+ LOG.info("Written block #" + i + " of type " + bt
+ ", uncompressed size " + hbw.getUncompressedSizeWithoutHeader()
- + " at offset " + os.getPos());
+ + " at offset " + pos);
}
}
os.close();
@@ -730,7 +767,9 @@ public class TestHFileBlock {
byte[] byteArr = new byte[HFileBlock.HEADER_SIZE + size];
ByteBuffer buf = ByteBuffer.wrap(byteArr, 0, size);
HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf,
- HFileBlock.FILL_HEADER, -1, includesMemstoreTS);
+ HFileBlock.FILL_HEADER, -1, includesMemstoreTS,
+ HFileBlock.MINOR_VERSION_NO_CHECKSUM, 0, ChecksumType.NULL.getCode(),
+ 0);
long byteBufferExpectedSize =
ClassSize.align(ClassSize.estimateBase(buf.getClass(), true)
+ HFileBlock.HEADER_SIZE + size);
Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java?rev=1298641&view=auto
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java (added)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java Thu Mar 8 22:55:49 2012
@@ -0,0 +1,806 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import static org.junit.Assert.*;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.DoubleOutputStream;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ClassSize;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.io.compress.CompressionOutputStream;
+import org.apache.hadoop.io.compress.Compressor;
+import org.apache.hadoop.hbase.io.hfile.HFileBlock.BlockWritable;
+import org.apache.hadoop.hbase.util.ChecksumType;
+import org.apache.hadoop.hbase.util.Pair;
+import com.google.common.base.Preconditions;
+
+import static org.apache.hadoop.hbase.io.hfile.Compression.Algorithm.*;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * This class has unit tests to prove that older versions of
+ * HFiles (without checksums) are compatible with current readers.
+ */
+@Category(MediumTests.class)
+@RunWith(Parameterized.class)
+public class TestHFileBlockCompatibility {
+ // change this value to activate more logs
+ private static final boolean[] BOOLEAN_VALUES = new boolean[] { false, true };
+
+ private static final Log LOG = LogFactory.getLog(TestHFileBlockCompatibility.class);
+
+ private static final Compression.Algorithm[] COMPRESSION_ALGORITHMS = {
+ NONE, GZ };
+
+ // The mnior version for pre-checksum files
+ private static int MINOR_VERSION = 0;
+
+ private static final HBaseTestingUtility TEST_UTIL =
+ new HBaseTestingUtility();
+ private HFileSystem fs;
+ private int uncompressedSizeV1;
+
+ private final boolean includesMemstoreTS;
+
+ public TestHFileBlockCompatibility(boolean includesMemstoreTS) {
+ this.includesMemstoreTS = includesMemstoreTS;
+ }
+
+ @Parameters
+ public static Collection<Object[]> parameters() {
+ return HBaseTestingUtility.BOOLEAN_PARAMETERIZED;
+ }
+
+ @Before
+ public void setUp() throws IOException {
+ fs = (HFileSystem)HFileSystem.get(TEST_UTIL.getConfiguration());
+ }
+
+ public byte[] createTestV1Block(Compression.Algorithm algo)
+ throws IOException {
+ Compressor compressor = algo.getCompressor();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ OutputStream os = algo.createCompressionStream(baos, compressor, 0);
+ DataOutputStream dos = new DataOutputStream(os);
+ BlockType.META.write(dos); // Let's make this a meta block.
+ TestHFileBlock.writeTestBlockContents(dos);
+ uncompressedSizeV1 = dos.size();
+ dos.flush();
+ algo.returnCompressor(compressor);
+ return baos.toByteArray();
+ }
+
+ private Writer createTestV2Block(Compression.Algorithm algo)
+ throws IOException {
+ final BlockType blockType = BlockType.DATA;
+ Writer hbw = new Writer(algo, null,
+ includesMemstoreTS);
+ DataOutputStream dos = hbw.startWriting(blockType);
+ TestHFileBlock.writeTestBlockContents(dos);
+ byte[] headerAndData = hbw.getHeaderAndData();
+ assertEquals(1000 * 4, hbw.getUncompressedSizeWithoutHeader());
+ hbw.releaseCompressor();
+ return hbw;
+ }
+
+ private String createTestBlockStr(Compression.Algorithm algo,
+ int correctLength) throws IOException {
+ Writer hbw = createTestV2Block(algo);
+ byte[] testV2Block = hbw.getHeaderAndData();
+ int osOffset = HFileBlock.HEADER_SIZE_NO_CHECKSUM + 9;
+ if (testV2Block.length == correctLength) {
+ // Force-set the "OS" field of the gzip header to 3 (Unix) to avoid
+ // variations across operating systems.
+ // See http://www.gzip.org/zlib/rfc-gzip.html for gzip format.
+ testV2Block[osOffset] = 3;
+ }
+ return Bytes.toStringBinary(testV2Block);
+ }
+
+ @Test
+ public void testNoCompression() throws IOException {
+ assertEquals(4000, createTestV2Block(NONE).getBlockForCaching().
+ getUncompressedSizeWithoutHeader());
+ }
+
+ @Test
+ public void testGzipCompression() throws IOException {
+ final String correctTestBlockStr =
+ "DATABLK*\\x00\\x00\\x00:\\x00\\x00\\x0F\\xA0\\xFF\\xFF\\xFF\\xFF"
+ + "\\xFF\\xFF\\xFF\\xFF"
+ // gzip-compressed block: http://www.gzip.org/zlib/rfc-gzip.html
+ + "\\x1F\\x8B" // gzip magic signature
+ + "\\x08" // Compression method: 8 = "deflate"
+ + "\\x00" // Flags
+ + "\\x00\\x00\\x00\\x00" // mtime
+ + "\\x00" // XFL (extra flags)
+ // OS (0 = FAT filesystems, 3 = Unix). However, this field
+ // sometimes gets set to 0 on Linux and Mac, so we reset it to 3.
+ + "\\x03"
+ + "\\xED\\xC3\\xC1\\x11\\x00 \\x08\\xC00DD\\xDD\\x7Fa"
+ + "\\xD6\\xE8\\xA3\\xB9K\\x84`\\x96Q\\xD3\\xA8\\xDB\\xA8e\\xD4c"
+ + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\s\\xA0\\x0F\\x00\\x00";
+ final int correctGzipBlockLength = 82;
+ assertEquals(correctTestBlockStr, createTestBlockStr(GZ,
+ correctGzipBlockLength));
+ }
+
+ @Test
+ public void testReaderV1() throws IOException {
+ for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
+ for (boolean pread : new boolean[] { false, true }) {
+ byte[] block = createTestV1Block(algo);
+ Path path = new Path(TEST_UTIL.getDataTestDir(),
+ "blocks_v1_"+ algo);
+ LOG.info("Creating temporary file at " + path);
+ FSDataOutputStream os = fs.create(path);
+ int totalSize = 0;
+ int numBlocks = 50;
+ for (int i = 0; i < numBlocks; ++i) {
+ os.write(block);
+ totalSize += block.length;
+ }
+ os.close();
+
+ FSDataInputStream is = fs.open(path);
+ HFileBlock.FSReader hbr = new HFileBlock.FSReaderV1(is, algo,
+ totalSize);
+ HFileBlock b;
+ int numBlocksRead = 0;
+ long pos = 0;
+ while (pos < totalSize) {
+ b = hbr.readBlockData(pos, block.length, uncompressedSizeV1, pread);
+ b.sanityCheck();
+ pos += block.length;
+ numBlocksRead++;
+ }
+ assertEquals(numBlocks, numBlocksRead);
+ is.close();
+ }
+ }
+ }
+
+ @Test
+ public void testReaderV2() throws IOException {
+ for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
+ for (boolean pread : new boolean[] { false, true }) {
+ LOG.info("testReaderV2: Compression algorithm: " + algo +
+ ", pread=" + pread);
+ Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
+ + algo);
+ FSDataOutputStream os = fs.create(path);
+ Writer hbw = new Writer(algo, null,
+ includesMemstoreTS);
+ long totalSize = 0;
+ for (int blockId = 0; blockId < 2; ++blockId) {
+ DataOutputStream dos = hbw.startWriting(BlockType.DATA);
+ for (int i = 0; i < 1234; ++i)
+ dos.writeInt(i);
+ hbw.writeHeaderAndData(os);
+ totalSize += hbw.getOnDiskSizeWithHeader();
+ }
+ os.close();
+
+ FSDataInputStream is = fs.open(path);
+ HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, is, algo,
+ totalSize, MINOR_VERSION, fs, path);
+ HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
+ is.close();
+
+ b.sanityCheck();
+ assertEquals(4936, b.getUncompressedSizeWithoutHeader());
+ assertEquals(algo == GZ ? 2173 : 4936,
+ b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
+ String blockStr = b.toString();
+
+ if (algo == GZ) {
+ is = fs.open(path);
+ hbr = new HFileBlock.FSReaderV2(is, is, algo, totalSize, MINOR_VERSION,
+ fs, path);
+ b = hbr.readBlockData(0, 2173 + HFileBlock.HEADER_SIZE_NO_CHECKSUM +
+ b.totalChecksumBytes(), -1, pread);
+ assertEquals(blockStr, b.toString());
+ int wrongCompressedSize = 2172;
+ try {
+ b = hbr.readBlockData(0, wrongCompressedSize
+ + HFileBlock.HEADER_SIZE_NO_CHECKSUM, -1, pread);
+ fail("Exception expected");
+ } catch (IOException ex) {
+ String expectedPrefix = "On-disk size without header provided is "
+ + wrongCompressedSize + ", but block header contains "
+ + b.getOnDiskSizeWithoutHeader() + ".";
+ assertTrue("Invalid exception message: '" + ex.getMessage()
+ + "'.\nMessage is expected to start with: '" + expectedPrefix
+ + "'", ex.getMessage().startsWith(expectedPrefix));
+ }
+ is.close();
+ }
+ }
+ }
+ }
+
+ /**
+ * Test encoding/decoding data blocks.
+ * @throws IOException a bug or a problem with temporary files.
+ */
+ @Test
+ public void testDataBlockEncoding() throws IOException {
+ final int numBlocks = 5;
+ for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
+ for (boolean pread : new boolean[] { false, true }) {
+ for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
+ LOG.info("testDataBlockEncoding algo " + algo +
+ " pread = " + pread +
+ " encoding " + encoding);
+ Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
+ + algo + "_" + encoding.toString());
+ FSDataOutputStream os = fs.create(path);
+ HFileDataBlockEncoder dataBlockEncoder =
+ new HFileDataBlockEncoderImpl(encoding);
+ Writer hbw = new Writer(algo, dataBlockEncoder,
+ includesMemstoreTS);
+ long totalSize = 0;
+ final List<Integer> encodedSizes = new ArrayList<Integer>();
+ final List<ByteBuffer> encodedBlocks = new ArrayList<ByteBuffer>();
+ for (int blockId = 0; blockId < numBlocks; ++blockId) {
+ DataOutputStream dos = hbw.startWriting(BlockType.DATA);
+ TestHFileBlock.writeEncodedBlock(encoding, dos, encodedSizes, encodedBlocks,
+ blockId, includesMemstoreTS);
+
+ hbw.writeHeaderAndData(os);
+ totalSize += hbw.getOnDiskSizeWithHeader();
+ }
+ os.close();
+
+ FSDataInputStream is = fs.open(path);
+ HFileBlock.FSReaderV2 hbr = new HFileBlock.FSReaderV2(is, is, algo,
+ totalSize, MINOR_VERSION, fs, path);
+ hbr.setDataBlockEncoder(dataBlockEncoder);
+ hbr.setIncludesMemstoreTS(includesMemstoreTS);
+
+ HFileBlock b;
+ int pos = 0;
+ for (int blockId = 0; blockId < numBlocks; ++blockId) {
+ b = hbr.readBlockData(pos, -1, -1, pread);
+ b.sanityCheck();
+ pos += b.getOnDiskSizeWithHeader();
+
+ assertEquals((int) encodedSizes.get(blockId),
+ b.getUncompressedSizeWithoutHeader());
+ ByteBuffer actualBuffer = b.getBufferWithoutHeader();
+ if (encoding != DataBlockEncoding.NONE) {
+ // We expect a two-byte big-endian encoding id.
+ assertEquals(0, actualBuffer.get(0));
+ assertEquals(encoding.getId(), actualBuffer.get(1));
+ actualBuffer.position(2);
+ actualBuffer = actualBuffer.slice();
+ }
+
+ ByteBuffer expectedBuffer = encodedBlocks.get(blockId);
+ expectedBuffer.rewind();
+
+ // test if content matches, produce nice message
+ TestHFileBlock.assertBuffersEqual(expectedBuffer, actualBuffer, algo, encoding,
+ pread);
+ }
+ is.close();
+ }
+ }
+ }
+ }
+
+ @org.junit.Rule
+ public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+ new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+
+
+ /**
+ * This is the version of the HFileBlock.Writer that is used to
+ * create V2 blocks with minor version 0. These blocks do not
+ * have hbase-level checksums. The code is here to test
+ * backward compatibility. The reason we do not inherit from
+ * HFileBlock.Writer is because we never ever want to change the code
+ * in this class but the code in HFileBlock.Writer will continually
+ * evolve.
+ */
+ public static final class Writer {
+
+ // These constants are as they were in minorVersion 0.
+ private static final int HEADER_SIZE = HFileBlock.HEADER_SIZE_NO_CHECKSUM;
+ private static final boolean DONT_FILL_HEADER = HFileBlock.DONT_FILL_HEADER;
+ private static final byte[] DUMMY_HEADER =
+ HFileBlock.DUMMY_HEADER_NO_CHECKSUM;
+
+ private enum State {
+ INIT,
+ WRITING,
+ BLOCK_READY
+ };
+
+ /** Writer state. Used to ensure the correct usage protocol. */
+ private State state = State.INIT;
+
+ /** Compression algorithm for all blocks this instance writes. */
+ private final Compression.Algorithm compressAlgo;
+
+ /** Data block encoder used for data blocks */
+ private final HFileDataBlockEncoder dataBlockEncoder;
+
+ /**
+ * The stream we use to accumulate data in uncompressed format for each
+ * block. We reset this stream at the end of each block and reuse it. The
+ * header is written as the first {@link #HEADER_SIZE} bytes into this
+ * stream.
+ */
+ private ByteArrayOutputStream baosInMemory;
+
+ /** Compressor, which is also reused between consecutive blocks. */
+ private Compressor compressor;
+
+ /** Compression output stream */
+ private CompressionOutputStream compressionStream;
+
+ /** Underlying stream to write compressed bytes to */
+ private ByteArrayOutputStream compressedByteStream;
+
+ /**
+ * Current block type. Set in {@link #startWriting(BlockType)}. Could be
+ * changed in {@link #encodeDataBlockForDisk()} from {@link BlockType#DATA}
+ * to {@link BlockType#ENCODED_DATA}.
+ */
+ private BlockType blockType;
+
+ /**
+ * A stream that we write uncompressed bytes to, which compresses them and
+ * writes them to {@link #baosInMemory}.
+ */
+ private DataOutputStream userDataStream;
+
+ /**
+ * Bytes to be written to the file system, including the header. Compressed
+ * if compression is turned on.
+ */
+ private byte[] onDiskBytesWithHeader;
+
+ /**
+ * Valid in the READY state. Contains the header and the uncompressed (but
+ * potentially encoded, if this is a data block) bytes, so the length is
+ * {@link #uncompressedSizeWithoutHeader} + {@link HFileBlock#HEADER_SIZE}.
+ */
+ private byte[] uncompressedBytesWithHeader;
+
+ /**
+ * Current block's start offset in the {@link HFile}. Set in
+ * {@link #writeHeaderAndData(FSDataOutputStream)}.
+ */
+ private long startOffset;
+
+ /**
+ * Offset of previous block by block type. Updated when the next block is
+ * started.
+ */
+ private long[] prevOffsetByType;
+
+ /** The offset of the previous block of the same type */
+ private long prevOffset;
+
+ /** Whether we are including memstore timestamp after every key/value */
+ private boolean includesMemstoreTS;
+
+ /**
+ * @param compressionAlgorithm compression algorithm to use
+ * @param dataBlockEncoderAlgo data block encoding algorithm to use
+ */
+ public Writer(Compression.Algorithm compressionAlgorithm,
+ HFileDataBlockEncoder dataBlockEncoder, boolean includesMemstoreTS) {
+ compressAlgo = compressionAlgorithm == null ? NONE : compressionAlgorithm;
+ this.dataBlockEncoder = dataBlockEncoder != null
+ ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
+
+ baosInMemory = new ByteArrayOutputStream();
+ if (compressAlgo != NONE) {
+ compressor = compressionAlgorithm.getCompressor();
+ compressedByteStream = new ByteArrayOutputStream();
+ try {
+ compressionStream =
+ compressionAlgorithm.createPlainCompressionStream(
+ compressedByteStream, compressor);
+ } catch (IOException e) {
+ throw new RuntimeException("Could not create compression stream " +
+ "for algorithm " + compressionAlgorithm, e);
+ }
+ }
+
+ prevOffsetByType = new long[BlockType.values().length];
+ for (int i = 0; i < prevOffsetByType.length; ++i)
+ prevOffsetByType[i] = -1;
+
+ this.includesMemstoreTS = includesMemstoreTS;
+ }
+
+ /**
+ * Starts writing into the block. The previous block's data is discarded.
+ *
+ * @return the stream the user can write their data into
+ * @throws IOException
+ */
+ public DataOutputStream startWriting(BlockType newBlockType)
+ throws IOException {
+ if (state == State.BLOCK_READY && startOffset != -1) {
+ // We had a previous block that was written to a stream at a specific
+ // offset. Save that offset as the last offset of a block of that type.
+ prevOffsetByType[blockType.getId()] = startOffset;
+ }
+
+ startOffset = -1;
+ blockType = newBlockType;
+
+ baosInMemory.reset();
+ baosInMemory.write(DUMMY_HEADER);
+
+ state = State.WRITING;
+
+ // We will compress it later in finishBlock()
+ userDataStream = new DataOutputStream(baosInMemory);
+ return userDataStream;
+ }
+
+ /**
+ * Returns the stream for the user to write to. The block writer takes care
+ * of handling compression and buffering for caching on write. Can only be
+ * called in the "writing" state.
+ *
+ * @return the data output stream for the user to write to
+ */
+ DataOutputStream getUserDataStream() {
+ expectState(State.WRITING);
+ return userDataStream;
+ }
+
+ /**
+ * Transitions the block writer from the "writing" state to the "block
+ * ready" state. Does nothing if a block is already finished.
+ */
+ private void ensureBlockReady() throws IOException {
+ Preconditions.checkState(state != State.INIT,
+ "Unexpected state: " + state);
+
+ if (state == State.BLOCK_READY)
+ return;
+
+ // This will set state to BLOCK_READY.
+ finishBlock();
+ }
+
+ /**
+ * An internal method that flushes the compressing stream (if using
+ * compression), serializes the header, and takes care of the separate
+ * uncompressed stream for caching on write, if applicable. Sets block
+ * write state to "block ready".
+ */
+ private void finishBlock() throws IOException {
+ userDataStream.flush();
+
+ // This does an array copy, so it is safe to cache this byte array.
+ uncompressedBytesWithHeader = baosInMemory.toByteArray();
+ LOG.warn("Writer.finishBlock user data size with header before compression " +
+ uncompressedBytesWithHeader.length);
+ prevOffset = prevOffsetByType[blockType.getId()];
+
+ // We need to set state before we can package the block up for
+ // cache-on-write. In a way, the block is ready, but not yet encoded or
+ // compressed.
+ state = State.BLOCK_READY;
+ encodeDataBlockForDisk();
+
+ doCompression();
+ putHeader(uncompressedBytesWithHeader, 0, onDiskBytesWithHeader.length,
+ uncompressedBytesWithHeader.length);
+ }
+
+ /**
+ * Do compression if it is enabled, or re-use the uncompressed buffer if
+ * it is not. Fills in the compressed block's header if doing compression.
+ */
+ private void doCompression() throws IOException {
+ // do the compression
+ if (compressAlgo != NONE) {
+ compressedByteStream.reset();
+ compressedByteStream.write(DUMMY_HEADER);
+
+ compressionStream.resetState();
+
+ compressionStream.write(uncompressedBytesWithHeader, HEADER_SIZE,
+ uncompressedBytesWithHeader.length - HEADER_SIZE);
+
+ compressionStream.flush();
+ compressionStream.finish();
+
+ onDiskBytesWithHeader = compressedByteStream.toByteArray();
+ putHeader(onDiskBytesWithHeader, 0, onDiskBytesWithHeader.length,
+ uncompressedBytesWithHeader.length);
+ } else {
+ onDiskBytesWithHeader = uncompressedBytesWithHeader;
+ }
+ }
+
+ /**
+ * Encodes this block if it is a data block and encoding is turned on in
+ * {@link #dataBlockEncoder}.
+ */
+ private void encodeDataBlockForDisk() throws IOException {
+ if (blockType != BlockType.DATA) {
+ return; // skip any non-data block
+ }
+
+ // do data block encoding, if data block encoder is set
+ ByteBuffer rawKeyValues = ByteBuffer.wrap(uncompressedBytesWithHeader,
+ HEADER_SIZE, uncompressedBytesWithHeader.length -
+ HEADER_SIZE).slice();
+ Pair<ByteBuffer, BlockType> encodingResult =
+ dataBlockEncoder.beforeWriteToDisk(rawKeyValues,
+ includesMemstoreTS, DUMMY_HEADER);
+
+ BlockType encodedBlockType = encodingResult.getSecond();
+ if (encodedBlockType == BlockType.ENCODED_DATA) {
+ uncompressedBytesWithHeader = encodingResult.getFirst().array();
+ blockType = BlockType.ENCODED_DATA;
+ } else {
+ // There is no encoding configured. Do some extra sanity-checking.
+ if (encodedBlockType != BlockType.DATA) {
+ throw new IOException("Unexpected block type coming out of data " +
+ "block encoder: " + encodedBlockType);
+ }
+ if (userDataStream.size() !=
+ uncompressedBytesWithHeader.length - HEADER_SIZE) {
+ throw new IOException("Uncompressed size mismatch: "
+ + userDataStream.size() + " vs. "
+ + (uncompressedBytesWithHeader.length - HEADER_SIZE));
+ }
+ }
+ }
+
+ /**
+ * Put the header into the given byte array at the given offset.
+ * @param onDiskSize size of the block on disk
+ * @param uncompressedSize size of the block after decompression (but
+ * before optional data block decoding)
+ */
+ private void putHeader(byte[] dest, int offset, int onDiskSize,
+ int uncompressedSize) {
+ offset = blockType.put(dest, offset);
+ offset = Bytes.putInt(dest, offset, onDiskSize - HEADER_SIZE);
+ offset = Bytes.putInt(dest, offset, uncompressedSize - HEADER_SIZE);
+ Bytes.putLong(dest, offset, prevOffset);
+ }
+
+ /**
+ * Similar to {@link #writeHeaderAndData(FSDataOutputStream)}, but records
+ * the offset of this block so that it can be referenced in the next block
+ * of the same type.
+ *
+ * @param out
+ * @throws IOException
+ */
+ public void writeHeaderAndData(FSDataOutputStream out) throws IOException {
+ long offset = out.getPos();
+ if (startOffset != -1 && offset != startOffset) {
+ throw new IOException("A " + blockType + " block written to a "
+ + "stream twice, first at offset " + startOffset + ", then at "
+ + offset);
+ }
+ startOffset = offset;
+
+ writeHeaderAndData((DataOutputStream) out);
+ }
+
+ /**
+ * Writes the header and the compressed data of this block (or uncompressed
+ * data when not using compression) into the given stream. Can be called in
+ * the "writing" state or in the "block ready" state. If called in the
+ * "writing" state, transitions the writer to the "block ready" state.
+ *
+ * @param out the output stream to write the
+ * @throws IOException
+ */
+ private void writeHeaderAndData(DataOutputStream out) throws IOException {
+ ensureBlockReady();
+ out.write(onDiskBytesWithHeader);
+ }
+
+ /**
+ * Returns the header or the compressed data (or uncompressed data when not
+ * using compression) as a byte array. Can be called in the "writing" state
+ * or in the "block ready" state. If called in the "writing" state,
+ * transitions the writer to the "block ready" state.
+ *
+ * @return header and data as they would be stored on disk in a byte array
+ * @throws IOException
+ */
+ public byte[] getHeaderAndData() throws IOException {
+ ensureBlockReady();
+ return onDiskBytesWithHeader;
+ }
+
+ /**
+ * Releases the compressor this writer uses to compress blocks into the
+ * compressor pool. Needs to be called before the writer is discarded.
+ */
+ public void releaseCompressor() {
+ if (compressor != null) {
+ compressAlgo.returnCompressor(compressor);
+ compressor = null;
+ }
+ }
+
+ /**
+ * Returns the on-disk size of the data portion of the block. This is the
+ * compressed size if compression is enabled. Can only be called in the
+ * "block ready" state. Header is not compressed, and its size is not
+ * included in the return value.
+ *
+ * @return the on-disk size of the block, not including the header.
+ */
+ public int getOnDiskSizeWithoutHeader() {
+ expectState(State.BLOCK_READY);
+ return onDiskBytesWithHeader.length - HEADER_SIZE;
+ }
+
+ /**
+ * Returns the on-disk size of the block. Can only be called in the
+ * "block ready" state.
+ *
+ * @return the on-disk size of the block ready to be written, including the
+ * header size
+ */
+ public int getOnDiskSizeWithHeader() {
+ expectState(State.BLOCK_READY);
+ return onDiskBytesWithHeader.length;
+ }
+
+ /**
+ * The uncompressed size of the block data. Does not include header size.
+ */
+ public int getUncompressedSizeWithoutHeader() {
+ expectState(State.BLOCK_READY);
+ return uncompressedBytesWithHeader.length - HEADER_SIZE;
+ }
+
+ /**
+ * The uncompressed size of the block data, including header size.
+ */
+ public int getUncompressedSizeWithHeader() {
+ expectState(State.BLOCK_READY);
+ return uncompressedBytesWithHeader.length;
+ }
+
+ /** @return true if a block is being written */
+ public boolean isWriting() {
+ return state == State.WRITING;
+ }
+
+ /**
+ * Returns the number of bytes written into the current block so far, or
+ * zero if not writing the block at the moment. Note that this will return
+ * zero in the "block ready" state as well.
+ *
+ * @return the number of bytes written
+ */
+ public int blockSizeWritten() {
+ if (state != State.WRITING)
+ return 0;
+ return userDataStream.size();
+ }
+
+ /**
+ * Returns the header followed by the uncompressed data, even if using
+ * compression. This is needed for storing uncompressed blocks in the block
+ * cache. Can be called in the "writing" state or the "block ready" state.
+ *
+ * @return uncompressed block bytes for caching on write
+ */
+ private byte[] getUncompressedDataWithHeader() {
+ expectState(State.BLOCK_READY);
+
+ return uncompressedBytesWithHeader;
+ }
+
+ private void expectState(State expectedState) {
+ if (state != expectedState) {
+ throw new IllegalStateException("Expected state: " + expectedState +
+ ", actual state: " + state);
+ }
+ }
+
+ /**
+ * Similar to {@link #getUncompressedBufferWithHeader()} but returns a byte
+ * buffer.
+ *
+ * @return uncompressed block for caching on write in the form of a buffer
+ */
+ public ByteBuffer getUncompressedBufferWithHeader() {
+ byte[] b = getUncompressedDataWithHeader();
+ return ByteBuffer.wrap(b, 0, b.length);
+ }
+
+ /**
+ * Takes the given {@link BlockWritable} instance, creates a new block of
+ * its appropriate type, writes the writable into this block, and flushes
+ * the block into the output stream. The writer is instructed not to buffer
+ * uncompressed bytes for cache-on-write.
+ *
+ * @param bw the block-writable object to write as a block
+ * @param out the file system output stream
+ * @throws IOException
+ */
+ public void writeBlock(BlockWritable bw, FSDataOutputStream out)
+ throws IOException {
+ bw.writeToBlock(startWriting(bw.getBlockType()));
+ writeHeaderAndData(out);
+ }
+
+ /**
+ * Creates a new HFileBlock.
+ */
+ public HFileBlock getBlockForCaching() {
+ return new HFileBlock(blockType, getOnDiskSizeWithoutHeader(),
+ getUncompressedSizeWithoutHeader(), prevOffset,
+ getUncompressedBufferWithHeader(), DONT_FILL_HEADER, startOffset,
+ includesMemstoreTS, MINOR_VERSION, 0, ChecksumType.NULL.getCode(),
+ getOnDiskSizeWithoutHeader());
+ }
+ }
+}
+
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java Thu Mar 8 22:55:49 2012
@@ -44,6 +44,7 @@ import org.apache.hadoop.fs.FSDataOutput
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexReader;
import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexChunk;
import org.apache.hadoop.hbase.util.Bytes;
@@ -110,7 +111,7 @@ public class TestHFileBlockIndex {
// This test requires at least HFile format version 2.
conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION);
- fs = FileSystem.get(conf);
+ fs = HFileSystem.get(conf);
}
@Test
@@ -215,7 +216,8 @@ public class TestHFileBlockIndex {
private void writeWholeIndex() throws IOException {
assertEquals(0, keys.size());
HFileBlock.Writer hbw = new HFileBlock.Writer(compr, null,
- includesMemstoreTS);
+ includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+ HFile.DEFAULT_BYTES_PER_CHECKSUM);
FSDataOutputStream outputStream = fs.create(path);
HFileBlockIndex.BlockIndexWriter biw =
new HFileBlockIndex.BlockIndexWriter(hbw, null, null);