You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2012/03/08 23:55:50 UTC

svn commit: r1298641 [3/4] - in /hbase/trunk/src: main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/fs/ main/java/org/apache/hadoop/hbase/io/hfile/ main/java/org/apache/hadoop/hbase/mapreduce/ main/java/org/apache/hadoop/hbase/region...

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Thu Mar  8 22:55:49 2012
@@ -70,6 +70,7 @@ import org.apache.hadoop.hbase.regionser
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.hbase.util.CollectionBackedScanner;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -157,6 +158,10 @@ public class Store extends SchemaConfigu
   private final Compression.Algorithm compactionCompression;
   private HFileDataBlockEncoder dataBlockEncoder;
 
+  /** Checksum configuration */
+  private ChecksumType checksumType;
+  private int bytesPerChecksum;
+
   // Comparing KeyValues
   final KeyValue.KVComparator comparator;
 
@@ -246,6 +251,35 @@ public class Store extends SchemaConfigu
           "hbase.hstore.close.check.interval", 10*1000*1000 /* 10 MB */);
     }
     this.storefiles = sortAndClone(loadStoreFiles());
+
+    // Initialize checksum type from name. The names are CRC32, CRC32C, etc.
+    this.checksumType = getChecksumType(conf);
+    // initilize bytes per checksum
+    this.bytesPerChecksum = getBytesPerChecksum(conf);
+  }
+
+  /**
+   * Returns the configured bytesPerChecksum value.
+   * @param conf The configuration
+   * @return The bytesPerChecksum that is set in the configuration
+   */
+  public static int getBytesPerChecksum(Configuration conf) {
+    return conf.getInt(HConstants.BYTES_PER_CHECKSUM,
+                       HFile.DEFAULT_BYTES_PER_CHECKSUM);
+  }
+
+  /**
+   * Returns the configured checksum algorithm.
+   * @param conf The configuration
+   * @return The checksum algorithm that is set in the configuration
+   */
+  public static ChecksumType getChecksumType(Configuration conf) {
+    String checksumName = conf.get(HConstants.CHECKSUM_TYPE_NAME);
+    if (checksumName == null) {
+      return HFile.DEFAULT_CHECKSUM_TYPE;
+    } else {
+      return ChecksumType.nameToType(checksumName);
+    }
   }
 
   public HColumnDescriptor getFamily() {
@@ -799,6 +833,8 @@ public class Store extends SchemaConfigu
             .withComparator(comparator)
             .withBloomType(family.getBloomFilterType())
             .withMaxKeyCount(maxKeyCount)
+            .withChecksumType(checksumType)
+            .withBytesPerChecksum(bytesPerChecksum)
             .build();
     // The store file writer's path does not include the CF name, so we need
     // to configure the HFile writer directly.
@@ -2192,8 +2228,8 @@ public class Store extends SchemaConfigu
 
   public static final long FIXED_OVERHEAD = 
       ClassSize.align(SchemaConfigured.SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE +
-          + (19 * ClassSize.REFERENCE) + (6 * Bytes.SIZEOF_LONG)
-          + (5 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN);
+          + (20 * ClassSize.REFERENCE) + (6 * Bytes.SIZEOF_LONG)
+          + (6 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN);
 
   public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD
       + ClassSize.OBJECT + ClassSize.REENTRANT_LOCK

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Thu Mar  8 22:55:49 2012
@@ -60,6 +60,7 @@ import org.apache.hadoop.hbase.regionser
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
 import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
 import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
+import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.BloomFilter;
 import org.apache.hadoop.hbase.util.BloomFilterFactory;
 import org.apache.hadoop.hbase.util.BloomFilterWriter;
@@ -699,6 +700,8 @@ public class StoreFile extends SchemaCon
     private long maxKeyCount = 0;
     private Path dir;
     private Path filePath;
+    private ChecksumType checksumType = HFile.DEFAULT_CHECKSUM_TYPE;
+    private int bytesPerChecksum = HFile.DEFAULT_BYTES_PER_CHECKSUM;
 
     public WriterBuilder(Configuration conf, CacheConfig cacheConf,
         FileSystem fs, int blockSize) {
@@ -766,6 +769,24 @@ public class StoreFile extends SchemaCon
     }
 
     /**
+     * @param checksumType the type of checksum
+     * @return this (for chained invocation)
+     */
+    public WriterBuilder withChecksumType(ChecksumType checksumType) {
+      this.checksumType = checksumType;
+      return this;
+    }
+
+    /**
+     * @param bytesPerChecksum the number of bytes per checksum chunk
+     * @return this (for chained invocation)
+     */
+    public WriterBuilder withBytesPerChecksum(int bytesPerChecksum) {
+      this.bytesPerChecksum = bytesPerChecksum;
+      return this;
+    }
+
+    /**
      * Create a store file writer. Client is responsible for closing file when
      * done. If metadata, add BEFORE closing using
      * {@link Writer#appendMetadata}.
@@ -798,7 +819,8 @@ public class StoreFile extends SchemaCon
         comparator = KeyValue.COMPARATOR;
       }
       return new Writer(fs, filePath, blockSize, compressAlgo, dataBlockEncoder,
-          conf, cacheConf, comparator, bloomType, maxKeyCount);
+          conf, cacheConf, comparator, bloomType, maxKeyCount, checksumType,
+          bytesPerChecksum);
     }
   }
 
@@ -896,6 +918,12 @@ public class StoreFile extends SchemaCon
 
     protected HFileDataBlockEncoder dataBlockEncoder;
 
+    /** Checksum type */
+    protected ChecksumType checksumType;
+
+    /** Bytes per Checksum */
+    protected int bytesPerChecksum;
+    
     TimeRangeTracker timeRangeTracker = new TimeRangeTracker();
     /* isTimeRangeTrackerSet keeps track if the timeRange has already been set
      * When flushing a memstore, we set TimeRange and use this variable to
@@ -918,13 +946,16 @@ public class StoreFile extends SchemaCon
      * @param bloomType bloom filter setting
      * @param maxKeys the expected maximum number of keys to be added. Was used
      *        for Bloom filter size in {@link HFile} format version 1.
+     * @param checksumType the checksum type
+     * @param bytesPerChecksum the number of bytes per checksum value
      * @throws IOException problem writing to FS
      */
     private Writer(FileSystem fs, Path path, int blocksize,
         Compression.Algorithm compress,
         HFileDataBlockEncoder dataBlockEncoder, final Configuration conf,
         CacheConfig cacheConf,
-        final KVComparator comparator, BloomType bloomType, long maxKeys)
+        final KVComparator comparator, BloomType bloomType, long maxKeys,
+        final ChecksumType checksumType, final int bytesPerChecksum)
         throws IOException {
       this.dataBlockEncoder = dataBlockEncoder != null ?
           dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
@@ -934,6 +965,8 @@ public class StoreFile extends SchemaCon
           .withCompression(compress)
           .withDataBlockEncoder(dataBlockEncoder)
           .withComparator(comparator.getRawComparator())
+          .withChecksumType(checksumType)
+          .withBytesPerChecksum(bytesPerChecksum)
           .create();
 
       this.kvComparator = comparator;
@@ -964,6 +997,8 @@ public class StoreFile extends SchemaCon
         LOG.info("Delete Family Bloom filter type for " + path + ": "
             + deleteFamilyBloomFilterWriter.getClass().getSimpleName());
       }
+      this.checksumType = checksumType;
+      this.bytesPerChecksum = bytesPerChecksum;
     }
 
     /**
@@ -1660,7 +1695,7 @@ public class StoreFile extends SchemaCon
     }
 
     public int getHFileVersion() {
-      return reader.getTrailer().getVersion();
+      return reader.getTrailer().getMajorVersion();
     }
 
     HFile.Reader getHFileReader() {

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java Thu Mar  8 22:55:49 2012
@@ -248,6 +248,12 @@ public class RegionServerMetrics impleme
   public final MetricsTimeVaryingLong regionSplitFailureCount =
       new MetricsTimeVaryingLong("regionSplitFailureCount", registry);
 
+  /**
+   * Number of times checksum verification failed.
+   */
+  public final MetricsLongValue checksumFailuresCount =
+    new MetricsLongValue("checksumFailuresCount", registry);
+
   public RegionServerMetrics() {
     MetricsContext context = MetricsUtil.getContext("hbase");
     metricsRecord = MetricsUtil.createRecord(context, "regionserver");
@@ -346,6 +352,8 @@ public class RegionServerMetrics impleme
       // HFile metrics, positional reads
       ops = HFile.getPreadOps(); 
       if (ops != 0) this.fsPreadLatency.inc(ops, HFile.getPreadTimeMs());
+      this.checksumFailuresCount.set(HFile.getChecksumFailuresCount());
+
       /* NOTE: removed HFile write latency.  2 reasons:
        * 1) Mixing HLog latencies are far higher priority since they're 
        *      on-demand and HFile is used in background (compact/flush)
@@ -366,6 +374,7 @@ public class RegionServerMetrics impleme
       this.slowHLogAppendCount.pushMetric(this.metricsRecord);
       this.regionSplitSuccessCount.pushMetric(this.metricsRecord);
       this.regionSplitFailureCount.pushMetric(this.metricsRecord);
+      this.checksumFailuresCount.pushMetric(this.metricsRecord);
     }
     this.metricsRecord.update();
   }

Added: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/ChecksumFactory.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/ChecksumFactory.java?rev=1298641&view=auto
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/ChecksumFactory.java (added)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/ChecksumFactory.java Thu Mar  8 22:55:49 2012
@@ -0,0 +1,99 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.lang.ClassNotFoundException;
+import java.util.zip.Checksum;
+import java.lang.reflect.Constructor;
+
+/**
+ * Utility class that is used to generate a Checksum object.
+ * The Checksum implementation is pluggable and an application
+ * can specify their own class that implements their own
+ * Checksum algorithm.
+ */
+public class ChecksumFactory {
+
+  static private final Class<?>[] EMPTY_ARRAY = new Class[]{};
+
+  /**
+   * Create a new instance of a Checksum object.
+   * @return The newly created Checksum object
+   */
+  static public Checksum newInstance(String className) throws IOException {
+    try {
+      Class<?> clazz = getClassByName(className);
+      return (Checksum)newInstance(clazz);
+    } catch (ClassNotFoundException e) {
+      throw new IOException(e);
+    }
+  }
+
+  /**
+   * Returns a Constructor that can be used to create a Checksum object.
+   * @return The Constructor that can be used to create a 
+   * new Checksum object.
+   * @param theClass classname for which an constructor is created
+   * @return a new Constructor object
+   */
+  static public Constructor<?> newConstructor(String className) 
+    throws IOException {
+    try {
+      Class<?> clazz = getClassByName(className);
+      Constructor<?> ctor = clazz.getDeclaredConstructor(EMPTY_ARRAY);
+      ctor.setAccessible(true);
+      return ctor;
+    } catch (ClassNotFoundException e) {
+      throw new IOException(e);
+    } catch (java.lang.NoSuchMethodException e) {
+      throw new IOException(e);
+    }
+  }
+
+  /** Create an object for the given class and initialize it from conf
+   *
+   * @param theClass class of which an object is created
+   * @return a new object
+   */
+  static private <T> T newInstance(Class<T> theClass) {
+    T result;
+    try {
+      Constructor<T> ctor = theClass.getDeclaredConstructor(EMPTY_ARRAY);
+      ctor.setAccessible(true);
+      result = ctor.newInstance();
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+    return result;
+  }
+
+  /**
+   * Load a class by name.
+   * @param name the class name.
+   * @return the class object.
+   * @throws ClassNotFoundException if the class is not found.
+   */
+  static private Class<?> getClassByName(String name) 
+    throws ClassNotFoundException {
+    ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+    return Class.forName(name, true, classLoader);
+  }
+}

Added: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java?rev=1298641&view=auto
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java (added)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java Thu Mar  8 22:55:49 2012
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.util.zip.Checksum;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.util.ChecksumFactory;
+
+/**
+ * Checksum types. The Checksum type is a one byte number
+ * that stores a representation of the checksum algorithm
+ * used to encode a hfile. The ordinal of these cannot 
+ * change or else you risk breaking all existing HFiles out there.
+ */
+public enum ChecksumType {
+
+  NULL((byte)0) {
+    @Override
+    public String getName() {
+      return "NULL";
+    }
+    @Override
+    public void initialize() {
+      // do nothing
+    }
+    @Override
+    public Checksum getChecksumObject() throws IOException {
+      return null; // checksums not used
+    }
+  },
+
+  CRC32((byte)1) {
+    private volatile Constructor<?> ctor;
+
+    @Override
+    public String getName() {
+      return "CRC32";
+    }
+
+    @Override
+    public void initialize() {
+      final String PURECRC32 = "org.apache.hadoop.util.PureJavaCrc32";
+      final String JDKCRC = "java.util.zip.CRC32";
+      LOG = LogFactory.getLog(ChecksumType.class);
+
+      // check if hadoop library is available
+      try {
+        ctor = ChecksumFactory.newConstructor(PURECRC32);
+        LOG.info("Checksum using " + PURECRC32);
+      } catch (Exception e) {
+        LOG.info(PURECRC32 + " not available.");
+      }
+      try {
+        // The default checksum class name is java.util.zip.CRC32. 
+        // This is available on all JVMs.
+        if (ctor == null) {
+          ctor = ChecksumFactory.newConstructor(JDKCRC);
+          LOG.info("Checksum can use " + JDKCRC);
+        }
+      } catch (Exception e) {
+        LOG.warn(JDKCRC + " not available. ",  e);
+      }
+    }
+
+    @Override
+    public Checksum getChecksumObject() throws IOException {
+      if (ctor == null) {
+        throw new IOException("Bad constructor for " + getName());
+      }
+      try {
+        return (Checksum)ctor.newInstance();
+      } catch (Exception e) {
+        throw new IOException(e);
+      }
+    }
+  },
+
+  CRC32C((byte)2) {
+    private transient Constructor<?> ctor;
+
+    @Override
+    public String getName() {
+      return "CRC32C";
+    }
+
+    @Override
+    public void initialize() {
+      final String PURECRC32C = "org.apache.hadoop.util.PureJavaCrc32C";
+      LOG = LogFactory.getLog(ChecksumType.class);
+      try {
+        ctor = ChecksumFactory.newConstructor(PURECRC32C);
+        LOG.info("Checksum can use " + PURECRC32C);
+      } catch (Exception e) {
+        LOG.info(PURECRC32C + " not available. ");
+      }
+    }
+
+    @Override
+    public Checksum getChecksumObject() throws IOException {
+      if (ctor == null) {
+        throw new IOException("Bad constructor for " + getName());
+      }
+      try {
+        return (Checksum)ctor.newInstance();
+      } catch (Exception e) {
+        throw new IOException(e);
+      }
+    }
+  };
+
+  private final byte code;
+  protected Log LOG;
+
+  /** initializes the relevant checksum class object */
+  abstract void initialize();
+
+  /** returns the name of this checksum type */
+  public abstract String getName();
+
+  private ChecksumType(final byte c) {
+    this.code = c;
+    initialize();
+  }
+
+  /** returns a object that can be used to generate/validate checksums */
+  public abstract Checksum getChecksumObject() throws IOException;
+
+  public byte getCode() {
+    return this.code;
+  }
+
+  /**
+   * Cannot rely on enum ordinals . They change if item is removed or moved.
+   * Do our own codes.
+   * @param b
+   * @return Type associated with passed code.
+   */
+  public static ChecksumType codeToType(final byte b) {
+    for (ChecksumType t : ChecksumType.values()) {
+      if (t.getCode() == b) {
+        return t;
+      }
+    }
+    throw new RuntimeException("Unknown checksum type code " + b);
+  }
+
+  /**
+   * Map a checksum name to a specific type.
+   * Do our own names.
+   * @param b
+   * @return Type associated with passed code.
+   */
+  public static ChecksumType nameToType(final String name) {
+    for (ChecksumType t : ChecksumType.values()) {
+      if (t.getName().equals(name)) {
+        return t;
+      }
+    }
+    throw new RuntimeException("Unknown checksum type name " + name);
+  }
+}

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java Thu Mar  8 22:55:49 2012
@@ -109,7 +109,7 @@ public class CompoundBloomFilter extends
 
       ByteBuffer bloomBuf = bloomBlock.getBufferReadOnly();
       result = ByteBloomFilter.contains(key, keyOffset, keyLength,
-          bloomBuf.array(), bloomBuf.arrayOffset() + HFileBlock.HEADER_SIZE,
+          bloomBuf.array(), bloomBuf.arrayOffset() + bloomBlock.headerSize(),
           bloomBlock.getUncompressedSizeWithoutHeader(), hash, hashCount);
     }
 

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Thu Mar  8 22:55:49 2012
@@ -58,7 +58,9 @@ import org.apache.hadoop.hbase.client.Pu
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.ChecksumUtil;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
 import org.apache.hadoop.hbase.master.HMaster;
@@ -189,6 +191,9 @@ public class HBaseTestingUtility {
 
   public HBaseTestingUtility(Configuration conf) {
     this.conf = conf;
+
+    // a hbase checksum verification failure will cause unit tests to fail
+    ChecksumUtil.generateExceptionForChecksumFailureForTest(true);
   }
 
   /**
@@ -1437,7 +1442,7 @@ public class HBaseTestingUtility {
   }
 
   public FileSystem getTestFileSystem() throws IOException {
-    return FileSystem.get(conf);
+    return HFileSystem.get(conf);
   }
 
   /**

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java Thu Mar  8 22:55:49 2012
@@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.hbase.MultithreadedTestUtil;
 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
 import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 
 public class CacheTestUtils {
@@ -323,7 +324,9 @@ public class CacheTestUtils {
       HFileBlock generated = new HFileBlock(BlockType.DATA,
           onDiskSizeWithoutHeader, uncompressedSizeWithoutHeader,
           prevBlockOffset, cachedBuffer, HFileBlock.DONT_FILL_HEADER,
-          blockSize, includesMemstoreTS);
+          blockSize, includesMemstoreTS, HFileBlock.MINOR_VERSION_NO_CHECKSUM,
+          0, ChecksumType.NULL.getCode(),
+          onDiskSizeWithoutHeader + HFileBlock.HEADER_SIZE);
 
       String strKey;
       /* No conflicting keys */

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java Thu Mar  8 22:55:49 2012
@@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.HColumnDe
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
@@ -49,6 +50,7 @@ import org.apache.hadoop.hbase.regionser
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.util.BloomFilterFactory;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.junit.After;
 import org.junit.Before;
@@ -88,6 +90,8 @@ public class TestCacheOnWrite {
   private static final int INDEX_BLOCK_SIZE = 512;
   private static final int BLOOM_BLOCK_SIZE = 4096;
   private static final BloomType BLOOM_TYPE = StoreFile.BloomType.ROWCOL;
+  private static final ChecksumType CKTYPE = ChecksumType.CRC32;
+  private static final int CKBYTES = 512;
 
   /** The number of valid key types possible in a store file */
   private static final int NUM_VALID_KEY_TYPES =
@@ -192,7 +196,7 @@ public class TestCacheOnWrite {
     conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
         cowType.shouldBeCached(BlockType.BLOOM_CHUNK));
     cowType.modifyConf(conf);
-    fs = FileSystem.get(conf);
+    fs = HFileSystem.get(conf);
     cacheConf = new CacheConfig(conf);
     blockCache = cacheConf.getBlockCache();
   }
@@ -292,6 +296,8 @@ public class TestCacheOnWrite {
             .withComparator(KeyValue.COMPARATOR)
             .withBloomType(BLOOM_TYPE)
             .withMaxKeyCount(NUM_KV)
+            .withChecksumType(CKTYPE)
+            .withBytesPerChecksum(CKBYTES)
             .build();
 
     final int rowLen = 32;

Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java?rev=1298641&view=auto
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java (added)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java Thu Mar  8 22:55:49 2012
@@ -0,0 +1,290 @@
+/*
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import static org.junit.Assert.*;
+
+import java.io.ByteArrayOutputStream;
+import java.io.ByteArrayInputStream;
+import java.io.DataOutputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.zip.Checksum;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.io.compress.Compressor;
+
+import static org.apache.hadoop.hbase.io.hfile.Compression.Algorithm.*;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(MediumTests.class)
+public class TestChecksum {
+  // change this value to activate more logs
+  private static final boolean detailedLogging = true;
+  private static final boolean[] BOOLEAN_VALUES = new boolean[] { false, true };
+
+  private static final Log LOG = LogFactory.getLog(TestHFileBlock.class);
+
+  static final Compression.Algorithm[] COMPRESSION_ALGORITHMS = {
+      NONE, GZ };
+
+  static final int[] BYTES_PER_CHECKSUM = {
+      50, 500, 688, 16*1024, (16*1024+980), 64 * 1024};
+
+  private static final HBaseTestingUtility TEST_UTIL =
+    new HBaseTestingUtility();
+  private FileSystem fs;
+  private HFileSystem hfs;
+
+  @Before
+  public void setUp() throws Exception {
+    fs = HFileSystem.get(TEST_UTIL.getConfiguration());
+    hfs = (HFileSystem)fs;
+  }
+
+  /**
+   * Introduce checksum failures and check that we can still read
+   * the data
+   */
+  @Test
+  public void testChecksumCorruption() throws IOException {
+    for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
+      for (boolean pread : new boolean[] { false, true }) {
+        LOG.info("testChecksumCorruption: Compression algorithm: " + algo +
+                   ", pread=" + pread);
+        Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
+            + algo);
+        FSDataOutputStream os = fs.create(path);
+        HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
+            true, HFile.DEFAULT_CHECKSUM_TYPE,
+            HFile.DEFAULT_BYTES_PER_CHECKSUM);
+        long totalSize = 0;
+        for (int blockId = 0; blockId < 2; ++blockId) {
+          DataOutputStream dos = hbw.startWriting(BlockType.DATA);
+          for (int i = 0; i < 1234; ++i)
+            dos.writeInt(i);
+          hbw.writeHeaderAndData(os);
+          totalSize += hbw.getOnDiskSizeWithHeader();
+        }
+        os.close();
+
+        // Use hbase checksums. 
+        assertEquals(true, hfs.useHBaseChecksum());
+        assertEquals(true, hfs.getNoChecksumFs() != hfs.getBackingFs());
+
+        // Do a read that purposely introduces checksum verification failures.
+        FSDataInputStream is = fs.open(path);
+        HFileBlock.FSReader hbr = new FSReaderV2Test(is, algo,
+            totalSize, HFile.MAX_FORMAT_VERSION, fs, path);
+        HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
+        b.sanityCheck();
+        assertEquals(4936, b.getUncompressedSizeWithoutHeader());
+        assertEquals(algo == GZ ? 2173 : 4936, 
+                     b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
+        // read data back from the hfile, exclude header and checksum
+        ByteBuffer bb = b.getBufferWithoutHeader(); // read back data
+        DataInputStream in = new DataInputStream(
+                               new ByteArrayInputStream(
+                                 bb.array(), bb.arrayOffset(), bb.limit()));
+
+        // assert that we encountered hbase checksum verification failures
+        // but still used hdfs checksums and read data successfully.
+        assertEquals(1, HFile.getChecksumFailuresCount());
+        validateData(in);
+
+        // A single instance of hbase checksum failure causes the reader to
+        // switch off hbase checksum verification for the next 100 read
+        // requests. Verify that this is correct.
+        for (int i = 0; i < 
+             HFileBlock.CHECKSUM_VERIFICATION_NUM_IO_THRESHOLD + 1; i++) {
+          b = hbr.readBlockData(0, -1, -1, pread);
+          assertEquals(0, HFile.getChecksumFailuresCount());
+        }
+        // The next read should have hbase checksum verification reanabled,
+        // we verify this by assertng that there was a hbase-checksum failure.
+        b = hbr.readBlockData(0, -1, -1, pread);
+        assertEquals(1, HFile.getChecksumFailuresCount());
+
+        // Since the above encountered a checksum failure, we switch
+        // back to not checking hbase checksums.
+        b = hbr.readBlockData(0, -1, -1, pread);
+        assertEquals(0, HFile.getChecksumFailuresCount());
+        is.close();
+
+        // Now, use a completely new reader. Switch off hbase checksums in 
+        // the configuration. In this case, we should not detect
+        // any retries within hbase. 
+        HFileSystem newfs = new HFileSystem(TEST_UTIL.getConfiguration(), false);
+        assertEquals(false, newfs.useHBaseChecksum());
+        is = newfs.open(path);
+        hbr = new FSReaderV2Test(is, algo,
+            totalSize, HFile.MAX_FORMAT_VERSION, newfs, path);
+        b = hbr.readBlockData(0, -1, -1, pread);
+        is.close();
+        b.sanityCheck();
+        assertEquals(4936, b.getUncompressedSizeWithoutHeader());
+        assertEquals(algo == GZ ? 2173 : 4936, 
+                     b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
+        // read data back from the hfile, exclude header and checksum
+        bb = b.getBufferWithoutHeader(); // read back data
+        in = new DataInputStream(new ByteArrayInputStream(
+                                 bb.array(), bb.arrayOffset(), bb.limit()));
+
+        // assert that we did not encounter hbase checksum verification failures
+        // but still used hdfs checksums and read data successfully.
+        assertEquals(0, HFile.getChecksumFailuresCount());
+        validateData(in);
+      }
+    }
+  }
+
+  /** 
+   * Test different values of bytesPerChecksum
+   */
+  @Test
+  public void testChecksumChunks() throws IOException {
+    Compression.Algorithm algo = NONE;
+    for (boolean pread : new boolean[] { false, true }) {
+      for (int bytesPerChecksum : BYTES_PER_CHECKSUM) {
+        Path path = new Path(TEST_UTIL.getDataTestDir(), "checksumChunk_" + 
+                             algo + bytesPerChecksum);
+        FSDataOutputStream os = fs.create(path);
+        HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
+          true, HFile.DEFAULT_CHECKSUM_TYPE, bytesPerChecksum);
+
+        // write one block. The block has data
+        // that is at least 6 times more than the checksum chunk size
+        long dataSize = 0;
+        DataOutputStream dos = hbw.startWriting(BlockType.DATA);
+        for (; dataSize < 6 * bytesPerChecksum;) {
+          for (int i = 0; i < 1234; ++i) {
+            dos.writeInt(i);
+            dataSize += 4;
+          }
+        }
+        hbw.writeHeaderAndData(os);
+        long totalSize = hbw.getOnDiskSizeWithHeader();
+        os.close();
+
+        long expectedChunks = ChecksumUtil.numChunks(
+                               dataSize + HFileBlock.HEADER_SIZE,
+                               bytesPerChecksum);
+        LOG.info("testChecksumChunks: pread=" + pread +
+                   ", bytesPerChecksum=" + bytesPerChecksum +
+                   ", fileSize=" + totalSize +
+                   ", dataSize=" + dataSize +
+                   ", expectedChunks=" + expectedChunks);
+
+        // Verify hbase checksums. 
+        assertEquals(true, hfs.useHBaseChecksum());
+        assertEquals(true, hfs.getNoChecksumFs() != hfs.getBackingFs());
+
+        // Read data back from file.
+        FSDataInputStream is = fs.open(path);
+        FSDataInputStream nochecksum = hfs.getNoChecksumFs().open(path);
+        HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, nochecksum, 
+            algo, totalSize, HFile.MAX_FORMAT_VERSION, hfs, path);
+        HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
+        is.close();
+        b.sanityCheck();
+        assertEquals(dataSize, b.getUncompressedSizeWithoutHeader());
+
+        // verify that we have the expected number of checksum chunks
+        assertEquals(totalSize, HFileBlock.HEADER_SIZE + dataSize + 
+                     expectedChunks * HFileBlock.CHECKSUM_SIZE);
+
+        // assert that we did not encounter hbase checksum verification failures
+        assertEquals(0, HFile.getChecksumFailuresCount());
+      }
+    }
+  }
+
+  /** 
+   * Test to ensure that these is at least one valid checksum implementation
+   */
+  @Test
+  public void testChecksumAlgorithm() throws IOException {
+    ChecksumType type = ChecksumType.CRC32;
+    assertEquals(ChecksumType.nameToType(type.getName()), type);
+    assertEquals(ChecksumType.valueOf(type.toString()), type);
+  }
+
+  private void validateData(DataInputStream in) throws IOException {
+    // validate data
+    for (int i = 0; i < 1234; i++) {
+      int val = in.readInt();
+      if (val != i) {
+        String msg = "testChecksumCorruption: data mismatch at index " +
+                     i + " expected " + i + " found " + val;
+        LOG.warn(msg);
+        assertEquals(i, val);
+      }
+    }
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+
+  /**
+   * A class that introduces hbase-checksum failures while 
+   * reading  data from hfiles. This should trigger the hdfs level
+   * checksum validations.
+   */
+  static private class FSReaderV2Test extends HFileBlock.FSReaderV2 {
+
+    FSReaderV2Test(FSDataInputStream istream, Algorithm algo,
+                   long fileSize, int minorVersion, FileSystem fs,
+                   Path path) throws IOException {
+      super(istream, istream, algo, fileSize, minorVersion, 
+            (HFileSystem)fs, path);
+    }
+
+    @Override
+    protected boolean validateBlockChecksum(HFileBlock block, 
+      byte[] data, int hdrSize) throws IOException {
+      return false;  // checksum validation failure
+    }
+  }
+}
+

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java Thu Mar  8 22:55:49 2012
@@ -52,7 +52,7 @@ public class TestFixedFileTrailer {
   private static final Log LOG = LogFactory.getLog(TestFixedFileTrailer.class);
 
   /** The number of used fields by version. Indexed by version minus one. */
-  private static final int[] NUM_FIELDS_BY_VERSION = new int[] { 8, 13 };
+  private static final int[] NUM_FIELDS_BY_VERSION = new int[] { 9, 14 };
 
   private HBaseTestingUtility util = new HBaseTestingUtility();
   private FileSystem fs;
@@ -83,7 +83,8 @@ public class TestFixedFileTrailer {
 
   @Test
   public void testTrailer() throws IOException {
-    FixedFileTrailer t = new FixedFileTrailer(version);
+    FixedFileTrailer t = new FixedFileTrailer(version, 
+                           HFileBlock.MINOR_VERSION_NO_CHECKSUM);
     t.setDataIndexCount(3);
     t.setEntryCount(((long) Integer.MAX_VALUE) + 1);
 
@@ -121,7 +122,8 @@ public class TestFixedFileTrailer {
     // Finished writing, trying to read.
     {
       DataInputStream dis = new DataInputStream(bais);
-      FixedFileTrailer t2 = new FixedFileTrailer(version);
+      FixedFileTrailer t2 = new FixedFileTrailer(version, 
+                              HFileBlock.MINOR_VERSION_NO_CHECKSUM);
       t2.deserialize(dis);
       assertEquals(-1, bais.read()); // Ensure we have read everything.
       checkLoadedTrailer(version, t, t2);
@@ -191,7 +193,7 @@ public class TestFixedFileTrailer {
 
   private void checkLoadedTrailer(int version, FixedFileTrailer expected,
       FixedFileTrailer loaded) throws IOException {
-    assertEquals(version, loaded.getVersion());
+    assertEquals(version, loaded.getMajorVersion());
     assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount());
 
     assertEquals(Math.min(expected.getEntryCount(),

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java Thu Mar  8 22:55:49 2012
@@ -48,9 +48,11 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.DoubleOutputStream;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.io.compress.CompressionOutputStream;
@@ -102,16 +104,16 @@ public class TestHFileBlock {
 
   @Before
   public void setUp() throws IOException {
-    fs = FileSystem.get(TEST_UTIL.getConfiguration());
+    fs = HFileSystem.get(TEST_UTIL.getConfiguration());
   }
 
-  public void writeTestBlockContents(DataOutputStream dos) throws IOException {
+  static void writeTestBlockContents(DataOutputStream dos) throws IOException {
     // This compresses really well.
     for (int i = 0; i < 1000; ++i)
       dos.writeInt(i / 100);
   }
 
-  private int writeTestKeyValues(OutputStream dos, int seed)
+ static int writeTestKeyValues(OutputStream dos, int seed, boolean includesMemstoreTS)
       throws IOException {
     List<KeyValue> keyValues = new ArrayList<KeyValue>();
     Random randomizer = new Random(42l + seed); // just any fixed number
@@ -191,22 +193,24 @@ public class TestHFileBlock {
     return baos.toByteArray();
   }
 
-  private byte[] createTestV2Block(Compression.Algorithm algo)
-      throws IOException {
+  static HFileBlock.Writer createTestV2Block(Compression.Algorithm algo,
+      boolean includesMemstoreTS) throws IOException {
     final BlockType blockType = BlockType.DATA;
     HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
-        includesMemstoreTS);
+        includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+        HFile.DEFAULT_BYTES_PER_CHECKSUM);
     DataOutputStream dos = hbw.startWriting(blockType);
     writeTestBlockContents(dos);
-    byte[] headerAndData = hbw.getHeaderAndData();
+    byte[] headerAndData = hbw.getHeaderAndDataForTest();
     assertEquals(1000 * 4, hbw.getUncompressedSizeWithoutHeader());
     hbw.releaseCompressor();
-    return headerAndData;
+    return hbw;
   }
 
   public String createTestBlockStr(Compression.Algorithm algo,
       int correctLength) throws IOException {
-    byte[] testV2Block = createTestV2Block(algo);
+    HFileBlock.Writer hbw = createTestV2Block(algo, includesMemstoreTS);
+    byte[] testV2Block = hbw.getHeaderAndDataForTest();
     int osOffset = HFileBlock.HEADER_SIZE + 9;
     if (testV2Block.length == correctLength) {
       // Force-set the "OS" field of the gzip header to 3 (Unix) to avoid
@@ -221,14 +225,16 @@ public class TestHFileBlock {
 
   @Test
   public void testNoCompression() throws IOException {
-    assertEquals(4000 + HFileBlock.HEADER_SIZE, createTestV2Block(NONE).length);
+    assertEquals(4000, createTestV2Block(NONE, includesMemstoreTS).
+                 getBlockForCaching().getUncompressedSizeWithoutHeader());
   }
 
   @Test
   public void testGzipCompression() throws IOException {
     final String correctTestBlockStr =
-        "DATABLK*\\x00\\x00\\x00:\\x00\\x00\\x0F\\xA0\\xFF\\xFF\\xFF\\xFF"
+        "DATABLK*\\x00\\x00\\x00>\\x00\\x00\\x0F\\xA0\\xFF\\xFF\\xFF\\xFF"
             + "\\xFF\\xFF\\xFF\\xFF"
+            + "\\x01\\x00\\x00@\\x00\\x00\\x00\\x00["
             // gzip-compressed block: http://www.gzip.org/zlib/rfc-gzip.html
             + "\\x1F\\x8B"  // gzip magic signature
             + "\\x08"  // Compression method: 8 = "deflate"
@@ -240,8 +246,9 @@ public class TestHFileBlock {
             + "\\x03"
             + "\\xED\\xC3\\xC1\\x11\\x00 \\x08\\xC00DD\\xDD\\x7Fa"
             + "\\xD6\\xE8\\xA3\\xB9K\\x84`\\x96Q\\xD3\\xA8\\xDB\\xA8e\\xD4c"
-            + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\s\\xA0\\x0F\\x00\\x00";
-    final int correctGzipBlockLength = 82;
+            + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\s\\xA0\\x0F\\x00\\x00"
+            + "\\xAB\\x85g\\x91"; //  4 byte checksum
+    final int correctGzipBlockLength = 95;
     assertEquals(correctTestBlockStr, createTestBlockStr(GZ,
         correctGzipBlockLength));
   }
@@ -285,11 +292,14 @@ public class TestHFileBlock {
   public void testReaderV2() throws IOException {
     for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
       for (boolean pread : new boolean[] { false, true }) {
+          LOG.info("testReaderV2: Compression algorithm: " + algo + 
+                   ", pread=" + pread);
         Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
             + algo);
         FSDataOutputStream os = fs.create(path);
         HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
-            includesMemstoreTS);
+            includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+            HFile.DEFAULT_BYTES_PER_CHECKSUM);
         long totalSize = 0;
         for (int blockId = 0; blockId < 2; ++blockId) {
           DataOutputStream dos = hbw.startWriting(BlockType.DATA);
@@ -305,16 +315,19 @@ public class TestHFileBlock {
             totalSize);
         HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
         is.close();
+        assertEquals(0, HFile.getChecksumFailuresCount());
 
         b.sanityCheck();
         assertEquals(4936, b.getUncompressedSizeWithoutHeader());
-        assertEquals(algo == GZ ? 2173 : 4936, b.getOnDiskSizeWithoutHeader());
+        assertEquals(algo == GZ ? 2173 : 4936, 
+                     b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
         String blockStr = b.toString();
 
         if (algo == GZ) {
           is = fs.open(path);
           hbr = new HFileBlock.FSReaderV2(is, algo, totalSize);
-          b = hbr.readBlockData(0, 2173 + HFileBlock.HEADER_SIZE, -1, pread);
+          b = hbr.readBlockData(0, 2173 + HFileBlock.HEADER_SIZE +
+                                b.totalChecksumBytes(), -1, pread);
           assertEquals(blockStr, b.toString());
           int wrongCompressedSize = 2172;
           try {
@@ -351,13 +364,15 @@ public class TestHFileBlock {
           HFileDataBlockEncoder dataBlockEncoder =
               new HFileDataBlockEncoderImpl(encoding);
           HFileBlock.Writer hbw = new HFileBlock.Writer(algo, dataBlockEncoder,
-              includesMemstoreTS);
+              includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+              HFile.DEFAULT_BYTES_PER_CHECKSUM);
           long totalSize = 0;
           final List<Integer> encodedSizes = new ArrayList<Integer>();
           final List<ByteBuffer> encodedBlocks = new ArrayList<ByteBuffer>();
           for (int blockId = 0; blockId < numBlocks; ++blockId) {
-            writeEncodedBlock(encoding, hbw, encodedSizes, encodedBlocks,
-                blockId);
+            DataOutputStream dos = hbw.startWriting(BlockType.DATA);
+            writeEncodedBlock(encoding, dos, encodedSizes, encodedBlocks,
+                blockId, includesMemstoreTS);
 
             hbw.writeHeaderAndData(os);
             totalSize += hbw.getOnDiskSizeWithHeader();
@@ -374,6 +389,7 @@ public class TestHFileBlock {
           int pos = 0;
           for (int blockId = 0; blockId < numBlocks; ++blockId) {
             b = hbr.readBlockData(pos, -1, -1, pread);
+            assertEquals(0, HFile.getChecksumFailuresCount());
             b.sanityCheck();
             pos += b.getOnDiskSizeWithHeader();
 
@@ -401,16 +417,16 @@ public class TestHFileBlock {
     }
   }
 
-  private void writeEncodedBlock(DataBlockEncoding encoding,
-      HFileBlock.Writer hbw, final List<Integer> encodedSizes,
-      final List<ByteBuffer> encodedBlocks, int blockId) throws IOException {
-    DataOutputStream dos = hbw.startWriting(BlockType.DATA);
+  static void writeEncodedBlock(DataBlockEncoding encoding,
+      DataOutputStream dos, final List<Integer> encodedSizes,
+      final List<ByteBuffer> encodedBlocks, int blockId, 
+      boolean includesMemstoreTS) throws IOException {
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     DoubleOutputStream doubleOutputStream =
         new DoubleOutputStream(dos, baos);
 
     final int rawBlockSize = writeTestKeyValues(doubleOutputStream,
-        blockId);
+        blockId, includesMemstoreTS);
 
     ByteBuffer rawBuf = ByteBuffer.wrap(baos.toByteArray());
     rawBuf.rewind();
@@ -434,7 +450,7 @@ public class TestHFileBlock {
     encodedBlocks.add(encodedBuf);
   }
 
-  private void assertBuffersEqual(ByteBuffer expectedBuffer,
+  static void assertBuffersEqual(ByteBuffer expectedBuffer,
       ByteBuffer actualBuffer, Compression.Algorithm compression,
       DataBlockEncoding encoding, boolean pread) {
     if (!actualBuffer.equals(expectedBuffer)) {
@@ -471,7 +487,9 @@ public class TestHFileBlock {
       for (boolean pread : BOOLEAN_VALUES) {
         for (boolean cacheOnWrite : BOOLEAN_VALUES) {
           Random rand = defaultRandom();
-          LOG.info("Compression algorithm: " + algo + ", pread=" + pread);
+          LOG.info("testPreviousOffset:Compression algorithm: " + algo + 
+                   ", pread=" + pread +
+                   ", cacheOnWrite=" + cacheOnWrite);
           Path path = new Path(TEST_UTIL.getDataTestDir(), "prev_offset");
           List<Long> expectedOffsets = new ArrayList<Long>();
           List<Long> expectedPrevOffsets = new ArrayList<Long>();
@@ -522,17 +540,23 @@ public class TestHFileBlock {
                 b2.getUncompressedSizeWithoutHeader());
             assertEquals(b.getPrevBlockOffset(), b2.getPrevBlockOffset());
             assertEquals(curOffset, b2.getOffset());
+            assertEquals(b.getBytesPerChecksum(), b2.getBytesPerChecksum());
+            assertEquals(b.getOnDiskDataSizeWithHeader(), 
+                         b2.getOnDiskDataSizeWithHeader());
+            assertEquals(0, HFile.getChecksumFailuresCount());
 
             curOffset += b.getOnDiskSizeWithHeader();
 
             if (cacheOnWrite) {
               // In the cache-on-write mode we store uncompressed bytes so we
               // can compare them to what was read by the block reader.
-
+              // b's buffer has header + data + checksum while 
+              // expectedContents have header + data only
               ByteBuffer bufRead = b.getBufferWithHeader();
               ByteBuffer bufExpected = expectedContents.get(i);
               boolean bytesAreCorrect = Bytes.compareTo(bufRead.array(),
-                  bufRead.arrayOffset(), bufRead.limit(),
+                  bufRead.arrayOffset(), 
+                  bufRead.limit() - b.totalChecksumBytes(),
                   bufExpected.array(), bufExpected.arrayOffset(),
                   bufExpected.limit()) == 0;
               String wrongBytesMsg = "";
@@ -541,15 +565,26 @@ public class TestHFileBlock {
                 // Optimization: only construct an error message in case we
                 // will need it.
                 wrongBytesMsg = "Expected bytes in block #" + i + " (algo="
-                    + algo + ", pread=" + pread + "):\n";
+                    + algo + ", pread=" + pread
+                    + ", cacheOnWrite=" + cacheOnWrite + "):\n";
                 wrongBytesMsg += Bytes.toStringBinary(bufExpected.array(),
                     bufExpected.arrayOffset(), Math.min(32,
                         bufExpected.limit()))
                     + ", actual:\n"
                     + Bytes.toStringBinary(bufRead.array(),
                         bufRead.arrayOffset(), Math.min(32, bufRead.limit()));
+                if (detailedLogging) {
+                  LOG.warn("expected header" + 
+                           HFileBlock.toStringHeader(bufExpected) +
+                           "\nfound    header" + 
+                           HFileBlock.toStringHeader(bufRead));
+                  LOG.warn("bufread offset " + bufRead.arrayOffset() +
+                           " limit " + bufRead.limit() +
+                           " expected offset " + bufExpected.arrayOffset() +
+                           " limit " + bufExpected.limit());
+                  LOG.warn(wrongBytesMsg);
+                }
               }
-
               assertTrue(wrongBytesMsg, bytesAreCorrect);
             }
           }
@@ -672,10 +707,12 @@ public class TestHFileBlock {
     boolean cacheOnWrite = expectedContents != null;
     FSDataOutputStream os = fs.create(path);
     HFileBlock.Writer hbw = new HFileBlock.Writer(compressAlgo, null,
-        includesMemstoreTS);
+        includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+        HFile.DEFAULT_BYTES_PER_CHECKSUM);
     Map<BlockType, Long> prevOffsetByType = new HashMap<BlockType, Long>();
     long totalSize = 0;
     for (int i = 0; i < NUM_TEST_BLOCKS; ++i) {
+      long pos = os.getPos();
       int blockTypeOrdinal = rand.nextInt(BlockType.values().length);
       if (blockTypeOrdinal == BlockType.ENCODED_DATA.ordinal()) {
         blockTypeOrdinal = BlockType.DATA.ordinal();
@@ -706,9 +743,9 @@ public class TestHFileBlock {
         expectedContents.add(hbw.getUncompressedBufferWithHeader());
 
       if (detailedLogging) {
-        LOG.info("Writing block #" + i + " of type " + bt
+        LOG.info("Written block #" + i + " of type " + bt
             + ", uncompressed size " + hbw.getUncompressedSizeWithoutHeader()
-            + " at offset " + os.getPos());
+            + " at offset " + pos);
       }
     }
     os.close();
@@ -730,7 +767,9 @@ public class TestHFileBlock {
       byte[] byteArr = new byte[HFileBlock.HEADER_SIZE + size];
       ByteBuffer buf = ByteBuffer.wrap(byteArr, 0, size);
       HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf,
-          HFileBlock.FILL_HEADER, -1, includesMemstoreTS);
+          HFileBlock.FILL_HEADER, -1, includesMemstoreTS, 
+          HFileBlock.MINOR_VERSION_NO_CHECKSUM, 0, ChecksumType.NULL.getCode(),
+          0);
       long byteBufferExpectedSize =
           ClassSize.align(ClassSize.estimateBase(buf.getClass(), true)
               + HFileBlock.HEADER_SIZE + size);

Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java?rev=1298641&view=auto
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java (added)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java Thu Mar  8 22:55:49 2012
@@ -0,0 +1,806 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import static org.junit.Assert.*;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.DoubleOutputStream;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ClassSize;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.io.compress.CompressionOutputStream;
+import org.apache.hadoop.io.compress.Compressor;
+import org.apache.hadoop.hbase.io.hfile.HFileBlock.BlockWritable;
+import org.apache.hadoop.hbase.util.ChecksumType;
+import org.apache.hadoop.hbase.util.Pair;
+import com.google.common.base.Preconditions;
+
+import static org.apache.hadoop.hbase.io.hfile.Compression.Algorithm.*;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * This class has unit tests to prove that older versions of
+ * HFiles (without checksums) are compatible with current readers.
+ */
+@Category(MediumTests.class)
+@RunWith(Parameterized.class)
+public class TestHFileBlockCompatibility {
+  // change this value to activate more logs
+  private static final boolean[] BOOLEAN_VALUES = new boolean[] { false, true };
+
+  private static final Log LOG = LogFactory.getLog(TestHFileBlockCompatibility.class);
+
+  private static final Compression.Algorithm[] COMPRESSION_ALGORITHMS = {
+      NONE, GZ };
+
+  // The mnior version for pre-checksum files
+  private static int MINOR_VERSION = 0;
+
+  private static final HBaseTestingUtility TEST_UTIL =
+    new HBaseTestingUtility();
+  private HFileSystem fs;
+  private int uncompressedSizeV1;
+
+  private final boolean includesMemstoreTS;
+
+  public TestHFileBlockCompatibility(boolean includesMemstoreTS) {
+    this.includesMemstoreTS = includesMemstoreTS;
+  }
+
+  @Parameters
+  public static Collection<Object[]> parameters() {
+    return HBaseTestingUtility.BOOLEAN_PARAMETERIZED;
+  }
+
+  @Before
+  public void setUp() throws IOException {
+    fs = (HFileSystem)HFileSystem.get(TEST_UTIL.getConfiguration());
+  }
+
+  public byte[] createTestV1Block(Compression.Algorithm algo)
+      throws IOException {
+    Compressor compressor = algo.getCompressor();
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    OutputStream os = algo.createCompressionStream(baos, compressor, 0);
+    DataOutputStream dos = new DataOutputStream(os);
+    BlockType.META.write(dos); // Let's make this a meta block.
+    TestHFileBlock.writeTestBlockContents(dos);
+    uncompressedSizeV1 = dos.size();
+    dos.flush();
+    algo.returnCompressor(compressor);
+    return baos.toByteArray();
+  }
+
+  private Writer createTestV2Block(Compression.Algorithm algo)
+      throws IOException {
+    final BlockType blockType = BlockType.DATA;
+    Writer hbw = new Writer(algo, null,
+        includesMemstoreTS);
+    DataOutputStream dos = hbw.startWriting(blockType);
+    TestHFileBlock.writeTestBlockContents(dos);
+    byte[] headerAndData = hbw.getHeaderAndData();
+    assertEquals(1000 * 4, hbw.getUncompressedSizeWithoutHeader());
+    hbw.releaseCompressor();
+    return hbw;
+  }
+
+ private String createTestBlockStr(Compression.Algorithm algo,
+      int correctLength) throws IOException {
+    Writer hbw = createTestV2Block(algo);
+    byte[] testV2Block = hbw.getHeaderAndData();
+    int osOffset = HFileBlock.HEADER_SIZE_NO_CHECKSUM + 9;
+    if (testV2Block.length == correctLength) {
+      // Force-set the "OS" field of the gzip header to 3 (Unix) to avoid
+      // variations across operating systems.
+      // See http://www.gzip.org/zlib/rfc-gzip.html for gzip format.
+      testV2Block[osOffset] = 3;
+    }   
+    return Bytes.toStringBinary(testV2Block);
+  }
+
+  @Test
+  public void testNoCompression() throws IOException {
+    assertEquals(4000, createTestV2Block(NONE).getBlockForCaching().
+                       getUncompressedSizeWithoutHeader());
+  }
+
+  @Test
+  public void testGzipCompression() throws IOException {
+    final String correctTestBlockStr =
+        "DATABLK*\\x00\\x00\\x00:\\x00\\x00\\x0F\\xA0\\xFF\\xFF\\xFF\\xFF"
+            + "\\xFF\\xFF\\xFF\\xFF"
+            // gzip-compressed block: http://www.gzip.org/zlib/rfc-gzip.html
+            + "\\x1F\\x8B"  // gzip magic signature
+            + "\\x08"  // Compression method: 8 = "deflate"
+            + "\\x00"  // Flags
+            + "\\x00\\x00\\x00\\x00"  // mtime
+            + "\\x00"  // XFL (extra flags)
+            // OS (0 = FAT filesystems, 3 = Unix). However, this field
+            // sometimes gets set to 0 on Linux and Mac, so we reset it to 3.
+            + "\\x03"
+            + "\\xED\\xC3\\xC1\\x11\\x00 \\x08\\xC00DD\\xDD\\x7Fa"
+            + "\\xD6\\xE8\\xA3\\xB9K\\x84`\\x96Q\\xD3\\xA8\\xDB\\xA8e\\xD4c"
+            + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\s\\xA0\\x0F\\x00\\x00";
+    final int correctGzipBlockLength = 82;
+    assertEquals(correctTestBlockStr, createTestBlockStr(GZ,
+        correctGzipBlockLength));
+  }
+
+  @Test
+  public void testReaderV1() throws IOException {
+    for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
+      for (boolean pread : new boolean[] { false, true }) {
+        byte[] block = createTestV1Block(algo);
+        Path path = new Path(TEST_UTIL.getDataTestDir(),
+          "blocks_v1_"+ algo);
+        LOG.info("Creating temporary file at " + path);
+        FSDataOutputStream os = fs.create(path);
+        int totalSize = 0;
+        int numBlocks = 50;
+        for (int i = 0; i < numBlocks; ++i) {
+          os.write(block);
+          totalSize += block.length;
+        }
+        os.close();
+
+        FSDataInputStream is = fs.open(path);
+        HFileBlock.FSReader hbr = new HFileBlock.FSReaderV1(is, algo,
+            totalSize);
+        HFileBlock b;
+        int numBlocksRead = 0;
+        long pos = 0;
+        while (pos < totalSize) {
+          b = hbr.readBlockData(pos, block.length, uncompressedSizeV1, pread);
+          b.sanityCheck();
+          pos += block.length;
+          numBlocksRead++;
+        }
+        assertEquals(numBlocks, numBlocksRead);
+        is.close();
+      }
+    }
+  }
+
+  @Test
+  public void testReaderV2() throws IOException {
+    for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
+      for (boolean pread : new boolean[] { false, true }) {
+          LOG.info("testReaderV2: Compression algorithm: " + algo + 
+                   ", pread=" + pread);
+        Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
+            + algo);
+        FSDataOutputStream os = fs.create(path);
+        Writer hbw = new Writer(algo, null,
+            includesMemstoreTS);
+        long totalSize = 0;
+        for (int blockId = 0; blockId < 2; ++blockId) {
+          DataOutputStream dos = hbw.startWriting(BlockType.DATA);
+          for (int i = 0; i < 1234; ++i)
+            dos.writeInt(i);
+          hbw.writeHeaderAndData(os);
+          totalSize += hbw.getOnDiskSizeWithHeader();
+        }
+        os.close();
+
+        FSDataInputStream is = fs.open(path);
+        HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, is, algo,
+            totalSize, MINOR_VERSION, fs, path);
+        HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
+        is.close();
+
+        b.sanityCheck();
+        assertEquals(4936, b.getUncompressedSizeWithoutHeader());
+        assertEquals(algo == GZ ? 2173 : 4936, 
+                     b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
+        String blockStr = b.toString();
+
+        if (algo == GZ) {
+          is = fs.open(path);
+          hbr = new HFileBlock.FSReaderV2(is, is, algo, totalSize, MINOR_VERSION,
+                                          fs, path);
+          b = hbr.readBlockData(0, 2173 + HFileBlock.HEADER_SIZE_NO_CHECKSUM +
+                                b.totalChecksumBytes(), -1, pread);
+          assertEquals(blockStr, b.toString());
+          int wrongCompressedSize = 2172;
+          try {
+            b = hbr.readBlockData(0, wrongCompressedSize
+                + HFileBlock.HEADER_SIZE_NO_CHECKSUM, -1, pread);
+            fail("Exception expected");
+          } catch (IOException ex) {
+            String expectedPrefix = "On-disk size without header provided is "
+                + wrongCompressedSize + ", but block header contains "
+                + b.getOnDiskSizeWithoutHeader() + ".";
+            assertTrue("Invalid exception message: '" + ex.getMessage()
+                + "'.\nMessage is expected to start with: '" + expectedPrefix
+                + "'", ex.getMessage().startsWith(expectedPrefix));
+          }
+          is.close();
+        }
+      }
+    }
+  }
+
+  /**
+   * Test encoding/decoding data blocks.
+   * @throws IOException a bug or a problem with temporary files.
+   */
+  @Test
+  public void testDataBlockEncoding() throws IOException {
+    final int numBlocks = 5;
+    for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
+      for (boolean pread : new boolean[] { false, true }) {
+        for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
+          LOG.info("testDataBlockEncoding algo " + algo + 
+                   " pread = " + pread +
+                   " encoding " + encoding);
+          Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
+              + algo + "_" + encoding.toString());
+          FSDataOutputStream os = fs.create(path);
+          HFileDataBlockEncoder dataBlockEncoder =
+              new HFileDataBlockEncoderImpl(encoding);
+          Writer hbw = new Writer(algo, dataBlockEncoder,
+              includesMemstoreTS);
+          long totalSize = 0;
+          final List<Integer> encodedSizes = new ArrayList<Integer>();
+          final List<ByteBuffer> encodedBlocks = new ArrayList<ByteBuffer>();
+          for (int blockId = 0; blockId < numBlocks; ++blockId) {
+            DataOutputStream dos = hbw.startWriting(BlockType.DATA);
+            TestHFileBlock.writeEncodedBlock(encoding, dos, encodedSizes, encodedBlocks,
+                blockId, includesMemstoreTS);
+
+            hbw.writeHeaderAndData(os);
+            totalSize += hbw.getOnDiskSizeWithHeader();
+          }
+          os.close();
+
+          FSDataInputStream is = fs.open(path);
+          HFileBlock.FSReaderV2 hbr = new HFileBlock.FSReaderV2(is, is, algo,
+              totalSize, MINOR_VERSION, fs, path);
+          hbr.setDataBlockEncoder(dataBlockEncoder);
+          hbr.setIncludesMemstoreTS(includesMemstoreTS);
+
+          HFileBlock b;
+          int pos = 0;
+          for (int blockId = 0; blockId < numBlocks; ++blockId) {
+            b = hbr.readBlockData(pos, -1, -1, pread);
+            b.sanityCheck();
+            pos += b.getOnDiskSizeWithHeader();
+
+            assertEquals((int) encodedSizes.get(blockId),
+                b.getUncompressedSizeWithoutHeader());
+            ByteBuffer actualBuffer = b.getBufferWithoutHeader();
+            if (encoding != DataBlockEncoding.NONE) {
+              // We expect a two-byte big-endian encoding id.
+              assertEquals(0, actualBuffer.get(0));
+              assertEquals(encoding.getId(), actualBuffer.get(1));
+              actualBuffer.position(2);
+              actualBuffer = actualBuffer.slice();
+            }
+
+            ByteBuffer expectedBuffer = encodedBlocks.get(blockId);
+            expectedBuffer.rewind();
+
+            // test if content matches, produce nice message
+            TestHFileBlock.assertBuffersEqual(expectedBuffer, actualBuffer, algo, encoding,
+                pread);
+          }
+          is.close();
+        }
+      }
+    }
+  }
+
+  @org.junit.Rule
+  public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
+    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
+
+
+  /**
+   * This is the version of the HFileBlock.Writer that is used to 
+   * create V2 blocks with minor version 0. These blocks do not 
+   * have hbase-level checksums. The code is here to test 
+   * backward compatibility. The reason we do not inherit from 
+   * HFileBlock.Writer is because we never ever want to change the code
+   * in this class but the code in HFileBlock.Writer will continually 
+   * evolve.
+   */
+  public static final class Writer {
+
+    // These constants are as they were in minorVersion 0.
+    private static final int HEADER_SIZE = HFileBlock.HEADER_SIZE_NO_CHECKSUM;
+    private static final boolean DONT_FILL_HEADER = HFileBlock.DONT_FILL_HEADER;
+    private static final byte[] DUMMY_HEADER = 
+      HFileBlock.DUMMY_HEADER_NO_CHECKSUM;
+
+    private enum State {
+      INIT,
+      WRITING,
+      BLOCK_READY
+    };
+
+    /** Writer state. Used to ensure the correct usage protocol. */
+    private State state = State.INIT;
+
+    /** Compression algorithm for all blocks this instance writes. */
+    private final Compression.Algorithm compressAlgo;
+
+    /** Data block encoder used for data blocks */
+    private final HFileDataBlockEncoder dataBlockEncoder;
+
+    /**
+     * The stream we use to accumulate data in uncompressed format for each
+     * block. We reset this stream at the end of each block and reuse it. The
+     * header is written as the first {@link #HEADER_SIZE} bytes into this
+     * stream.
+     */
+    private ByteArrayOutputStream baosInMemory;
+
+    /** Compressor, which is also reused between consecutive blocks. */
+    private Compressor compressor;
+
+    /** Compression output stream */
+    private CompressionOutputStream compressionStream;
+
+    /** Underlying stream to write compressed bytes to */
+    private ByteArrayOutputStream compressedByteStream;
+
+    /**
+     * Current block type. Set in {@link #startWriting(BlockType)}. Could be
+     * changed in {@link #encodeDataBlockForDisk()} from {@link BlockType#DATA}
+     * to {@link BlockType#ENCODED_DATA}.
+     */
+    private BlockType blockType;
+
+    /**
+     * A stream that we write uncompressed bytes to, which compresses them and
+     * writes them to {@link #baosInMemory}.
+     */
+    private DataOutputStream userDataStream;
+
+    /**
+     * Bytes to be written to the file system, including the header. Compressed
+     * if compression is turned on.
+     */
+    private byte[] onDiskBytesWithHeader;
+
+    /**
+     * Valid in the READY state. Contains the header and the uncompressed (but
+     * potentially encoded, if this is a data block) bytes, so the length is
+     * {@link #uncompressedSizeWithoutHeader} + {@link HFileBlock#HEADER_SIZE}.
+     */
+    private byte[] uncompressedBytesWithHeader;
+
+    /**
+     * Current block's start offset in the {@link HFile}. Set in
+     * {@link #writeHeaderAndData(FSDataOutputStream)}.
+     */
+    private long startOffset;
+
+    /**
+     * Offset of previous block by block type. Updated when the next block is
+     * started.
+     */
+    private long[] prevOffsetByType;
+
+    /** The offset of the previous block of the same type */
+    private long prevOffset;
+
+    /** Whether we are including memstore timestamp after every key/value */
+    private boolean includesMemstoreTS;
+
+    /**
+     * @param compressionAlgorithm compression algorithm to use
+     * @param dataBlockEncoderAlgo data block encoding algorithm to use
+     */
+    public Writer(Compression.Algorithm compressionAlgorithm,
+          HFileDataBlockEncoder dataBlockEncoder, boolean includesMemstoreTS) {
+      compressAlgo = compressionAlgorithm == null ? NONE : compressionAlgorithm;
+      this.dataBlockEncoder = dataBlockEncoder != null
+          ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
+
+      baosInMemory = new ByteArrayOutputStream();
+      if (compressAlgo != NONE) {
+        compressor = compressionAlgorithm.getCompressor();
+        compressedByteStream = new ByteArrayOutputStream();
+        try {
+          compressionStream =
+              compressionAlgorithm.createPlainCompressionStream(
+                  compressedByteStream, compressor);
+        } catch (IOException e) {
+          throw new RuntimeException("Could not create compression stream " +
+              "for algorithm " + compressionAlgorithm, e);
+        }
+      }
+
+      prevOffsetByType = new long[BlockType.values().length];
+      for (int i = 0; i < prevOffsetByType.length; ++i)
+        prevOffsetByType[i] = -1;
+
+      this.includesMemstoreTS = includesMemstoreTS;
+    }
+
+    /**
+     * Starts writing into the block. The previous block's data is discarded.
+     *
+     * @return the stream the user can write their data into
+     * @throws IOException
+     */
+    public DataOutputStream startWriting(BlockType newBlockType)
+        throws IOException {
+      if (state == State.BLOCK_READY && startOffset != -1) {
+        // We had a previous block that was written to a stream at a specific
+        // offset. Save that offset as the last offset of a block of that type.
+        prevOffsetByType[blockType.getId()] = startOffset;
+      }
+
+      startOffset = -1;
+      blockType = newBlockType;
+
+      baosInMemory.reset();
+      baosInMemory.write(DUMMY_HEADER);
+
+      state = State.WRITING;
+
+      // We will compress it later in finishBlock()
+      userDataStream = new DataOutputStream(baosInMemory);
+      return userDataStream;
+    }
+
+    /**
+     * Returns the stream for the user to write to. The block writer takes care
+     * of handling compression and buffering for caching on write. Can only be
+     * called in the "writing" state.
+     *
+     * @return the data output stream for the user to write to
+     */
+    DataOutputStream getUserDataStream() {
+      expectState(State.WRITING);
+      return userDataStream;
+    }
+
+    /**
+     * Transitions the block writer from the "writing" state to the "block
+     * ready" state.  Does nothing if a block is already finished.
+     */
+    private void ensureBlockReady() throws IOException {
+      Preconditions.checkState(state != State.INIT,
+          "Unexpected state: " + state);
+
+      if (state == State.BLOCK_READY)
+        return;
+
+      // This will set state to BLOCK_READY.
+      finishBlock();
+    }
+
+    /**
+     * An internal method that flushes the compressing stream (if using
+     * compression), serializes the header, and takes care of the separate
+     * uncompressed stream for caching on write, if applicable. Sets block
+     * write state to "block ready".
+     */
+    private void finishBlock() throws IOException {
+      userDataStream.flush();
+
+      // This does an array copy, so it is safe to cache this byte array.
+      uncompressedBytesWithHeader = baosInMemory.toByteArray();
+      LOG.warn("Writer.finishBlock user data size with header before compression " +
+               uncompressedBytesWithHeader.length);
+      prevOffset = prevOffsetByType[blockType.getId()];
+
+      // We need to set state before we can package the block up for
+      // cache-on-write. In a way, the block is ready, but not yet encoded or
+      // compressed.
+      state = State.BLOCK_READY;
+      encodeDataBlockForDisk();
+
+      doCompression();
+      putHeader(uncompressedBytesWithHeader, 0, onDiskBytesWithHeader.length,
+          uncompressedBytesWithHeader.length);
+    }
+
+    /**
+     * Do compression if it is enabled, or re-use the uncompressed buffer if
+     * it is not. Fills in the compressed block's header if doing compression.
+     */
+    private void doCompression() throws IOException {
+      // do the compression
+      if (compressAlgo != NONE) {
+        compressedByteStream.reset();
+        compressedByteStream.write(DUMMY_HEADER);
+
+        compressionStream.resetState();
+
+        compressionStream.write(uncompressedBytesWithHeader, HEADER_SIZE,
+            uncompressedBytesWithHeader.length - HEADER_SIZE);
+
+        compressionStream.flush();
+        compressionStream.finish();
+
+        onDiskBytesWithHeader = compressedByteStream.toByteArray();
+        putHeader(onDiskBytesWithHeader, 0, onDiskBytesWithHeader.length,
+            uncompressedBytesWithHeader.length);
+      } else {
+        onDiskBytesWithHeader = uncompressedBytesWithHeader;
+      }
+    }
+
+    /**
+     * Encodes this block if it is a data block and encoding is turned on in
+     * {@link #dataBlockEncoder}.
+     */
+    private void encodeDataBlockForDisk() throws IOException {
+      if (blockType != BlockType.DATA) {
+        return; // skip any non-data block
+      }
+
+      // do data block encoding, if data block encoder is set
+      ByteBuffer rawKeyValues = ByteBuffer.wrap(uncompressedBytesWithHeader,
+          HEADER_SIZE, uncompressedBytesWithHeader.length -
+          HEADER_SIZE).slice();
+      Pair<ByteBuffer, BlockType> encodingResult =
+          dataBlockEncoder.beforeWriteToDisk(rawKeyValues,
+              includesMemstoreTS, DUMMY_HEADER);
+
+      BlockType encodedBlockType = encodingResult.getSecond();
+      if (encodedBlockType == BlockType.ENCODED_DATA) {
+        uncompressedBytesWithHeader = encodingResult.getFirst().array();
+        blockType = BlockType.ENCODED_DATA;
+      } else {
+        // There is no encoding configured. Do some extra sanity-checking.
+        if (encodedBlockType != BlockType.DATA) {
+          throw new IOException("Unexpected block type coming out of data " +
+              "block encoder: " + encodedBlockType);
+        }
+        if (userDataStream.size() !=
+            uncompressedBytesWithHeader.length - HEADER_SIZE) {
+          throw new IOException("Uncompressed size mismatch: "
+              + userDataStream.size() + " vs. "
+              + (uncompressedBytesWithHeader.length - HEADER_SIZE));
+        }
+      }
+    }
+
+    /**
+     * Put the header into the given byte array at the given offset.
+     * @param onDiskSize size of the block on disk
+     * @param uncompressedSize size of the block after decompression (but
+     *          before optional data block decoding)
+     */
+    private void putHeader(byte[] dest, int offset, int onDiskSize,
+        int uncompressedSize) {
+      offset = blockType.put(dest, offset);
+      offset = Bytes.putInt(dest, offset, onDiskSize - HEADER_SIZE);
+      offset = Bytes.putInt(dest, offset, uncompressedSize - HEADER_SIZE);
+      Bytes.putLong(dest, offset, prevOffset);
+    }
+
+    /**
+     * Similar to {@link #writeHeaderAndData(FSDataOutputStream)}, but records
+     * the offset of this block so that it can be referenced in the next block
+     * of the same type.
+     *
+     * @param out
+     * @throws IOException
+     */
+    public void writeHeaderAndData(FSDataOutputStream out) throws IOException {
+      long offset = out.getPos();
+      if (startOffset != -1 && offset != startOffset) {
+        throw new IOException("A " + blockType + " block written to a "
+            + "stream twice, first at offset " + startOffset + ", then at "
+            + offset);
+      }
+      startOffset = offset;
+
+      writeHeaderAndData((DataOutputStream) out);
+    }
+
+    /**
+     * Writes the header and the compressed data of this block (or uncompressed
+     * data when not using compression) into the given stream. Can be called in
+     * the "writing" state or in the "block ready" state. If called in the
+     * "writing" state, transitions the writer to the "block ready" state.
+     *
+     * @param out the output stream to write the
+     * @throws IOException
+     */
+    private void writeHeaderAndData(DataOutputStream out) throws IOException {
+      ensureBlockReady();
+      out.write(onDiskBytesWithHeader);
+    }
+
+    /**
+     * Returns the header or the compressed data (or uncompressed data when not
+     * using compression) as a byte array. Can be called in the "writing" state
+     * or in the "block ready" state. If called in the "writing" state,
+     * transitions the writer to the "block ready" state.
+     *
+     * @return header and data as they would be stored on disk in a byte array
+     * @throws IOException
+     */
+    public byte[] getHeaderAndData() throws IOException {
+      ensureBlockReady();
+      return onDiskBytesWithHeader;
+    }
+
+    /**
+     * Releases the compressor this writer uses to compress blocks into the
+     * compressor pool. Needs to be called before the writer is discarded.
+     */
+    public void releaseCompressor() {
+      if (compressor != null) {
+        compressAlgo.returnCompressor(compressor);
+        compressor = null;
+      }
+    }
+
+    /**
+     * Returns the on-disk size of the data portion of the block. This is the
+     * compressed size if compression is enabled. Can only be called in the
+     * "block ready" state. Header is not compressed, and its size is not
+     * included in the return value.
+     *
+     * @return the on-disk size of the block, not including the header.
+     */
+    public int getOnDiskSizeWithoutHeader() {
+      expectState(State.BLOCK_READY);
+      return onDiskBytesWithHeader.length - HEADER_SIZE;
+    }
+
+    /**
+     * Returns the on-disk size of the block. Can only be called in the
+     * "block ready" state.
+     *
+     * @return the on-disk size of the block ready to be written, including the
+     *         header size
+     */
+    public int getOnDiskSizeWithHeader() {
+      expectState(State.BLOCK_READY);
+      return onDiskBytesWithHeader.length;
+    }
+
+    /**
+     * The uncompressed size of the block data. Does not include header size.
+     */
+    public int getUncompressedSizeWithoutHeader() {
+      expectState(State.BLOCK_READY);
+      return uncompressedBytesWithHeader.length - HEADER_SIZE;
+    }
+
+    /**
+     * The uncompressed size of the block data, including header size.
+     */
+    public int getUncompressedSizeWithHeader() {
+      expectState(State.BLOCK_READY);
+      return uncompressedBytesWithHeader.length;
+    }
+
+    /** @return true if a block is being written  */
+    public boolean isWriting() {
+      return state == State.WRITING;
+    }
+
+    /**
+     * Returns the number of bytes written into the current block so far, or
+     * zero if not writing the block at the moment. Note that this will return
+     * zero in the "block ready" state as well.
+     *
+     * @return the number of bytes written
+     */
+    public int blockSizeWritten() {
+      if (state != State.WRITING)
+        return 0;
+      return userDataStream.size();
+    }
+
+    /**
+     * Returns the header followed by the uncompressed data, even if using
+     * compression. This is needed for storing uncompressed blocks in the block
+     * cache. Can be called in the "writing" state or the "block ready" state.
+     *
+     * @return uncompressed block bytes for caching on write
+     */
+    private byte[] getUncompressedDataWithHeader() {
+      expectState(State.BLOCK_READY);
+
+      return uncompressedBytesWithHeader;
+    }
+
+    private void expectState(State expectedState) {
+      if (state != expectedState) {
+        throw new IllegalStateException("Expected state: " + expectedState +
+            ", actual state: " + state);
+      }
+    }
+
+    /**
+     * Similar to {@link #getUncompressedBufferWithHeader()} but returns a byte
+     * buffer.
+     *
+     * @return uncompressed block for caching on write in the form of a buffer
+     */
+    public ByteBuffer getUncompressedBufferWithHeader() {
+      byte[] b = getUncompressedDataWithHeader();
+      return ByteBuffer.wrap(b, 0, b.length);
+    }
+
+    /**
+     * Takes the given {@link BlockWritable} instance, creates a new block of
+     * its appropriate type, writes the writable into this block, and flushes
+     * the block into the output stream. The writer is instructed not to buffer
+     * uncompressed bytes for cache-on-write.
+     *
+     * @param bw the block-writable object to write as a block
+     * @param out the file system output stream
+     * @throws IOException
+     */
+    public void writeBlock(BlockWritable bw, FSDataOutputStream out)
+        throws IOException {
+      bw.writeToBlock(startWriting(bw.getBlockType()));
+      writeHeaderAndData(out);
+    }
+
+    /**
+     * Creates a new HFileBlock. 
+     */
+    public HFileBlock getBlockForCaching() {
+      return new HFileBlock(blockType, getOnDiskSizeWithoutHeader(),
+          getUncompressedSizeWithoutHeader(), prevOffset,
+          getUncompressedBufferWithHeader(), DONT_FILL_HEADER, startOffset,
+          includesMemstoreTS, MINOR_VERSION, 0, ChecksumType.NULL.getCode(),
+          getOnDiskSizeWithoutHeader());
+    }
+  }
+}
+

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java Thu Mar  8 22:55:49 2012
@@ -44,6 +44,7 @@ import org.apache.hadoop.fs.FSDataOutput
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexReader;
 import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexChunk;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -110,7 +111,7 @@ public class TestHFileBlockIndex {
     // This test requires at least HFile format version 2.
     conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION);
 
-    fs = FileSystem.get(conf);
+    fs = HFileSystem.get(conf);
   }
 
   @Test
@@ -215,7 +216,8 @@ public class TestHFileBlockIndex {
   private void writeWholeIndex() throws IOException {
     assertEquals(0, keys.size());
     HFileBlock.Writer hbw = new HFileBlock.Writer(compr, null,
-        includesMemstoreTS);
+        includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
+        HFile.DEFAULT_BYTES_PER_CHECKSUM);
     FSDataOutputStream outputStream = fs.create(path);
     HFileBlockIndex.BlockIndexWriter biw =
         new HFileBlockIndex.BlockIndexWriter(hbw, null, null);