You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2013/11/26 05:22:35 UTC

svn commit: r1545536 [3/4] - in /hbase/trunk: hbase-client/src/main/java/org/apache/hadoop/hbase/ hbase-client/src/main/java/org/apache/hadoop/hbase/security/ hbase-client/src/test/java/org/apache/hadoop/hbase/security/ hbase-common/src/main/java/org/a...

Modified: hbase/trunk/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java (original)
+++ hbase/trunk/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java Tue Nov 26 04:22:33 2013
@@ -102,6 +102,16 @@ public final class WALProtos {
      * <code>optional bool has_compression = 1;</code>
      */
     boolean getHasCompression();
+
+    // optional bytes encryption_key = 2;
+    /**
+     * <code>optional bytes encryption_key = 2;</code>
+     */
+    boolean hasEncryptionKey();
+    /**
+     * <code>optional bytes encryption_key = 2;</code>
+     */
+    com.google.protobuf.ByteString getEncryptionKey();
   }
   /**
    * Protobuf type {@code WALHeader}
@@ -159,6 +169,11 @@ public final class WALProtos {
               hasCompression_ = input.readBool();
               break;
             }
+            case 18: {
+              bitField0_ |= 0x00000002;
+              encryptionKey_ = input.readBytes();
+              break;
+            }
           }
         }
       } catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -215,8 +230,25 @@ public final class WALProtos {
       return hasCompression_;
     }
 
+    // optional bytes encryption_key = 2;
+    public static final int ENCRYPTION_KEY_FIELD_NUMBER = 2;
+    private com.google.protobuf.ByteString encryptionKey_;
+    /**
+     * <code>optional bytes encryption_key = 2;</code>
+     */
+    public boolean hasEncryptionKey() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    /**
+     * <code>optional bytes encryption_key = 2;</code>
+     */
+    public com.google.protobuf.ByteString getEncryptionKey() {
+      return encryptionKey_;
+    }
+
     private void initFields() {
       hasCompression_ = false;
+      encryptionKey_ = com.google.protobuf.ByteString.EMPTY;
     }
     private byte memoizedIsInitialized = -1;
     public final boolean isInitialized() {
@@ -233,6 +265,9 @@ public final class WALProtos {
       if (((bitField0_ & 0x00000001) == 0x00000001)) {
         output.writeBool(1, hasCompression_);
       }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, encryptionKey_);
+      }
       getUnknownFields().writeTo(output);
     }
 
@@ -246,6 +281,10 @@ public final class WALProtos {
         size += com.google.protobuf.CodedOutputStream
           .computeBoolSize(1, hasCompression_);
       }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, encryptionKey_);
+      }
       size += getUnknownFields().getSerializedSize();
       memoizedSerializedSize = size;
       return size;
@@ -274,6 +313,11 @@ public final class WALProtos {
         result = result && (getHasCompression()
             == other.getHasCompression());
       }
+      result = result && (hasEncryptionKey() == other.hasEncryptionKey());
+      if (hasEncryptionKey()) {
+        result = result && getEncryptionKey()
+            .equals(other.getEncryptionKey());
+      }
       result = result &&
           getUnknownFields().equals(other.getUnknownFields());
       return result;
@@ -291,6 +335,10 @@ public final class WALProtos {
         hash = (37 * hash) + HAS_COMPRESSION_FIELD_NUMBER;
         hash = (53 * hash) + hashBoolean(getHasCompression());
       }
+      if (hasEncryptionKey()) {
+        hash = (37 * hash) + ENCRYPTION_KEY_FIELD_NUMBER;
+        hash = (53 * hash) + getEncryptionKey().hashCode();
+      }
       hash = (29 * hash) + getUnknownFields().hashCode();
       memoizedHashCode = hash;
       return hash;
@@ -402,6 +450,8 @@ public final class WALProtos {
         super.clear();
         hasCompression_ = false;
         bitField0_ = (bitField0_ & ~0x00000001);
+        encryptionKey_ = com.google.protobuf.ByteString.EMPTY;
+        bitField0_ = (bitField0_ & ~0x00000002);
         return this;
       }
 
@@ -434,6 +484,10 @@ public final class WALProtos {
           to_bitField0_ |= 0x00000001;
         }
         result.hasCompression_ = hasCompression_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.encryptionKey_ = encryptionKey_;
         result.bitField0_ = to_bitField0_;
         onBuilt();
         return result;
@@ -453,6 +507,9 @@ public final class WALProtos {
         if (other.hasHasCompression()) {
           setHasCompression(other.getHasCompression());
         }
+        if (other.hasEncryptionKey()) {
+          setEncryptionKey(other.getEncryptionKey());
+        }
         this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
@@ -513,6 +570,42 @@ public final class WALProtos {
         return this;
       }
 
+      // optional bytes encryption_key = 2;
+      private com.google.protobuf.ByteString encryptionKey_ = com.google.protobuf.ByteString.EMPTY;
+      /**
+       * <code>optional bytes encryption_key = 2;</code>
+       */
+      public boolean hasEncryptionKey() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      /**
+       * <code>optional bytes encryption_key = 2;</code>
+       */
+      public com.google.protobuf.ByteString getEncryptionKey() {
+        return encryptionKey_;
+      }
+      /**
+       * <code>optional bytes encryption_key = 2;</code>
+       */
+      public Builder setEncryptionKey(com.google.protobuf.ByteString value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000002;
+        encryptionKey_ = value;
+        onChanged();
+        return this;
+      }
+      /**
+       * <code>optional bytes encryption_key = 2;</code>
+       */
+      public Builder clearEncryptionKey() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        encryptionKey_ = getDefaultInstance().getEncryptionKey();
+        onChanged();
+        return this;
+      }
+
       // @@protoc_insertion_point(builder_scope:WALHeader)
     }
 
@@ -4991,24 +5084,25 @@ public final class WALProtos {
       descriptor;
   static {
     java.lang.String[] descriptorData = {
-      "\n\tWAL.proto\032\013HBase.proto\"$\n\tWALHeader\022\027\n" +
-      "\017has_compression\030\001 \001(\010\"\202\002\n\006WALKey\022\033\n\023enc" +
-      "oded_region_name\030\001 \002(\014\022\022\n\ntable_name\030\002 \002" +
-      "(\014\022\033\n\023log_sequence_number\030\003 \002(\004\022\022\n\nwrite" +
-      "_time\030\004 \002(\004\022\035\n\ncluster_id\030\005 \001(\0132\005.UUIDB\002" +
-      "\030\001\022\034\n\006scopes\030\006 \003(\0132\014.FamilyScope\022\032\n\022foll" +
-      "owing_kv_count\030\007 \001(\r\022\032\n\013cluster_ids\030\010 \003(" +
-      "\0132\005.UUID\022\022\n\nnonceGroup\030\t \001(\004\022\r\n\005nonce\030\n " +
-      "\001(\004\"=\n\013FamilyScope\022\016\n\006family\030\001 \002(\014\022\036\n\nsc" +
-      "ope_type\030\002 \002(\0162\n.ScopeType\"\251\001\n\024Compactio",
-      "nDescriptor\022\022\n\ntable_name\030\001 \002(\014\022\033\n\023encod" +
-      "ed_region_name\030\002 \002(\014\022\023\n\013family_name\030\003 \002(" +
-      "\014\022\030\n\020compaction_input\030\004 \003(\t\022\031\n\021compactio" +
-      "n_output\030\005 \003(\t\022\026\n\016store_home_dir\030\006 \002(\t\"\014" +
-      "\n\nWALTrailer*F\n\tScopeType\022\033\n\027REPLICATION" +
-      "_SCOPE_LOCAL\020\000\022\034\n\030REPLICATION_SCOPE_GLOB" +
-      "AL\020\001B?\n*org.apache.hadoop.hbase.protobuf" +
-      ".generatedB\tWALProtosH\001\210\001\000\240\001\001"
+      "\n\tWAL.proto\032\013HBase.proto\"<\n\tWALHeader\022\027\n" +
+      "\017has_compression\030\001 \001(\010\022\026\n\016encryption_key" +
+      "\030\002 \001(\014\"\202\002\n\006WALKey\022\033\n\023encoded_region_name" +
+      "\030\001 \002(\014\022\022\n\ntable_name\030\002 \002(\014\022\033\n\023log_sequen" +
+      "ce_number\030\003 \002(\004\022\022\n\nwrite_time\030\004 \002(\004\022\035\n\nc" +
+      "luster_id\030\005 \001(\0132\005.UUIDB\002\030\001\022\034\n\006scopes\030\006 \003" +
+      "(\0132\014.FamilyScope\022\032\n\022following_kv_count\030\007" +
+      " \001(\r\022\032\n\013cluster_ids\030\010 \003(\0132\005.UUID\022\022\n\nnonc" +
+      "eGroup\030\t \001(\004\022\r\n\005nonce\030\n \001(\004\"=\n\013FamilySco" +
+      "pe\022\016\n\006family\030\001 \002(\014\022\036\n\nscope_type\030\002 \002(\0162\n",
+      ".ScopeType\"\251\001\n\024CompactionDescriptor\022\022\n\nt" +
+      "able_name\030\001 \002(\014\022\033\n\023encoded_region_name\030\002" +
+      " \002(\014\022\023\n\013family_name\030\003 \002(\014\022\030\n\020compaction_" +
+      "input\030\004 \003(\t\022\031\n\021compaction_output\030\005 \003(\t\022\026" +
+      "\n\016store_home_dir\030\006 \002(\t\"\014\n\nWALTrailer*F\n\t" +
+      "ScopeType\022\033\n\027REPLICATION_SCOPE_LOCAL\020\000\022\034" +
+      "\n\030REPLICATION_SCOPE_GLOBAL\020\001B?\n*org.apac" +
+      "he.hadoop.hbase.protobuf.generatedB\tWALP" +
+      "rotosH\001\210\001\000\240\001\001"
     };
     com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
       new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -5020,7 +5114,7 @@ public final class WALProtos {
           internal_static_WALHeader_fieldAccessorTable = new
             com.google.protobuf.GeneratedMessage.FieldAccessorTable(
               internal_static_WALHeader_descriptor,
-              new java.lang.String[] { "HasCompression", });
+              new java.lang.String[] { "HasCompression", "EncryptionKey", });
           internal_static_WALKey_descriptor =
             getDescriptor().getMessageTypes().get(1);
           internal_static_WALKey_fieldAccessorTable = new

Added: hbase/trunk/hbase-protocol/src/main/protobuf/Encryption.proto
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-protocol/src/main/protobuf/Encryption.proto?rev=1545536&view=auto
==============================================================================
--- hbase/trunk/hbase-protocol/src/main/protobuf/Encryption.proto (added)
+++ hbase/trunk/hbase-protocol/src/main/protobuf/Encryption.proto Tue Nov 26 04:22:33 2013
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers used for encryption
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "EncryptionProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+message WrappedKey {
+  required string algorithm = 1;
+  required uint32 length = 2;
+  required bytes data = 3;
+  optional bytes iv = 4;
+  optional fixed32 crc = 5;
+}

Modified: hbase/trunk/hbase-protocol/src/main/protobuf/HFile.proto
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-protocol/src/main/protobuf/HFile.proto?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-protocol/src/main/protobuf/HFile.proto (original)
+++ hbase/trunk/hbase-protocol/src/main/protobuf/HFile.proto Tue Nov 26 04:22:33 2013
@@ -42,4 +42,5 @@ message FileTrailerProto {
   optional uint64 last_data_block_offset = 10;
   optional string comparator_class_name = 11;
   optional uint32 compression_codec = 12;
+  optional bytes encryption_key = 13;
 }

Modified: hbase/trunk/hbase-protocol/src/main/protobuf/WAL.proto
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-protocol/src/main/protobuf/WAL.proto?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-protocol/src/main/protobuf/WAL.proto (original)
+++ hbase/trunk/hbase-protocol/src/main/protobuf/WAL.proto Tue Nov 26 04:22:33 2013
@@ -25,6 +25,7 @@ import "HBase.proto";
 
 message WALHeader {
   optional bool has_compression = 1;
+  optional bytes encryption_key = 2;
 }
 
 // Protocol buffer version of HLogKey; see HLogKey comment, not really a key but WALEdit header for some KVs

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java Tue Nov 26 04:22:33 2013
@@ -24,6 +24,7 @@ import java.nio.ByteBuffer;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HConstants;
@@ -65,11 +66,13 @@ public class HalfStoreFileReader extends
    * @param p path to hfile
    * @param cacheConf
    * @param r original reference file (contains top or bottom)
+   * @param conf Configuration
    * @throws IOException
    */
   public HalfStoreFileReader(final FileSystem fs, final Path p,
-      final CacheConfig cacheConf, final Reference r) throws IOException {
-    super(fs, p, cacheConf);
+      final CacheConfig cacheConf, final Reference r, final Configuration conf)
+      throws IOException {
+    super(fs, p, cacheConf, conf);
     // This is not actual midkey for this half-file; its just border
     // around which we split top and bottom.  Have to look in files to find
     // actual last and first keys for bottom and top halves.  Half-files don't
@@ -88,11 +91,13 @@ public class HalfStoreFileReader extends
    * @param size Full size of the hfile file
    * @param cacheConf
    * @param r original reference file (contains top or bottom)
+   * @param conf Configuration
    * @throws IOException
    */
   public HalfStoreFileReader(final FileSystem fs, final Path p, final FSDataInputStreamWrapper in,
-      long size, final CacheConfig cacheConf,  final Reference r) throws IOException {
-    super(fs, p, in, size, cacheConf);
+      long size, final CacheConfig cacheConf,  final Reference r, final Configuration conf)
+      throws IOException {
+    super(fs, p, in, size, cacheConf, conf);
     // This is not actual midkey for this half-file; its just border
     // around which we split top and bottom.  Have to look in files to find
     // actual last and first keys for bottom and top halves.  Half-files don't

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java Tue Nov 26 04:22:33 2013
@@ -22,6 +22,8 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.KeyValue;
@@ -35,7 +37,8 @@ import org.apache.hadoop.hbase.io.hfile.
  * Common functionality needed by all versions of {@link HFile} readers.
  */
 @InterfaceAudience.Private
-public abstract class AbstractHFileReader implements HFile.Reader {
+public abstract class AbstractHFileReader
+    implements HFile.Reader, Configurable {
   /** Stream to read from. Does checksum verifications in file system */
   protected FSDataInputStream istream;
 
@@ -71,7 +74,7 @@ public abstract class AbstractHFileReade
   protected int avgValueLen = -1;
 
   /** Key comparator */
-  protected KVComparator comparator;
+  protected KVComparator comparator = new KVComparator();
 
   /** Size of this file. */
   protected final long fileSize;
@@ -90,8 +93,11 @@ public abstract class AbstractHFileReade
   /** The filesystem used for accesing data */
   protected HFileSystem hfs;
 
+  protected Configuration conf;
+
   protected AbstractHFileReader(Path path, FixedFileTrailer trailer,
-      final long fileSize, final CacheConfig cacheConf, final HFileSystem hfs) {
+      final long fileSize, final CacheConfig cacheConf, final HFileSystem hfs,
+      final Configuration conf) {
     this.trailer = trailer;
     this.compressAlgo = trailer.getCompressionCodec();
     this.cacheConf = cacheConf;
@@ -99,6 +105,7 @@ public abstract class AbstractHFileReade
     this.path = path;
     this.name = path.getName();
     this.hfs = hfs;
+    this.conf = conf;
   }
 
   @SuppressWarnings("serial")
@@ -330,4 +337,14 @@ public abstract class AbstractHFileReade
   }
 
   public abstract int getMajorVersion();
+
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = conf;
+  }
 }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java Tue Nov 26 04:22:33 2013
@@ -26,8 +26,6 @@ import java.io.DataOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.hbase.KeyValue;
@@ -36,6 +34,8 @@ import org.apache.hadoop.hbase.io.compre
 import org.apache.hadoop.hbase.protobuf.generated.HFileProtos;
 import org.apache.hadoop.hbase.util.Bytes;
 
+import com.google.protobuf.ZeroCopyLiteralByteString;
+
 /**
  * The {@link HFile} has a fixed trailer which contains offsets to other
  * variable parts of the file. Also includes basic metadata on this file. The
@@ -110,6 +110,9 @@ public class FixedFileTrailer {
   /** Raw key comparator class name in version 3 */
   private String comparatorClassName = KeyValue.COMPARATOR.getLegacyKeyComparatorName();
 
+  /** The encryption key */
+  private byte[] encryptionKey;
+
   /** The {@link HFile} format major version. */
   private final int majorVersion;
 
@@ -127,10 +130,10 @@ public class FixedFileTrailer {
     // We support only 2 major versions now. ie. V2, V3
     versionToSize[2] = 212;
     for (int version = 3; version <= HFile.MAX_FORMAT_VERSION; version++) {
-      // Max FFT size for V3 and above is taken as 1KB for future enhancements
+      // Max FFT size for V3 and above is taken as 4KB for future enhancements
       // if any.
-      // Unless the trailer size exceeds 1024 this can continue
-      versionToSize[version] = 1024;
+      // Unless the trailer size exceeds 4K this can continue
+      versionToSize[version] = 1024 * 4;
     }
     return versionToSize;
   }
@@ -187,7 +190,7 @@ public class FixedFileTrailer {
    */
   void serializeAsPB(DataOutputStream output) throws IOException {
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
-    HFileProtos.FileTrailerProto.newBuilder()
+    HFileProtos.FileTrailerProto.Builder builder = HFileProtos.FileTrailerProto.newBuilder()
       .setFileInfoOffset(fileInfoOffset)
       .setLoadOnOpenDataOffset(loadOnOpenDataOffset)
       .setUncompressedDataIndexSize(uncompressedDataIndexSize)
@@ -201,8 +204,11 @@ public class FixedFileTrailer {
       // TODO this is a classname encoded into an  HFile's trailer. We are going to need to have 
       // some compat code here.
       .setComparatorClassName(comparatorClassName)
-      .setCompressionCodec(compressionCodec.ordinal())
-      .build().writeDelimitedTo(baos);
+      .setCompressionCodec(compressionCodec.ordinal());
+    if (encryptionKey != null) {
+      builder.setEncryptionKey(ZeroCopyLiteralByteString.wrap(encryptionKey));
+    }
+    builder.build().writeDelimitedTo(baos);
     output.write(baos.toByteArray());
     // Pad to make up the difference between variable PB encoding length and the
     // length when encoded as writable under earlier V2 formats. Failure to pad
@@ -251,52 +257,55 @@ public class FixedFileTrailer {
   void deserializeFromPB(DataInputStream inputStream) throws IOException {
     // read PB and skip padding
     int start = inputStream.available();
-    HFileProtos.FileTrailerProto.Builder builder = HFileProtos.FileTrailerProto.newBuilder();
-    builder.mergeDelimitedFrom(inputStream);
+    HFileProtos.FileTrailerProto trailerProto =
+        HFileProtos.FileTrailerProto.PARSER.parseDelimitedFrom(inputStream);
     int size = start - inputStream.available();
     inputStream.skip(getTrailerSize() - NOT_PB_SIZE - size);
 
     // process the PB
-    if (builder.hasFileInfoOffset()) {
-      fileInfoOffset = builder.getFileInfoOffset();
+    if (trailerProto.hasFileInfoOffset()) {
+      fileInfoOffset = trailerProto.getFileInfoOffset();
     }
-    if (builder.hasLoadOnOpenDataOffset()) {
-      loadOnOpenDataOffset = builder.getLoadOnOpenDataOffset();
+    if (trailerProto.hasLoadOnOpenDataOffset()) {
+      loadOnOpenDataOffset = trailerProto.getLoadOnOpenDataOffset();
     }
-    if (builder.hasUncompressedDataIndexSize()) {
-      uncompressedDataIndexSize = builder.getUncompressedDataIndexSize();
+    if (trailerProto.hasUncompressedDataIndexSize()) {
+      uncompressedDataIndexSize = trailerProto.getUncompressedDataIndexSize();
     }
-    if (builder.hasTotalUncompressedBytes()) {
-      totalUncompressedBytes = builder.getTotalUncompressedBytes();
+    if (trailerProto.hasTotalUncompressedBytes()) {
+      totalUncompressedBytes = trailerProto.getTotalUncompressedBytes();
     }
-    if (builder.hasDataIndexCount()) {
-      dataIndexCount = builder.getDataIndexCount();
+    if (trailerProto.hasDataIndexCount()) {
+      dataIndexCount = trailerProto.getDataIndexCount();
     }
-    if (builder.hasMetaIndexCount()) {
-      metaIndexCount = builder.getMetaIndexCount();
+    if (trailerProto.hasMetaIndexCount()) {
+      metaIndexCount = trailerProto.getMetaIndexCount();
     }
-    if (builder.hasEntryCount()) {
-      entryCount = builder.getEntryCount();
+    if (trailerProto.hasEntryCount()) {
+      entryCount = trailerProto.getEntryCount();
     }
-    if (builder.hasNumDataIndexLevels()) {
-      numDataIndexLevels = builder.getNumDataIndexLevels();
+    if (trailerProto.hasNumDataIndexLevels()) {
+      numDataIndexLevels = trailerProto.getNumDataIndexLevels();
     }
-    if (builder.hasFirstDataBlockOffset()) {
-      firstDataBlockOffset = builder.getFirstDataBlockOffset();
+    if (trailerProto.hasFirstDataBlockOffset()) {
+      firstDataBlockOffset = trailerProto.getFirstDataBlockOffset();
     }
-    if (builder.hasLastDataBlockOffset()) {
-      lastDataBlockOffset = builder.getLastDataBlockOffset();
+    if (trailerProto.hasLastDataBlockOffset()) {
+      lastDataBlockOffset = trailerProto.getLastDataBlockOffset();
     }
-    if (builder.hasComparatorClassName()) {
+    if (trailerProto.hasComparatorClassName()) {
       // TODO this is a classname encoded into an  HFile's trailer. We are going to need to have 
       // some compat code here.
-      setComparatorClass(getComparatorClass(builder.getComparatorClassName()));
+      setComparatorClass(getComparatorClass(trailerProto.getComparatorClassName()));
     }
-    if (builder.hasCompressionCodec()) {
-      compressionCodec = Compression.Algorithm.values()[builder.getCompressionCodec()];
+    if (trailerProto.hasCompressionCodec()) {
+      compressionCodec = Compression.Algorithm.values()[trailerProto.getCompressionCodec()];
     } else {
       compressionCodec = Compression.Algorithm.NONE;
     }
+    if (trailerProto.hasEncryptionKey()) {
+      encryptionKey = trailerProto.getEncryptionKey().toByteArray();
+    }
   }
 
   /**
@@ -344,6 +353,9 @@ public class FixedFileTrailer {
     append(sb, "firstDataBlockOffset=" + firstDataBlockOffset);
     append(sb, "lastDataBlockOffset=" + lastDataBlockOffset);
     append(sb, "comparatorClassName=" + comparatorClassName);
+    if (majorVersion >= 3) {
+      append(sb, "encryptionKey=" + (encryptionKey != null ? "PRESENT" : "NONE"));
+    }
     append(sb, "majorVersion=" + majorVersion);
     append(sb, "minorVersion=" + minorVersion);
 
@@ -597,6 +609,15 @@ public class FixedFileTrailer {
     this.uncompressedDataIndexSize = uncompressedDataIndexSize;
   }
 
+  public byte[] getEncryptionKey() {
+    expectAtLeastMajorVersion(3);
+    return encryptionKey;
+  }
+
+  public void setEncryptionKey(byte[] keyBytes) {
+    this.encryptionKey = keyBytes;
+  }
+
   /**
    * Extracts the major version for a 4-byte serialized version data.
    * The major version is the 3 least significant bytes

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java Tue Nov 26 04:22:33 2013
@@ -320,6 +320,11 @@ public class HFile {
      * HFile V2.
      */
     void addDeleteFamilyBloomFilter(BloomFilterWriter bfw) throws IOException;
+
+    /**
+     * Return the file context for the HFile this writer belongs to
+     */
+    HFileContext getFileContext();
   }
 
   /**
@@ -499,6 +504,11 @@ public class HFile {
     DataBlockEncoding getDataBlockEncoding();
 
     boolean hasMVCCInfo();
+
+    /**
+     * Return the file context of the HFile this reader belongs to
+     */
+    HFileContext getFileContext();
   }
 
   /**
@@ -514,8 +524,7 @@ public class HFile {
    * @throws IOException If file is invalid, will throw CorruptHFileException flavored IOException
    */
   private static Reader pickReaderVersion(Path path, FSDataInputStreamWrapper fsdis,
-      long size, CacheConfig cacheConf,
-      HFileSystem hfs) throws IOException {
+      long size, CacheConfig cacheConf, HFileSystem hfs, Configuration conf) throws IOException {
     FixedFileTrailer trailer = null;
     try {
       boolean isHBaseChecksum = fsdis.shouldUseHBaseChecksum();
@@ -523,11 +532,9 @@ public class HFile {
       trailer = FixedFileTrailer.readFromStream(fsdis.getStream(isHBaseChecksum), size);
       switch (trailer.getMajorVersion()) {
       case 2:
-        return new HFileReaderV2(
-            path, trailer, fsdis, size, cacheConf, hfs);
+        return new HFileReaderV2(path, trailer, fsdis, size, cacheConf, hfs, conf);
       case 3 :
-        return new HFileReaderV3(
-            path, trailer, fsdis, size, cacheConf, hfs);
+        return new HFileReaderV3(path, trailer, fsdis, size, cacheConf, hfs, conf);
       default:
         throw new IllegalArgumentException("Invalid HFile version " + trailer.getMajorVersion());
       }
@@ -547,11 +554,13 @@ public class HFile {
    * @param fsdis a stream of path's file
    * @param size max size of the trailer.
    * @param cacheConf Cache configuration for hfile's contents
+   * @param conf Configuration
    * @return A version specific Hfile Reader
    * @throws IOException If file is invalid, will throw CorruptHFileException flavored IOException
    */
   public static Reader createReader(FileSystem fs, Path path,
-      FSDataInputStreamWrapper fsdis, long size, CacheConfig cacheConf) throws IOException {
+      FSDataInputStreamWrapper fsdis, long size, CacheConfig cacheConf, Configuration conf)
+      throws IOException {
     HFileSystem hfs = null;
 
     // If the fs is not an instance of HFileSystem, then create an
@@ -563,7 +572,7 @@ public class HFile {
     } else {
       hfs = (HFileSystem)fs;
     }
-    return pickReaderVersion(path, fsdis, size, cacheConf, hfs);
+    return pickReaderVersion(path, fsdis, size, cacheConf, hfs, conf);
   }
 
   /**
@@ -575,21 +584,21 @@ public class HFile {
    * @throws IOException Will throw a CorruptHFileException (DoNotRetryIOException subtype) if hfile is corrupt/invalid.
    */
   public static Reader createReader(
-      FileSystem fs, Path path, CacheConfig cacheConf) throws IOException {
+      FileSystem fs, Path path, CacheConfig cacheConf, Configuration conf) throws IOException {
     Preconditions.checkNotNull(cacheConf, "Cannot create Reader with null CacheConf");
     FSDataInputStreamWrapper stream = new FSDataInputStreamWrapper(fs, path);
     return pickReaderVersion(path, stream, fs.getFileStatus(path).getLen(),
-      cacheConf, stream.getHfs());
+      cacheConf, stream.getHfs(), conf);
   }
 
   /**
    * This factory method is used only by unit tests
    */
   static Reader createReaderFromStream(Path path,
-      FSDataInputStream fsdis, long size, CacheConfig cacheConf)
+      FSDataInputStream fsdis, long size, CacheConfig cacheConf, Configuration conf)
       throws IOException {
     FSDataInputStreamWrapper wrapper = new FSDataInputStreamWrapper(fsdis);
-    return pickReaderVersion(path, wrapper, size, cacheConf, null);
+    return pickReaderVersion(path, wrapper, size, cacheConf, null, conf);
   }
 
   /**

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java Tue Nov 26 04:22:33 2013
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.fs.HFileS
 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
 import org.apache.hadoop.hbase.io.compress.Compression;
 import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
+import org.apache.hadoop.hbase.io.crypto.Encryption;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
 import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultDecodingContext;
@@ -1491,11 +1492,17 @@ public class HFileBlock implements Cache
               - hdrSize, true, offset + hdrSize, pread);
         onDiskSizeWithHeader = b.onDiskSizeWithoutHeader + hdrSize;
       }
+
       Algorithm compressAlgo = fileContext.getCompression();
       boolean isCompressed =
         compressAlgo != null
             && compressAlgo != Compression.Algorithm.NONE;
-      if (!isCompressed) {
+
+      Encryption.Context cryptoContext = fileContext.getEncryptionContext();
+      boolean isEncrypted = cryptoContext != null
+          && cryptoContext != Encryption.Context.NONE;
+
+      if (!isCompressed && !isEncrypted) {
         b.assumeUncompressed();
       }
 
@@ -1504,7 +1511,7 @@ public class HFileBlock implements Cache
         return null;             // checksum mismatch
       }
 
-      if (isCompressed) {
+      if (isCompressed || isEncrypted) {
         // This will allocate a new buffer but keep header bytes.
         b.allocateBuffer(nextBlockOnDiskSize > 0);
         if (b.blockType == BlockType.ENCODED_DATA) {

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java Tue Nov 26 04:22:33 2013
@@ -214,7 +214,7 @@ public class HFilePrettyPrinter {
       System.err.println("ERROR, file doesnt exist: " + file);
     }
 
-    HFile.Reader reader = HFile.createReader(fs, file, new CacheConfig(conf));
+    HFile.Reader reader = HFile.createReader(fs, file, new CacheConfig(conf), conf);
 
     Map<byte[], byte[]> fileInfo = reader.loadFileInfo();
 

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java Tue Nov 26 04:22:33 2013
@@ -26,6 +26,7 @@ import java.util.List;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
@@ -98,6 +99,7 @@ public class HFileReaderV2 extends Abstr
 
   /** Minor versions starting with this number have faked index key */
   static final int MINOR_VERSION_WITH_FAKED_KEY = 3;
+
   protected HFileContext hfileContext;
 
   /**
@@ -110,15 +112,16 @@ public class HFileReaderV2 extends Abstr
    * @param size Length of the stream.
    * @param cacheConf Cache configuration.
    * @param hfs
+   * @param conf
    */
   public HFileReaderV2(Path path, FixedFileTrailer trailer,
       final FSDataInputStreamWrapper fsdis, final long size, final CacheConfig cacheConf,
-      final HFileSystem hfs)
-      throws IOException {
-    super(path, trailer, size, cacheConf, hfs);
+      final HFileSystem hfs, final Configuration conf) throws IOException {
+    super(path, trailer, size, cacheConf, hfs, conf);
+    this.conf = conf;
     trailer.expectMajorVersion(getMajorVersion());
     validateMinorVersion(path, trailer.getMinorVersion());
-    this.hfileContext = createHFileContext(trailer);
+    this.hfileContext = createHFileContext(fsdis, fileSize, hfs, path, trailer);
     HFileBlock.FSReaderV2 fsBlockReaderV2 = new HFileBlock.FSReaderV2(fsdis, fileSize, hfs, path,
         hfileContext);
     this.fsBlockReader = fsBlockReaderV2; // upcast
@@ -174,13 +177,13 @@ public class HFileReaderV2 extends Abstr
     }
   }
 
-  protected HFileContext createHFileContext(FixedFileTrailer trailer) {
-    HFileContext hFileContext  = new HFileContextBuilder()
-                                 .withIncludesMvcc(this.includesMemstoreTS)
-                                 .withCompression(this.compressAlgo)
-                                 .withHBaseCheckSum(trailer.getMinorVersion() >= MINOR_VERSION_WITH_CHECKSUM)
-                                 .build();
-    return hFileContext;
+  protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
+      HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
+    return new HFileContextBuilder()
+      .withIncludesMvcc(this.includesMemstoreTS)
+      .withCompression(this.compressAlgo)
+      .withHBaseCheckSum(trailer.getMinorVersion() >= MINOR_VERSION_WITH_CHECKSUM)
+      .build();
   }
 
   /**
@@ -1185,4 +1188,9 @@ public class HFileReaderV2 extends Abstr
   public int getMajorVersion() {
     return 2;
   }
+
+  @Override
+  public HFileContext getFileContext() {
+    return hfileContext;
+  }
 }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV3.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV3.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV3.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV3.java Tue Nov 26 04:22:33 2013
@@ -18,25 +18,36 @@
 package org.apache.hadoop.hbase.io.hfile;
 
 import java.io.IOException;
+import java.security.Key;
+import java.security.KeyException;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
+import org.apache.hadoop.hbase.io.crypto.Cipher;
+import org.apache.hadoop.hbase.io.crypto.Encryption;
 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
+import org.apache.hadoop.hbase.security.EncryptionUtil;
+import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.ByteBufferUtils;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.WritableUtils;
 
 /**
  * {@link HFile} reader for version 3.
- * This Reader is aware of Tags.
  */
 @InterfaceAudience.Private
 public class HFileReaderV3 extends HFileReaderV2 {
 
+  private static final Log LOG = LogFactory.getLog(HFileReaderV3.class);
+
   public static final int MAX_MINOR_VERSION = 0;
+
   /**
    * Opens a HFile. You must load the index before you can use it by calling
    * {@link #loadFileInfo()}.
@@ -50,11 +61,15 @@ public class HFileReaderV3 extends HFile
    *          Length of the stream.
    * @param cacheConf
    *          Cache configuration.
+   * @param hfs
+   *          The file system.
+   * @param conf
+   *          Configuration
    */
   public HFileReaderV3(Path path, FixedFileTrailer trailer, final FSDataInputStreamWrapper fsdis,
-      final long size, final CacheConfig cacheConf,
-      final HFileSystem hfs) throws IOException {
-    super(path, trailer, fsdis, size, cacheConf, hfs);
+      final long size, final CacheConfig cacheConf, final HFileSystem hfs,
+      final Configuration conf) throws IOException {
+    super(path, trailer, fsdis, size, cacheConf, hfs, conf);
     byte[] tmp = fileInfo.get(FileInfo.MAX_TAGS_LEN);
     // max tag length is not present in the HFile means tags were not at all written to file.
     if (tmp != null) {
@@ -67,13 +82,62 @@ public class HFileReaderV3 extends HFile
   }
 
   @Override
-  protected HFileContext createHFileContext(FixedFileTrailer trailer) {
-    HFileContext hfileContext = new HFileContextBuilder()
-                                .withIncludesMvcc(this.includesMemstoreTS)
-                                .withHBaseCheckSum(true)
-                                .withCompression(this.compressAlgo)
-                                .build();
-    return hfileContext;
+  protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
+      HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
+    trailer.expectMajorVersion(3);
+    HFileContextBuilder builder = new HFileContextBuilder()
+      .withIncludesMvcc(this.includesMemstoreTS)
+      .withHBaseCheckSum(true)
+      .withCompression(this.compressAlgo);
+
+    // Check for any key material available
+    byte[] keyBytes = trailer.getEncryptionKey();
+    if (keyBytes != null) {
+      Encryption.Context cryptoContext = Encryption.newContext(conf);
+      Key key;
+      String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
+        User.getCurrent().getShortName());
+      try {
+        // First try the master key
+        key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes);
+      } catch (KeyException e) {
+        // If the current master key fails to unwrap, try the alternate, if
+        // one is configured
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'");
+        }
+        String alternateKeyName =
+          conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY);
+        if (alternateKeyName != null) {
+          try {
+            key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes);
+          } catch (KeyException ex) {
+            throw new IOException(ex);
+          }
+        } else {
+          throw new IOException(e);
+        }
+      }
+      // Use the algorithm the key wants
+      Cipher cipher = Encryption.getCipher(conf, key.getAlgorithm());
+      if (cipher == null) {
+        throw new IOException("Cipher '" + key.getAlgorithm() + "' is not available");
+      }
+      cryptoContext.setCipher(cipher);
+      cryptoContext.setKey(key);
+      builder.withEncryptionContext(cryptoContext);
+    }
+
+    HFileContext context = builder.build();
+
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("Reader" + (path != null ? " for " + path : "" ) +
+        " initialized with cacheConf: " + cacheConf +
+        " comparator: " + comparator.getClass().getSimpleName() +
+        " fileContext: " + context);
+    }
+
+    return context;
   }
 
   /**
@@ -140,19 +204,24 @@ public class HFileReaderV3 extends HFile
       blockBuffer.mark();
       currKeyLen = blockBuffer.getInt();
       currValueLen = blockBuffer.getInt();
+      if (currKeyLen < 0 || currValueLen < 0 || currKeyLen > blockBuffer.limit()
+          || currValueLen > blockBuffer.limit()) {
+        throw new IllegalStateException("Invalid currKeyLen " + currKeyLen + " or currValueLen "
+            + currValueLen + ". Block offset: "
+            + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
+            + blockBuffer.position() + " (without header).");
+      }
       ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
       if (reader.hfileContext.isIncludesTags()) {
         currTagsLen = blockBuffer.getShort();
+        if (currTagsLen < 0 || currTagsLen > blockBuffer.limit()) {
+          throw new IllegalStateException("Invalid currTagsLen " + currTagsLen + ". Block offset: "
+              + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
+              + blockBuffer.position() + " (without header).");
+        }
         ByteBufferUtils.skip(blockBuffer, currTagsLen);
       }
       readMvccVersion();
-      if (currKeyLen < 0 || currValueLen < 0 || currTagsLen < 0 || currKeyLen > blockBuffer.limit()
-          || currValueLen > blockBuffer.limit() || currTagsLen > blockBuffer.limit()) {
-        throw new IllegalStateException("Invalid currKeyLen " + currKeyLen + " or currValueLen "
-            + currValueLen + " or currTagLen " + currTagsLen + ". Block offset: "
-            + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
-            + blockBuffer.position() + " (without header).");
-      }
       blockBuffer.reset();
     }
 
@@ -184,9 +253,21 @@ public class HFileReaderV3 extends HFile
         blockBuffer.mark();
         klen = blockBuffer.getInt();
         vlen = blockBuffer.getInt();
+        if (klen < 0 || vlen < 0 || klen > blockBuffer.limit()
+            || vlen > blockBuffer.limit()) {
+          throw new IllegalStateException("Invalid klen " + klen + " or vlen "
+              + vlen + ". Block offset: "
+              + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
+              + blockBuffer.position() + " (without header).");
+        }
         ByteBufferUtils.skip(blockBuffer, klen + vlen);
         if (reader.hfileContext.isIncludesTags()) {
           tlen = blockBuffer.getShort();
+          if (tlen < 0 || tlen > blockBuffer.limit()) {
+            throw new IllegalStateException("Invalid tlen " + tlen + ". Block offset: "
+                + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
+                + blockBuffer.position() + " (without header).");
+          }
           ByteBufferUtils.skip(blockBuffer, tlen);
         }
         if (this.reader.shouldIncludeMemstoreTS()) {
@@ -262,8 +343,8 @@ public class HFileReaderV3 extends HFile
    */
   protected static class EncodedScannerV3 extends EncodedScannerV2 {
     public EncodedScannerV3(HFileReaderV3 reader, boolean cacheBlocks, boolean pread,
-        boolean isCompaction, HFileContext meta) {
-      super(reader, cacheBlocks, pread, isCompaction, meta);
+        boolean isCompaction, HFileContext context) {
+      super(reader, cacheBlocks, pread, isCompaction, context);
     }
   }
 

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java Tue Nov 26 04:22:33 2013
@@ -459,4 +459,9 @@ public class HFileWriterV2 extends Abstr
   protected int getMinorVersion() {
     return HFileReaderV2.MAX_MINOR_VERSION;
   }
+
+  @Override
+  public HFileContext getFileContext() {
+    return hFileContext;
+  }
 }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV3.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV3.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV3.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV3.java Tue Nov 26 04:22:33 2013
@@ -17,9 +17,12 @@
  */
 package org.apache.hadoop.hbase.io.hfile;
 
+import java.io.ByteArrayOutputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -28,18 +31,23 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValue.KVComparator;
+import org.apache.hadoop.hbase.io.crypto.Encryption;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
 import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
+import org.apache.hadoop.hbase.security.EncryptionUtil;
+import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.WritableUtils;
 
 /**
- * This is an extension of HFileWriterV2 that is tags aware.
+ * {@link HFile} writer for version 3.
  */
 @InterfaceAudience.Private
 public class HFileWriterV3 extends HFileWriterV2 {
 
+  private static final Log LOG = LogFactory.getLog(HFileWriterV3.class);
+
   private int maxTagsLength = 0;
 
   static class WriterFactoryV3 extends HFile.WriterFactory {
@@ -60,6 +68,12 @@ public class HFileWriterV3 extends HFile
       FSDataOutputStream ostream, final KVComparator comparator,
       final HFileContext fileContext) throws IOException {
     super(conf, cacheConf, fs, path, ostream, comparator, fileContext);
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("Writer" + (path != null ? " for " + path : "") +
+        " initialized with cacheConf: " + cacheConf +
+        " comparator: " + comparator.getClass().getSimpleName() +
+        " fileContext: " + fileContext);
+    }
   }
 
   /**
@@ -194,4 +208,25 @@ public class HFileWriterV3 extends HFile
   protected int getMinorVersion() {
     return HFileReaderV3.MAX_MINOR_VERSION;
   }
+
+  @Override
+  protected void finishClose(FixedFileTrailer trailer) throws IOException {
+    // Write out encryption metadata before finalizing if we have a valid crypto context
+    Encryption.Context cryptoContext = hFileContext.getEncryptionContext();
+    if (cryptoContext != Encryption.Context.NONE) {
+      // Wrap the context's key and write it as the encryption metadata, the wrapper includes
+      // all information needed for decryption
+      ByteArrayOutputStream os = new ByteArrayOutputStream();
+      os.write(EncryptionUtil.wrapKey(cryptoContext.getConf(),
+        cryptoContext.getConf().get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
+          User.getCurrent().getShortName()),
+        cryptoContext.getKey()));
+      os.close();
+      trailer.setEncryptionKey(os.toByteArray());
+    }
+
+    // Now we can finish the close
+    super.finishClose(trailer);
+  }
+
 }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java Tue Nov 26 04:22:33 2013
@@ -474,7 +474,7 @@ public class LoadIncrementalHFiles exten
     final Path hfilePath = item.hfilePath;
     final FileSystem fs = hfilePath.getFileSystem(getConf());
     HFile.Reader hfr = HFile.createReader(fs, hfilePath,
-        new CacheConfig(getConf()));
+        new CacheConfig(getConf()), getConf());
     final byte[] first, last;
     try {
       hfr.loadFileInfo();
@@ -651,7 +651,7 @@ public class LoadIncrementalHFiles exten
     HalfStoreFileReader halfReader = null;
     StoreFile.Writer halfWriter = null;
     try {
-      halfReader = new HalfStoreFileReader(fs, inFile, cacheConf, reference);
+      halfReader = new HalfStoreFileReader(fs, inFile, cacheConf, reference, conf);
       Map<byte[], byte[]> fileInfo = halfReader.loadFileInfo();
 
       int blocksize = familyDescriptor.getBlocksize();
@@ -769,11 +769,11 @@ public class LoadIncrementalHFiles exten
       for (Path hfile : hfiles) {
         if (hfile.getName().startsWith("_")) continue;
         HFile.Reader reader = HFile.createReader(fs, hfile,
-            new CacheConfig(getConf()));
+            new CacheConfig(getConf()), getConf());
         final byte[] first, last;
         try {
-          if (hcd.getCompressionType() != reader.getCompressionAlgorithm()) {
-            hcd.setCompressionType(reader.getCompressionAlgorithm());
+          if (hcd.getCompressionType() != reader.getFileContext().getCompression()) {
+            hcd.setCompressionType(reader.getFileContext().getCompression());
             LOG.info("Setting compression " + hcd.getCompressionType().name() +
                      " for family " + hcd.toString());
           }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java Tue Nov 26 04:22:33 2013
@@ -21,6 +21,8 @@ package org.apache.hadoop.hbase.regionse
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.net.InetSocketAddress;
+import java.security.Key;
+import java.security.KeyException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -56,6 +58,8 @@ import org.apache.hadoop.hbase.RemoteExc
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.compress.Compression;
+import org.apache.hadoop.hbase.io.crypto.Cipher;
+import org.apache.hadoop.hbase.io.crypto.Encryption;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.io.hfile.HFileContext;
@@ -73,6 +77,8 @@ import org.apache.hadoop.hbase.regionser
 import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
 import org.apache.hadoop.hbase.regionserver.compactions.OffPeakHours;
 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
+import org.apache.hadoop.hbase.security.EncryptionUtil;
+import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.ClassSize;
@@ -174,6 +180,8 @@ public class HStore implements Store {
   private long blockingFileCount;
   private int compactionCheckMultiplier;
 
+  private Encryption.Context cryptoContext = Encryption.Context.NONE;
+
   /**
    * Constructor
    * @param region
@@ -254,6 +262,63 @@ public class HStore implements Store {
           "hbase.hstore.flush.retries.number must be > 0, not "
               + flushRetriesNumber);
     }
+
+    // Crypto context for new store files
+    String cipherName = family.getEncryptionType();
+    if (cipherName != null) {
+      Cipher cipher;
+      Key key;
+      byte[] keyBytes = family.getEncryptionKey();
+      if (keyBytes != null) {
+        // Family provides specific key material
+        String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
+          User.getCurrent().getShortName());
+        try {
+          // First try the master key
+          key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes);
+        } catch (KeyException e) {
+          // If the current master key fails to unwrap, try the alternate, if
+          // one is configured
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'");
+          }
+          String alternateKeyName =
+            conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY);
+          if (alternateKeyName != null) {
+            try {
+              key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes);
+            } catch (KeyException ex) {
+              throw new IOException(ex);
+            }
+          } else {
+            throw new IOException(e);
+          }
+        }
+        // Use the algorithm the key wants
+        cipher = Encryption.getCipher(conf, key.getAlgorithm());
+        if (cipher == null) {
+          throw new RuntimeException("Cipher '" + cipher + "' is not available");
+        }
+        // Fail if misconfigured
+        // We use the encryption type specified in the column schema as a sanity check on
+        // what the wrapped key is telling us
+        if (!cipher.getName().equalsIgnoreCase(cipherName)) {
+          throw new RuntimeException("Encryption for family '" + family.getNameAsString() +
+            "' configured with type '" + cipherName +
+            "' but key specifies algorithm '" + cipher.getName() + "'");
+        }
+      } else {
+        // Family does not provide key material, create a random key
+        cipher = Encryption.getCipher(conf, cipherName);
+        if (cipher == null) {
+          throw new RuntimeException("Cipher '" + cipher + "' is not available");
+        }
+        key = cipher.getRandomKey();
+      }
+      cryptoContext = Encryption.newContext(conf);
+      cryptoContext.setCipher(cipher);
+      cryptoContext.setKey(key);
+    }
   }
 
   /**
@@ -534,7 +599,7 @@ public class HStore implements Store {
       LOG.info("Validating hfile at " + srcPath + " for inclusion in "
           + "store " + this + " region " + this.getRegionInfo().getRegionNameAsString());
       reader = HFile.createReader(srcPath.getFileSystem(conf),
-          srcPath, cacheConf);
+          srcPath, cacheConf, conf);
       reader.loadFileInfo();
 
       byte[] firstKey = reader.getFirstRowKey();
@@ -805,7 +870,8 @@ public class HStore implements Store {
       favoredNodes = region.getRegionServerServices().getFavoredNodesForRegion(
           region.getRegionInfo().getEncodedName());
     }
-    HFileContext hFileContext = createFileContext(compression, includeMVCCReadpoint, includesTag);
+    HFileContext hFileContext = createFileContext(compression, includeMVCCReadpoint, includesTag,
+      cryptoContext);
     StoreFile.Writer w = new StoreFile.WriterBuilder(conf, writerCacheConf,
         this.getFileSystem())
             .withFilePath(fs.createTempName())
@@ -819,7 +885,7 @@ public class HStore implements Store {
   }
 
   private HFileContext createFileContext(Compression.Algorithm compression,
-      boolean includeMVCCReadpoint, boolean includesTag) {
+      boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context cryptoContext) {
     if (compression == null) {
       compression = HFile.DEFAULT_COMPRESSION_ALGORITHM;
     }
@@ -833,6 +899,7 @@ public class HStore implements Store {
                                 .withBlockSize(blocksize)
                                 .withHBaseCheckSum(true)
                                 .withDataBlockEncoding(family.getDataBlockEncoding())
+                                .withEncryptionContext(cryptoContext)
                                 .build();
     return hFileContext;
   }
@@ -1916,7 +1983,7 @@ public class HStore implements Store {
   }
 
   public static final long FIXED_OVERHEAD =
-      ClassSize.align(ClassSize.OBJECT + (15 * ClassSize.REFERENCE) + (4 * Bytes.SIZEOF_LONG)
+      ClassSize.align(ClassSize.OBJECT + (16 * ClassSize.REFERENCE) + (4 * Bytes.SIZEOF_LONG)
               + (5 * Bytes.SIZEOF_INT) + (2 * Bytes.SIZEOF_BOOLEAN));
 
   public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Tue Nov 26 04:22:33 2013
@@ -1012,14 +1012,15 @@ public class StoreFile {
     private byte[] lastBloomKey;
     private long deleteFamilyCnt = -1;
 
-    public Reader(FileSystem fs, Path path, CacheConfig cacheConf) throws IOException {
-      reader = HFile.createReader(fs, path, cacheConf);
+    public Reader(FileSystem fs, Path path, CacheConfig cacheConf, Configuration conf)
+        throws IOException {
+      reader = HFile.createReader(fs, path, cacheConf, conf);
       bloomFilterType = BloomType.NONE;
     }
 
     public Reader(FileSystem fs, Path path, FSDataInputStreamWrapper in, long size,
-        CacheConfig cacheConf) throws IOException {
-      reader = HFile.createReader(fs, path, in, size, cacheConf);
+        CacheConfig cacheConf, Configuration conf) throws IOException {
+      reader = HFile.createReader(fs, path, in, size, cacheConf, conf);
       bloomFilterType = BloomType.NONE;
     }
 

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java Tue Nov 26 04:22:33 2013
@@ -67,6 +67,9 @@ public class StoreFileInfo {
     Pattern.compile(String.format("^(%s|%s)\\.(.+)$",
       HFILE_NAME_REGEX, HFileLink.LINK_NAME_REGEX));
 
+  // Configuration
+  private Configuration conf;
+
   // HDFS blocks distribution information
   private HDFSBlocksDistribution hdfsBlocksDistribution = null;
 
@@ -100,6 +103,7 @@ public class StoreFileInfo {
    */
   public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus)
       throws IOException {
+    this.conf = conf;
     this.fileStatus = fileStatus;
     Path p = fileStatus.getPath();
     if (HFileLink.isHFileLink(p)) {
@@ -201,9 +205,10 @@ public class StoreFileInfo {
     }
     if (reader == null) {
       if (this.reference != null) {
-        reader = new HalfStoreFileReader(fs, this.getPath(), in, length, cacheConf, reference);
+        reader = new HalfStoreFileReader(fs, this.getPath(), in, length, cacheConf, reference,
+          conf);
       } else {
-        reader = new StoreFile.Reader(fs, this.getPath(), in, length, cacheConf);
+        reader = new StoreFile.Reader(fs, this.getPath(), in, length, cacheConf, conf);
       }
     }
     if (this.coprocessorHost != null) {

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java Tue Nov 26 04:22:33 2013
@@ -28,10 +28,12 @@ import java.util.Arrays;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.hbase.codec.Codec;
 import org.apache.hadoop.hbase.io.LimitInputStream;
 import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
+import org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALHeader.Builder;
 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALKey;
 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALTrailer;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -55,10 +57,10 @@ public class ProtobufLogReader extends R
   private static final Log LOG = LogFactory.getLog(ProtobufLogReader.class);
   static final byte[] PB_WAL_MAGIC = Bytes.toBytes("PWAL");
   static final byte[] PB_WAL_COMPLETE_MAGIC = Bytes.toBytes("LAWP");
-  private FSDataInputStream inputStream;
-  private Codec.Decoder cellDecoder;
-  private WALCellCodec.ByteStringUncompressor byteStringUncompressor;
-  private boolean hasCompression = false;
+  protected FSDataInputStream inputStream;
+  protected Codec.Decoder cellDecoder;
+  protected WALCellCodec.ByteStringUncompressor byteStringUncompressor;
+  protected boolean hasCompression = false;
   // walEditsStopOffset is the position of the last byte to read. After reading the last WALEdit entry
   // in the hlog, the inputstream's position is equal to walEditsStopOffset.
   private long walEditsStopOffset;
@@ -92,6 +94,10 @@ public class ProtobufLogReader extends R
     initInternal(stream, true);
   }
 
+  protected boolean readHeader(Builder builder, FSDataInputStream stream) throws IOException {
+    return builder.mergeDelimitedFrom(stream);
+  }
+
   private void initInternal(FSDataInputStream stream, boolean isFirst) throws IOException {
     close();
     long expectedPos = PB_WAL_MAGIC.length;
@@ -104,7 +110,7 @@ public class ProtobufLogReader extends R
     }
     // Initialize metadata or, when we reset, just skip the header.
     WALProtos.WALHeader.Builder builder = WALProtos.WALHeader.newBuilder();
-    boolean hasHeader = builder.mergeDelimitedFrom(stream);
+    boolean hasHeader = readHeader(builder, stream);
     if (!hasHeader) {
       throw new EOFException("Couldn't read WAL PB header");
     }
@@ -176,9 +182,14 @@ public class ProtobufLogReader extends R
     return false;
   }
 
+  protected WALCellCodec getCodec(Configuration conf, CompressionContext compressionContext)
+      throws IOException {
+    return WALCellCodec.create(conf, compressionContext);
+  }
+
   @Override
   protected void initAfterCompression() throws IOException {
-    WALCellCodec codec = WALCellCodec.create(this.conf, this.compressionContext);
+    WALCellCodec codec = getCodec(this.conf, this.compressionContext);
     this.cellDecoder = codec.getDecoder(this.inputStream);
     if (this.hasCompression) {
       this.byteStringUncompressor = codec.getByteStringUncompressor();

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java Tue Nov 26 04:22:33 2013
@@ -40,9 +40,9 @@ import org.apache.hadoop.hbase.util.FSUt
 @InterfaceAudience.Private
 public class ProtobufLogWriter extends WriterBase {
   private final Log LOG = LogFactory.getLog(this.getClass());
-  private FSDataOutputStream output;
-  private Codec.Encoder cellEncoder;
-  private WALCellCodec.ByteStringCompressor compressor;
+  protected FSDataOutputStream output;
+  protected Codec.Encoder cellEncoder;
+  protected WALCellCodec.ByteStringCompressor compressor;
   private boolean trailerWritten;
   private WALTrailer trailer;
   // maximum size of the wal Trailer in bytes. If a user writes/reads a trailer with size larger
@@ -53,9 +53,19 @@ public class ProtobufLogWriter extends W
     super();
   }
 
+  protected WALCellCodec getCodec(Configuration conf, CompressionContext compressionContext)
+      throws IOException {
+    return WALCellCodec.create(conf, compressionContext);
+  }
+
+  protected WALHeader buildWALHeader(WALHeader.Builder builder) throws IOException {
+    return builder.build();
+  }
+
   @Override
   @SuppressWarnings("deprecation")
   public void init(FileSystem fs, Path path, Configuration conf, boolean overwritable) throws IOException {
+    super.init(fs, path, conf, overwritable);
     assert this.output == null;
     boolean doCompress = initializeCompressionContext(conf, path);
     this.trailerWarnSize = conf.getInt(HLog.WAL_TRAILER_WARN_SIZE,
@@ -67,13 +77,10 @@ public class ProtobufLogWriter extends W
         FSUtils.getDefaultBlockSize(fs, path));
     output = fs.createNonRecursive(path, overwritable, bufferSize, replication, blockSize, null);
     output.write(ProtobufLogReader.PB_WAL_MAGIC);
-    WALHeader.newBuilder().setHasCompression(doCompress).build().writeDelimitedTo(output);
+    buildWALHeader(WALHeader.newBuilder().setHasCompression(doCompress)).writeDelimitedTo(output);
+
+    initAfterHeader(doCompress);
 
-    WALCellCodec codec = WALCellCodec.create(conf, this.compressionContext);
-    this.cellEncoder = codec.getEncoder(this.output);
-    if (doCompress) {
-      this.compressor = codec.getByteStringCompressor();
-    }
     // instantiate trailer to default value.
     trailer = WALTrailer.newBuilder().build();
     if (LOG.isTraceEnabled()) {
@@ -81,6 +88,14 @@ public class ProtobufLogWriter extends W
     }
   }
 
+  protected void initAfterHeader(boolean doCompress) throws IOException {
+    WALCellCodec codec = getCodec(conf, this.compressionContext);
+    this.cellEncoder = codec.getEncoder(this.output);
+    if (doCompress) {
+      this.compressor = codec.getByteStringCompressor();
+    }
+  }
+
   @Override
   public void append(HLog.Entry entry) throws IOException {
     entry.setCompressionContext(compressionContext);
@@ -106,13 +121,17 @@ public class ProtobufLogWriter extends W
     }
   }
 
+  protected WALTrailer buildWALTrailer(WALTrailer.Builder builder) {
+    return builder.build();
+  }
+
   private void writeWALTrailer() {
     try {
       int trailerSize = 0;
       if (this.trailer == null) {
         // use default trailer.
         LOG.warn("WALTrailer is null. Continuing with default.");
-        this.trailer = WALTrailer.newBuilder().build();
+        this.trailer = buildWALTrailer(WALTrailer.newBuilder());
         trailerSize = this.trailer.getSerializedSize();
       } else if ((trailerSize = this.trailer.getSerializedSize()) > this.trailerWarnSize) {
         // continue writing after warning the user.
@@ -120,8 +139,8 @@ public class ProtobufLogWriter extends W
           trailerSize + " > " + this.trailerWarnSize);
       }
       this.trailer.writeTo(output);
-      this.output.writeInt(trailerSize);
-      this.output.write(ProtobufLogReader.PB_WAL_COMPLETE_MAGIC);
+      output.writeInt(trailerSize);
+      output.write(ProtobufLogReader.PB_WAL_COMPLETE_MAGIC);
       this.trailerWritten = true;
     } catch (IOException ioe) {
       LOG.error("Got IOException while writing trailer", ioe);

Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogReader.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogReader.java?rev=1545536&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogReader.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogReader.java Tue Nov 26 04:22:33 2013
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import java.io.IOException;
+import java.security.Key;
+import java.security.KeyException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.io.crypto.Cipher;
+import org.apache.hadoop.hbase.io.crypto.Decryptor;
+import org.apache.hadoop.hbase.io.crypto.Encryption;
+import org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALHeader;
+import org.apache.hadoop.hbase.security.EncryptionUtil;
+import org.apache.hadoop.hbase.security.User;
+
+public class SecureProtobufLogReader extends ProtobufLogReader {
+
+  private static final Log LOG = LogFactory.getLog(SecureProtobufLogReader.class);
+
+  private Decryptor decryptor = null;
+
+  @Override
+  protected boolean readHeader(WALHeader.Builder builder, FSDataInputStream stream)
+      throws IOException {
+    boolean result = super.readHeader(builder, stream);
+    // We need to unconditionally handle the case where the WAL has a key in
+    // the header, meaning it is encrypted, even if ENABLE_WAL_ENCRYPTION is
+    // no longer set in the site configuration.
+    if (result && builder.hasEncryptionKey()) {
+      // Serialized header data has been merged into the builder from the
+      // stream.
+
+      // Retrieve a usable key
+
+      byte[] keyBytes = builder.getEncryptionKey().toByteArray();
+      Key key = null;
+      String walKeyName = conf.get(HConstants.CRYPTO_WAL_KEY_NAME_CONF_KEY);
+      // First try the WAL key, if one is configured
+      if (walKeyName != null) {
+        try {
+          key = EncryptionUtil.unwrapKey(conf, walKeyName, keyBytes);
+        } catch (KeyException e) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Unable to unwrap key with WAL key '" + walKeyName + "'");
+          }
+          key = null;
+        }
+      }
+      if (key == null) {
+        String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
+          User.getCurrent().getShortName());
+        try {
+          // Then, try the cluster master key
+          key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes);
+        } catch (KeyException e) {
+          // If the current master key fails to unwrap, try the alternate, if
+          // one is configured
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'");
+          }
+          String alternateKeyName =
+            conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY);
+          if (alternateKeyName != null) {
+            try {
+              key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes);
+            } catch (KeyException ex) {
+              throw new IOException(ex);
+            }
+          } else {
+            throw new IOException(e);
+          }
+        }
+      }
+
+      // Use the algorithm the key wants
+
+      Cipher cipher = Encryption.getCipher(conf, key.getAlgorithm());
+      if (cipher == null) {
+        throw new IOException("Cipher '" + key.getAlgorithm() + "' is not available");
+      }
+
+      // Set up the decryptor for this WAL
+
+      decryptor = cipher.getDecryptor();
+      decryptor.setKey(key);
+
+      if (LOG.isTraceEnabled()) {
+        LOG.trace("Initialized secure protobuf WAL: cipher=" + cipher.getName());
+      }
+    }
+
+    return result;
+  }
+
+  @Override
+  protected void initAfterCompression() throws IOException {
+    WALCellCodec codec = SecureWALCellCodec.getCodec(this.conf, decryptor);
+    this.cellDecoder = codec.getDecoder(this.inputStream);
+    // We do not support compression
+    this.compressionContext = null;
+    this.hasCompression = false;
+  }
+
+}

Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogWriter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogWriter.java?rev=1545536&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogWriter.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogWriter.java Tue Nov 26 04:22:33 2013
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import java.io.IOException;
+import java.security.Key;
+import java.security.SecureRandom;
+
+import javax.crypto.spec.SecretKeySpec;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.io.crypto.Cipher;
+import org.apache.hadoop.hbase.io.crypto.Encryption;
+import org.apache.hadoop.hbase.io.crypto.Encryptor;
+import org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALHeader;
+import org.apache.hadoop.hbase.security.EncryptionUtil;
+import org.apache.hadoop.hbase.security.User;
+
+import com.google.protobuf.ZeroCopyLiteralByteString;
+
+public class SecureProtobufLogWriter extends ProtobufLogWriter {
+
+  private static final Log LOG = LogFactory.getLog(SecureProtobufLogWriter.class);
+  private static final String DEFAULT_CIPHER = "AES";
+
+  private Encryptor encryptor = null;
+
+  @Override
+  protected WALHeader buildWALHeader(WALHeader.Builder builder) throws IOException {
+    if (conf.getBoolean(HConstants.ENABLE_WAL_ENCRYPTION, false)) {
+      // Get an instance of our cipher
+      Cipher cipher = Encryption.getCipher(conf,
+        conf.get(HConstants.CRYPTO_WAL_ALGORITHM_CONF_KEY, DEFAULT_CIPHER));
+      if (cipher == null) {
+        throw new RuntimeException("Cipher '" + cipher + "' is not available");
+      }
+
+      // Generate an encryption key for this WAL
+      SecureRandom rng = new SecureRandom();
+      byte[] keyBytes = new byte[cipher.getKeyLength()];
+      rng.nextBytes(keyBytes);
+      Key key = new SecretKeySpec(keyBytes, cipher.getName());
+      builder.setEncryptionKey(ZeroCopyLiteralByteString.wrap(EncryptionUtil.wrapKey(conf,
+        conf.get(HConstants.CRYPTO_WAL_KEY_NAME_CONF_KEY,
+          conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
+            User.getCurrent().getShortName())),
+        key)));
+
+      // Set up the encryptor
+      encryptor = cipher.getEncryptor();
+      encryptor.setKey(key);
+
+      if (LOG.isTraceEnabled()) {
+        LOG.trace("Initialized secure protobuf WAL: cipher=" + cipher.getName());
+      }
+    }
+
+    return super.buildWALHeader(builder);
+  }
+
+  @Override
+  protected void initAfterHeader(boolean doCompress) throws IOException {
+    if (conf.getBoolean(HConstants.ENABLE_WAL_ENCRYPTION, false) && encryptor != null) {
+      WALCellCodec codec = SecureWALCellCodec.getCodec(this.conf, encryptor);
+      this.cellEncoder = codec.getEncoder(this.output);
+      // We do not support compression
+      this.compressionContext = null;
+    } else {
+      super.initAfterHeader(doCompress);
+    }
+  }
+
+}

Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java?rev=1545536&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java Tue Nov 26 04:22:33 2013
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.SecureRandom;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.codec.KeyValueCodec;
+import org.apache.hadoop.hbase.io.crypto.Decryptor;
+import org.apache.hadoop.hbase.io.crypto.Encryptor;
+import org.apache.hadoop.hbase.io.util.StreamUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * A WALCellCodec that encrypts the WALedits.
+ */
+public class SecureWALCellCodec extends WALCellCodec {
+
+  private static final SecureRandom RNG = new SecureRandom();
+
+  private Encryptor encryptor;
+  private Decryptor decryptor;
+
+  public SecureWALCellCodec(Configuration conf, Encryptor encryptor) {
+    super(conf, null);
+    this.encryptor = encryptor;
+  }
+
+  public SecureWALCellCodec(Configuration conf, Decryptor decryptor) {
+    super(conf, null);
+    this.decryptor = decryptor;
+  }
+
+  static class EncryptedKvDecoder extends KeyValueCodec.KeyValueDecoder {
+
+    private Decryptor decryptor;
+    private byte[] iv;
+
+    public EncryptedKvDecoder(InputStream in) {
+      super(in);
+    }
+
+    public EncryptedKvDecoder(InputStream in, Decryptor decryptor) {
+      super(in);
+      this.decryptor = decryptor;
+      this.iv = new byte[decryptor.getIvLength()];
+    }
+
+    @Override
+    protected Cell parseCell() throws IOException {
+      int ivLength = 0;
+      try {
+        ivLength = StreamUtils.readRawVarint32(in);
+      } catch (EOFException e) {
+        // EOF at start is OK
+        return null;
+      }
+
+      // TODO: An IV length of 0 could signify an unwrapped cell, when the
+      // encoder supports that just read the remainder in directly
+
+      if (ivLength != this.iv.length) {
+        throw new IOException("Incorrect IV length: expected=" + iv.length + " have=" +
+          ivLength);
+      }
+      IOUtils.readFully(in, this.iv);
+
+      int codedLength = StreamUtils.readRawVarint32(in);
+      byte[] codedBytes = new byte[codedLength];
+      IOUtils.readFully(in, codedBytes);
+
+      decryptor.setIv(iv);
+      decryptor.reset();
+
+      InputStream cin = decryptor.createDecryptionStream(new ByteArrayInputStream(codedBytes));
+
+      // TODO: Add support for WAL compression
+
+      int keylength = StreamUtils.readRawVarint32(cin);
+      int vlength = StreamUtils.readRawVarint32(cin);
+      int tagsLength = StreamUtils.readRawVarint32(cin);
+      int length = 0;
+      if (tagsLength == 0) {
+        length = KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength;
+      } else {
+        length = KeyValue.KEYVALUE_WITH_TAGS_INFRASTRUCTURE_SIZE + keylength + vlength + tagsLength;
+      }
+
+      byte[] backingArray = new byte[length];
+      int pos = 0;
+      pos = Bytes.putInt(backingArray, pos, keylength);
+      pos = Bytes.putInt(backingArray, pos, vlength);
+
+      // Row
+      int elemLen = StreamUtils.readRawVarint32(cin);
+      pos = Bytes.putShort(backingArray, pos, (short)elemLen);
+      IOUtils.readFully(cin, backingArray, pos, elemLen);
+      pos += elemLen;
+      // Family
+      elemLen = StreamUtils.readRawVarint32(cin);
+      pos = Bytes.putByte(backingArray, pos, (byte)elemLen);
+      IOUtils.readFully(cin, backingArray, pos, elemLen);
+      pos += elemLen;
+      // Qualifier
+      elemLen = StreamUtils.readRawVarint32(cin);
+      IOUtils.readFully(cin, backingArray, pos, elemLen);
+      pos += elemLen;
+      // Remainder
+      IOUtils.readFully(cin, backingArray, pos, length - pos);
+      return new KeyValue(backingArray, 0, length);
+    }
+
+  }
+
+  static class EncryptedKvEncoder extends KeyValueCodec.KeyValueEncoder {
+
+    private Encryptor encryptor;
+    private byte[] iv;
+
+    public EncryptedKvEncoder(OutputStream os) {
+      super(os);
+    }
+
+    public EncryptedKvEncoder(OutputStream os, Encryptor encryptor) {
+      super(os);
+      this.encryptor = encryptor;
+      iv = new byte[encryptor.getIvLength()];
+    }
+
+    @Override
+    public void write(Cell cell) throws IOException {
+      if (!(cell instanceof KeyValue)) throw new IOException("Cannot write non-KV cells to WAL");
+
+      KeyValue kv = (KeyValue)cell;
+      byte[] kvBuffer = kv.getBuffer();
+      int offset = kv.getOffset();
+
+      RNG.nextBytes(iv);
+      encryptor.setIv(iv);
+      encryptor.reset();
+
+      // TODO: Check if this is a cell for an encrypted CF. If not, we can
+      // write a 0 here to signal an unwrapped cell and just dump the KV bytes
+      // afterward
+
+      StreamUtils.writeRawVInt32(out, iv.length);
+      out.write(iv);
+
+      // TODO: Add support for WAL compression
+
+      ByteArrayOutputStream baos = new ByteArrayOutputStream();
+      OutputStream cout = encryptor.createEncryptionStream(baos);
+
+      // Write the KeyValue infrastructure as VInts.
+      StreamUtils.writeRawVInt32(cout, kv.getKeyLength());
+      StreamUtils.writeRawVInt32(cout, kv.getValueLength());
+      // To support tags
+      StreamUtils.writeRawVInt32(cout, kv.getTagsLength());
+
+      // Write row, qualifier, and family
+      StreamUtils.writeRawVInt32(cout, kv.getRowLength());
+      cout.write(kvBuffer, kv.getRowOffset(), kv.getRowLength());
+      StreamUtils.writeRawVInt32(cout, kv.getFamilyLength());
+      cout.write(kvBuffer, kv.getFamilyOffset(), kv.getFamilyLength());
+      StreamUtils.writeRawVInt32(cout, kv.getQualifierLength());
+      cout.write(kvBuffer, kv.getQualifierOffset(), kv.getQualifierLength());
+      // Write the rest
+      int pos = kv.getTimestampOffset();
+      int remainingLength = kv.getLength() + offset - pos;
+      cout.write(kvBuffer, pos, remainingLength);
+      cout.close();
+
+      byte[] codedBytes = baos.toByteArray();
+      StreamUtils.writeRawVInt32(out, codedBytes.length);
+      out.write(codedBytes);
+    }
+
+  }
+
+  @Override
+  public Decoder getDecoder(InputStream is) {
+    return new EncryptedKvDecoder(is, decryptor);
+  }
+
+  @Override
+  public Encoder getEncoder(OutputStream os) {
+    return new EncryptedKvEncoder(os, encryptor);
+  }
+
+  public static WALCellCodec getCodec(Configuration conf, Encryptor encryptor) {
+    return new SecureWALCellCodec(conf, encryptor);
+  }
+
+  public static WALCellCodec getCodec(Configuration conf, Decryptor decryptor) {
+    return new SecureWALCellCodec(conf, decryptor);
+  }
+
+}