You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jm...@apache.org on 2015/07/22 21:52:33 UTC
[15/50] [abbrv] hbase git commit: HBASE-13693 [HBase MOB] Mob files
are not encrypting (Ashutosh Jindai)
HBASE-13693 [HBase MOB] Mob files are not encrypting (Ashutosh Jindai)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8a09d95c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8a09d95c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8a09d95c
Branch: refs/heads/master
Commit: 8a09d95c4635689dfe48faa7aeae4ac611488d08
Parents: 4da0149
Author: Jonathan M Hsieh <jm...@apache.org>
Authored: Tue May 19 16:10:36 2015 -0700
Committer: Jonathan M Hsieh <jm...@apache.org>
Committed: Tue May 19 16:11:45 2015 -0700
----------------------------------------------------------------------
.../hadoop/hbase/regionserver/HMobStore.java | 4 +-
.../hadoop/hbase/regionserver/HStore.java | 2 +-
.../hbase/regionserver/TestHMobStore.java | 89 +++++++++++++++++++-
3 files changed, 90 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/8a09d95c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
index 6016b39..b4d2213 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
@@ -1,5 +1,4 @@
/**
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -251,7 +250,8 @@ public class HMobStore extends HStore {
.withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE)
.withBytesPerCheckSum(HFile.DEFAULT_BYTES_PER_CHECKSUM)
.withBlockSize(getFamily().getBlocksize())
- .withHBaseCheckSum(true).withDataBlockEncoding(getFamily().getDataBlockEncoding()).build();
+ .withHBaseCheckSum(true).withDataBlockEncoding(getFamily().getDataBlockEncoding())
+ .withEncryptionContext(cryptoContext).build();
StoreFile.Writer w = new StoreFile.WriterBuilder(conf, writerCacheConf, region.getFilesystem())
.withFilePath(new Path(basePath, mobFileName.getFileName()))
http://git-wip-us.apache.org/repos/asf/hbase/blob/8a09d95c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index e0bb7cf..ad7ce98 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -193,7 +193,7 @@ public class HStore implements Store {
private long blockingFileCount;
private int compactionCheckMultiplier;
- private Encryption.Context cryptoContext = Encryption.Context.NONE;
+ protected Encryption.Context cryptoContext = Encryption.Context.NONE;
private volatile long flushedCellsCount = 0;
private volatile long compactedCellsCount = 0;
http://git-wip-us.apache.org/repos/asf/hbase/blob/8a09d95c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
index 39fd410..0be79bf 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
@@ -19,7 +19,10 @@
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
+import java.security.Key;
+import java.security.SecureRandom;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
@@ -27,31 +30,43 @@ import java.util.List;
import java.util.NavigableSet;
import java.util.concurrent.ConcurrentSkipListSet;
+import javax.crypto.spec.SecretKeySpec;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.HarFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.TagType;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
+import org.apache.hadoop.hbase.io.crypto.aes.AES;
+import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.regionserver.StoreFile.Reader;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.compactions.NoLimitCompactionThroughputController;
+import org.apache.hadoop.hbase.security.EncryptionUtil;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.wal.WAL;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.junit.Assert;
import org.junit.Before;
@@ -469,4 +484,74 @@ public class TestHMobStore {
storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
}
+
+ @Test
+ public void testMOBStoreEncryption() throws Exception {
+ final Configuration conf = TEST_UTIL.getConfiguration();
+
+ conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
+ conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
+ SecureRandom rng = new SecureRandom();
+ byte[] keyBytes = new byte[AES.KEY_LENGTH];
+ rng.nextBytes(keyBytes);
+ String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
+ Key cfKey = new SecretKeySpec(keyBytes, algorithm);
+
+ HColumnDescriptor hcd = new HColumnDescriptor(family);
+ hcd.setMobEnabled(true);
+ hcd.setMobThreshold(100);
+ hcd.setMaxVersions(4);
+ hcd.setEncryptionType(algorithm);
+ hcd.setEncryptionKey(EncryptionUtil.wrapKey(conf,
+ conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey));
+
+ init(name.getMethodName(), conf, hcd, false);
+
+ this.store.add(new KeyValue(row, family, qf1, 1, value));
+ this.store.add(new KeyValue(row, family, qf2, 1, value));
+ this.store.add(new KeyValue(row, family, qf3, 1, value));
+ flush(1);
+
+ this.store.add(new KeyValue(row, family, qf4, 1, value));
+ this.store.add(new KeyValue(row, family, qf5, 1, value));
+ this.store.add(new KeyValue(row, family, qf6, 1, value));
+ flush(2);
+
+ Collection<StoreFile> storefiles = this.store.getStorefiles();
+ checkMobHFileEncrytption(storefiles);
+
+ // Scan the values
+ Scan scan = new Scan(get);
+ InternalScanner scanner = (InternalScanner) store.getScanner(scan,
+ scan.getFamilyMap().get(store.getFamily().getName()),
+ 0);
+
+ List<Cell> results = new ArrayList<Cell>();
+ scanner.next(results);
+ Collections.sort(results, KeyValue.COMPARATOR);
+ scanner.close();
+ Assert.assertEquals(expected.size(), results.size());
+ for(int i=0; i<results.size(); i++) {
+ Assert.assertEquals(expected.get(i), results.get(i));
+ }
+
+ // Trigger major compaction
+ this.store.triggerMajorCompaction();
+ CompactionContext requestCompaction = this.store.requestCompaction(1, null);
+ this.store.compact(requestCompaction, NoLimitCompactionThroughputController.INSTANCE);
+ Assert.assertEquals(1, this.store.getStorefiles().size());
+
+ //Check encryption after compaction
+ checkMobHFileEncrytption(this.store.getStorefiles());
+ }
+
+ private void checkMobHFileEncrytption(Collection<StoreFile> storefiles) {
+ StoreFile storeFile = storefiles.iterator().next();
+ HFile.Reader reader = storeFile.getReader().getHFileReader();
+ byte[] encryptionKey = reader.getTrailer().getEncryptionKey();
+ Assert.assertTrue(null != encryptionKey);
+ Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName()
+ .equals(HConstants.CIPHER_AES));
+ }
+
}