You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ra...@apache.org on 2016/10/24 10:53:37 UTC
hbase git commit: HBASE-16463 Improve transparent table/CF encryption
with Commons Crypto (Dapeng Sun)
Repository: hbase
Updated Branches:
refs/heads/master 97cb1d71b -> 3584537b0
HBASE-16463 Improve transparent table/CF encryption with Commons Crypto
(Dapeng Sun)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3584537b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3584537b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3584537b
Branch: refs/heads/master
Commit: 3584537b0776b7393d24141dd624fa7551591964
Parents: 97cb1d7
Author: Ramkrishna <ra...@intel.com>
Authored: Mon Oct 24 16:22:30 2016 +0530
Committer: Ramkrishna <ra...@intel.com>
Committed: Mon Oct 24 16:22:30 2016 +0530
----------------------------------------------------------------------
.../apache/hadoop/hbase/io/crypto/Cipher.java | 8 +
.../hbase/io/crypto/CryptoCipherProvider.java | 76 +++++++++
.../apache/hadoop/hbase/io/crypto/aes/AES.java | 7 -
.../hbase/io/crypto/aes/CommonsCryptoAES.java | 166 +++++++++++++++++++
.../crypto/aes/CommonsCryptoAESDecryptor.java | 84 ++++++++++
.../crypto/aes/CommonsCryptoAESEncryptor.java | 98 +++++++++++
.../hbase/io/crypto/aes/TestCommonsAES.java | 131 +++++++++++++++
.../hbase/HFilePerformanceEvaluation.java | 47 +++++-
8 files changed, 604 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/3584537b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Cipher.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Cipher.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Cipher.java
index beda267..e19a13d 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Cipher.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Cipher.java
@@ -31,6 +31,14 @@ import org.apache.hadoop.hbase.classification.InterfaceStability;
@InterfaceStability.Evolving
public abstract class Cipher {
+ public static final int KEY_LENGTH = 16;
+ public static final int KEY_LENGTH_BITS = KEY_LENGTH * 8;
+ public static final int BLOCK_SIZE = 16;
+ public static final int IV_LENGTH = 16;
+
+ public static final String RNG_ALGORITHM_KEY = "hbase.crypto.algorithm.rng";
+ public static final String RNG_PROVIDER_KEY = "hbase.crypto.algorithm.rng.provider";
+
private final CipherProvider provider;
public Cipher(CipherProvider provider) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/3584537b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/CryptoCipherProvider.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/CryptoCipherProvider.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/CryptoCipherProvider.java
new file mode 100644
index 0000000..3f5cd2d
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/CryptoCipherProvider.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hbase.io.crypto;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.io.crypto.aes.CommonsCryptoAES;
+
+/**
+ * The default cipher provider. Supports AES via the Commons Crypto.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class CryptoCipherProvider implements CipherProvider {
+
+ private static CryptoCipherProvider instance;
+
+ public static CryptoCipherProvider getInstance() {
+ if (instance != null) {
+ return instance;
+ }
+ instance = new CryptoCipherProvider();
+ return instance;
+ }
+
+ private Configuration conf = HBaseConfiguration.create();
+
+ // Prevent instantiation
+ private CryptoCipherProvider() { }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ @Override
+ public String getName() {
+ return "commons";
+ }
+
+ @Override
+ public Cipher getCipher(String name) {
+ if (name.equalsIgnoreCase("AES")) {
+ return new CommonsCryptoAES(this);
+ }
+ throw new RuntimeException("Cipher '" + name + "' is not supported by provider '" +
+ getName() + "'");
+ }
+
+ @Override
+ public String[] getSupportedCiphers() {
+ return new String[] { "AES" };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3584537b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java
index 03e3161..302091f 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java
@@ -51,15 +51,8 @@ public class AES extends Cipher {
private static final Log LOG = LogFactory.getLog(AES.class);
- public static final int KEY_LENGTH = 16;
- public static final int KEY_LENGTH_BITS = KEY_LENGTH * 8;
- public static final int BLOCK_SIZE = 16;
- public static final int IV_LENGTH = 16;
-
public static final String CIPHER_MODE_KEY = "hbase.crypto.algorithm.aes.mode";
public static final String CIPHER_PROVIDER_KEY = "hbase.crypto.algorithm.aes.provider";
- public static final String RNG_ALGORITHM_KEY = "hbase.crypto.algorithm.rng";
- public static final String RNG_PROVIDER_KEY = "hbase.crypto.algorithm.rng.provider";
private final String rngAlgorithm;
private final String cipherMode;
http://git-wip-us.apache.org/repos/asf/hbase/blob/3584537b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAES.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAES.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAES.java
new file mode 100644
index 0000000..7ea4e63
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAES.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.crypto.aes;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.GeneralSecurityException;
+import java.security.Key;
+import java.security.SecureRandom;
+import java.util.Properties;
+
+import javax.crypto.spec.SecretKeySpec;
+
+import org.apache.commons.crypto.cipher.CryptoCipherFactory;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.io.crypto.Cipher;
+import org.apache.hadoop.hbase.io.crypto.CipherProvider;
+import org.apache.hadoop.hbase.io.crypto.Context;
+import org.apache.hadoop.hbase.io.crypto.Decryptor;
+import org.apache.hadoop.hbase.io.crypto.Encryptor;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class CommonsCryptoAES extends Cipher {
+
+ private static final Log LOG = LogFactory.getLog(CommonsCryptoAES.class);
+
+ public static final String CIPHER_MODE_KEY = "hbase.crypto.commons.mode";
+ public static final String CIPHER_CLASSES_KEY = "hbase.crypto.commons.cipher.classes";
+ public static final String CIPHER_JCE_PROVIDER_KEY = "hbase.crypto.commons.cipher.jce.provider";
+
+ private final String cipherMode;
+ private Properties props;
+ private final String rngAlgorithm;
+ private SecureRandom rng;
+
+ public CommonsCryptoAES(CipherProvider provider) {
+ super(provider);
+ // The mode for Commons Crypto Ciphers
+ cipherMode = provider.getConf().get(CIPHER_MODE_KEY, "AES/CTR/NoPadding");
+ // Reads Commons Crypto properties from HBase conf
+ props = readCryptoProps(provider.getConf());
+ // RNG algorithm
+ rngAlgorithm = provider.getConf().get(RNG_ALGORITHM_KEY, "SHA1PRNG");
+ // RNG provider, null if default
+ String rngProvider = provider.getConf().get(RNG_PROVIDER_KEY);
+ try {
+ if (rngProvider != null) {
+ rng = SecureRandom.getInstance(rngAlgorithm, rngProvider);
+ } else {
+ rng = SecureRandom.getInstance(rngAlgorithm);
+ }
+ } catch (GeneralSecurityException e) {
+ LOG.warn("Could not instantiate specified RNG, falling back to default", e);
+ rng = new SecureRandom();
+ }
+ }
+
+ private static Properties readCryptoProps(Configuration conf) {
+ Properties props = new Properties();
+
+ props.setProperty(CryptoCipherFactory.CLASSES_KEY, conf.get(CIPHER_CLASSES_KEY, ""));
+ props.setProperty(CryptoCipherFactory.JCE_PROVIDER_KEY, conf.get(CIPHER_JCE_PROVIDER_KEY, ""));
+
+ return props;
+ }
+
+ @Override
+ public String getName() {
+ return "AES";
+ }
+
+ @Override
+ public int getKeyLength() {
+ return KEY_LENGTH;
+ }
+
+ @Override
+ public int getIvLength() {
+ return IV_LENGTH;
+ }
+
+ @Override
+ public Key getRandomKey() {
+ byte[] keyBytes = new byte[getKeyLength()];
+ rng.nextBytes(keyBytes);
+ return new SecretKeySpec(keyBytes, getName());
+ }
+
+ @Override
+ public Encryptor getEncryptor() {
+ return new CommonsCryptoAESEncryptor(cipherMode, props, rng);
+ }
+
+ @Override
+ public Decryptor getDecryptor() {
+ return new CommonsCryptoAESDecryptor(cipherMode, props);
+ }
+
+ @Override
+ public OutputStream createEncryptionStream(OutputStream out, Context context,
+ byte[] iv) throws IOException {
+ Preconditions.checkNotNull(context);
+ Preconditions.checkState(context.getKey() != null, "Context does not have a key");
+ Preconditions.checkNotNull(iv);
+ Encryptor e = getEncryptor();
+ e.setKey(context.getKey());
+ e.setIv(iv);
+ return e.createEncryptionStream(out);
+ }
+
+ @Override
+ public OutputStream createEncryptionStream(OutputStream out,
+ Encryptor encryptor) throws
+ IOException {
+ return encryptor.createEncryptionStream(out);
+ }
+
+ @Override
+ public InputStream createDecryptionStream(InputStream in, Context context,
+ byte[] iv) throws IOException {
+ Preconditions.checkNotNull(context);
+ Preconditions.checkState(context.getKey() != null, "Context does not have a key");
+ Preconditions.checkNotNull(iv);
+ Decryptor d = getDecryptor();
+ d.setKey(context.getKey());
+ d.setIv(iv);
+ return d.createDecryptionStream(in);
+ }
+
+ @Override
+ public InputStream createDecryptionStream(InputStream in,
+ Decryptor decryptor) throws
+ IOException {
+ Preconditions.checkNotNull(decryptor);
+ return decryptor.createDecryptionStream(in);
+ }
+
+ @VisibleForTesting
+ SecureRandom getRNG() {
+ return rng;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3584537b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAESDecryptor.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAESDecryptor.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAESDecryptor.java
new file mode 100644
index 0000000..e33e366
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAESDecryptor.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.crypto.aes;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.crypto.stream.CryptoInputStream;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.io.crypto.Decryptor;
+
+import javax.crypto.spec.IvParameterSpec;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.Key;
+import java.util.Properties;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class CommonsCryptoAESDecryptor implements Decryptor {
+
+ private String cipherMode;
+ private Properties properties;
+ private Key key;
+ private byte[] iv;
+
+ public CommonsCryptoAESDecryptor(String cipherMode, Properties properties) {
+ this.cipherMode = cipherMode;
+ this.properties = properties;
+ }
+
+ @Override
+ public void setKey(Key key) {
+ Preconditions.checkNotNull(key, "Key cannot be null");
+ this.key = key;
+ }
+
+ @Override
+ public int getIvLength() {
+ return CommonsCryptoAES.IV_LENGTH;
+ }
+
+ @Override
+ public int getBlockSize() {
+ return CommonsCryptoAES.BLOCK_SIZE;
+ }
+
+ @Override
+ public void setIv(byte[] iv) {
+ Preconditions.checkNotNull(iv, "IV cannot be null");
+ Preconditions.checkArgument(iv.length == CommonsCryptoAES.IV_LENGTH, "Invalid IV length");
+ this.iv = iv;
+ }
+
+ @Override
+ public InputStream createDecryptionStream(InputStream in) {
+ try {
+ return new CryptoInputStream(cipherMode, properties, in, key, new
+ IvParameterSpec(iv));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void reset() {
+ ;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3584537b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAESEncryptor.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAESEncryptor.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAESEncryptor.java
new file mode 100644
index 0000000..346cd79
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAESEncryptor.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.crypto.aes;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.crypto.stream.CryptoOutputStream;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.io.crypto.Encryptor;
+
+import javax.crypto.spec.IvParameterSpec;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.security.Key;
+import java.security.SecureRandom;
+import java.util.Properties;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class CommonsCryptoAESEncryptor implements Encryptor {
+
+ private String cipherMode;
+ private Properties properties;
+ private Key key;
+ private byte[] iv;
+ private boolean initialized = false;
+ private SecureRandom rng;
+
+ public CommonsCryptoAESEncryptor(String cipherMode, Properties properties, SecureRandom rng) {
+ this.cipherMode = cipherMode;
+ this.properties = properties;
+ this.rng = rng;
+ }
+
+ @Override
+ public void setKey(Key key) {
+ this.key = key;
+ }
+
+ @Override
+ public int getIvLength() {
+ return CommonsCryptoAES.IV_LENGTH;
+ }
+
+ @Override
+ public int getBlockSize() {
+ return CommonsCryptoAES.BLOCK_SIZE;
+ }
+
+ @Override
+ public byte[] getIv() {
+ return iv;
+ }
+
+ @Override
+ public void setIv(byte[] iv) {
+ Preconditions.checkNotNull(iv, "IV cannot be null");
+ Preconditions.checkArgument(iv.length == CommonsCryptoAES.IV_LENGTH, "Invalid IV length");
+ this.iv = iv;
+ }
+
+ @Override
+ public OutputStream createEncryptionStream(OutputStream out) {
+ if (!initialized) {
+ reset();
+ }
+ try {
+ return new CryptoOutputStream(cipherMode, properties, out, key, new
+ IvParameterSpec(iv));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void reset() {
+ if (iv == null) {
+ iv = new byte[getIvLength()];
+ rng.nextBytes(iv);
+ }
+ initialized = true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3584537b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/aes/TestCommonsAES.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/aes/TestCommonsAES.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/aes/TestCommonsAES.java
new file mode 100644
index 0000000..dca62e5
--- /dev/null
+++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/aes/TestCommonsAES.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hbase.io.crypto.aes;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.io.crypto.Cipher;
+import org.apache.hadoop.hbase.io.crypto.DefaultCipherProvider;
+import org.apache.hadoop.hbase.io.crypto.Encryption;
+import org.apache.hadoop.hbase.io.crypto.Encryptor;
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import javax.crypto.spec.SecretKeySpec;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
+import java.security.*;
+
+import static org.junit.Assert.*;
+
+@Category({MiscTests.class, SmallTests.class})
+public class TestCommonsAES {
+
+ // Validation for AES in CTR mode with a 128 bit key
+ // From NIST Special Publication 800-38A
+ @Test
+ public void testAESAlgorithm() throws Exception {
+ Configuration conf = HBaseConfiguration.create();
+ Cipher aes = Encryption.getCipher(conf, "AES");
+ assertEquals(aes.getKeyLength(), CommonsCryptoAES.KEY_LENGTH);
+ assertEquals(aes.getIvLength(), CommonsCryptoAES.IV_LENGTH);
+ Encryptor e = aes.getEncryptor();
+ e.setKey(new SecretKeySpec(Bytes.fromHex("2b7e151628aed2a6abf7158809cf4f3c"), "AES"));
+ e.setIv(Bytes.fromHex("f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"));
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ OutputStream cout = e.createEncryptionStream(out);
+ cout.write(Bytes.fromHex("6bc1bee22e409f96e93d7e117393172a"));
+ cout.write(Bytes.fromHex("ae2d8a571e03ac9c9eb76fac45af8e51"));
+ cout.write(Bytes.fromHex("30c81c46a35ce411e5fbc1191a0a52ef"));
+ cout.write(Bytes.fromHex("f69f2445df4f9b17ad2b417be66c3710"));
+ cout.close();
+
+ ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+ byte[] b = new byte[16];
+ IOUtils.readFully(in, b);
+ assertTrue("Failed #1", Bytes.equals(b, Bytes.fromHex("874d6191b620e3261bef6864990db6ce")));
+ IOUtils.readFully(in, b);
+ assertTrue("Failed #2", Bytes.equals(b, Bytes.fromHex("9806f66b7970fdff8617187bb9fffdff")));
+ IOUtils.readFully(in, b);
+ assertTrue("Failed #3", Bytes.equals(b, Bytes.fromHex("5ae4df3edbd5d35e5b4f09020db03eab")));
+ IOUtils.readFully(in, b);
+ assertTrue("Failed #4", Bytes.equals(b, Bytes.fromHex("1e031dda2fbe03d1792170a0f3009cee")));
+ }
+
+ @Test
+ public void testAlternateRNG() throws Exception {
+ Security.addProvider(new TestProvider());
+
+ Configuration conf = new Configuration();
+ conf.set(AES.RNG_ALGORITHM_KEY, "TestRNG");
+ conf.set(AES.RNG_PROVIDER_KEY, "TEST");
+ DefaultCipherProvider.getInstance().setConf(conf);
+
+ AES aes = new AES(DefaultCipherProvider.getInstance());
+ assertEquals("AES did not find alternate RNG", aes.getRNG().getAlgorithm(),
+ "TestRNG");
+ }
+
+ static class TestProvider extends Provider {
+ private static final long serialVersionUID = 1L;
+ public TestProvider() {
+ super("TEST", 1.0, "Test provider");
+ AccessController.doPrivileged(new PrivilegedAction<Object>() {
+ public Object run() {
+ put("SecureRandom.TestRNG", TestCommonsAES.class.getName() + "$TestRNG");
+ return null;
+ }
+ });
+ }
+ }
+
+ // Must be public for instantiation by the SecureRandom SPI
+ public static class TestRNG extends SecureRandomSpi {
+ private static final long serialVersionUID = 1L;
+ private SecureRandom rng;
+
+ public TestRNG() {
+ try {
+ rng = SecureRandom.getInstance("SHA1PRNG");
+ } catch (NoSuchAlgorithmException e) {
+ fail("Unable to create SecureRandom instance");
+ }
+ }
+
+ @Override
+ protected void engineSetSeed(byte[] seed) {
+ rng.setSeed(seed);
+ }
+
+ @Override
+ protected void engineNextBytes(byte[] bytes) {
+ rng.nextBytes(bytes);
+ }
+
+ @Override
+ protected byte[] engineGenerateSeed(int numBytes) {
+ return rng.generateSeed(numBytes);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3584537b/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
index e5aec57..562630a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
@@ -31,6 +31,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.io.crypto.CryptoCipherProvider;
+import org.apache.hadoop.hbase.io.crypto.DefaultCipherProvider;
import org.apache.hadoop.hbase.io.crypto.Encryption;
import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
import org.apache.hadoop.hbase.io.crypto.aes.AES;
@@ -130,6 +132,23 @@ public class HFilePerformanceEvaluation {
runWriteBenchmark(aesconf, aesfs, aesmf, "gz", "aes");
runReadBenchmark(aesconf, aesfs, aesmf, "gz", "aes");
+ // Add configuration for Commons cipher
+ final Configuration cryptoconf = new Configuration();
+ cryptoconf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
+ cryptoconf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
+ cryptoconf.setInt("hfile.format.version", 3);
+ cryptoconf.set(HConstants.CRYPTO_CIPHERPROVIDER_CONF_KEY, CryptoCipherProvider.class.getName());
+ final FileSystem cryptofs = FileSystem.get(cryptoconf);
+ final Path cryptof = cryptofs.makeQualified(new Path("performanceevaluation.aes.mapfile"));
+
+ // codec=none cipher=aes
+ runWriteBenchmark(cryptoconf, cryptofs, aesmf, "none", "aes");
+ runReadBenchmark(cryptoconf, cryptofs, aesmf, "none", "aes");
+
+ // codec=gz cipher=aes
+ runWriteBenchmark(cryptoconf, aesfs, aesmf, "gz", "aes");
+ runReadBenchmark(cryptoconf, aesfs, aesmf, "gz", "aes");
+
// cleanup test files
if (fs.exists(mf)) {
fs.delete(mf, true);
@@ -137,7 +156,10 @@ public class HFilePerformanceEvaluation {
if (aesfs.exists(aesmf)) {
aesfs.delete(aesmf, true);
}
-
+ if (cryptofs.exists(aesmf)) {
+ cryptofs.delete(cryptof, true);
+ }
+
// Print Result Summary
LOG.info("\n***************\n" + "Result Summary" + "\n***************\n");
LOG.info(testSummary.toString());
@@ -160,7 +182,7 @@ public class HFilePerformanceEvaluation {
}
runBenchmark(new SequentialWriteBenchmark(conf, fs, mf, ROW_COUNT, codec, cipher),
- ROW_COUNT, codec, cipher);
+ ROW_COUNT, codec, getCipherName(conf, cipher));
}
@@ -179,7 +201,7 @@ public class HFilePerformanceEvaluation {
public void run() {
try {
runBenchmark(new UniformRandomSmallScan(conf, fs, mf, ROW_COUNT),
- ROW_COUNT, codec, cipher);
+ ROW_COUNT, codec, getCipherName(conf, cipher));
} catch (Exception e) {
testSummary.append("UniformRandomSmallScan failed " + e.getMessage());
e.printStackTrace();
@@ -192,7 +214,7 @@ public class HFilePerformanceEvaluation {
public void run() {
try {
runBenchmark(new UniformRandomReadBenchmark(conf, fs, mf, ROW_COUNT),
- ROW_COUNT, codec, cipher);
+ ROW_COUNT, codec, getCipherName(conf, cipher));
} catch (Exception e) {
testSummary.append("UniformRandomReadBenchmark failed " + e.getMessage());
e.printStackTrace();
@@ -205,7 +227,7 @@ public class HFilePerformanceEvaluation {
public void run() {
try {
runBenchmark(new GaussianRandomReadBenchmark(conf, fs, mf, ROW_COUNT),
- ROW_COUNT, codec, cipher);
+ ROW_COUNT, codec, getCipherName(conf, cipher));
} catch (Exception e) {
testSummary.append("GaussianRandomReadBenchmark failed " + e.getMessage());
e.printStackTrace();
@@ -218,7 +240,7 @@ public class HFilePerformanceEvaluation {
public void run() {
try {
runBenchmark(new SequentialReadBenchmark(conf, fs, mf, ROW_COUNT),
- ROW_COUNT, codec, cipher);
+ ROW_COUNT, codec, getCipherName(conf, cipher));
} catch (Exception e) {
testSummary.append("SequentialReadBenchmark failed " + e.getMessage());
e.printStackTrace();
@@ -530,4 +552,17 @@ public class HFilePerformanceEvaluation {
public static void main(String[] args) throws Exception {
new HFilePerformanceEvaluation().runBenchmarks();
}
+
+ private String getCipherName(Configuration conf, String cipherName) {
+ if (cipherName.equals("aes")) {
+ String provider = conf.get(HConstants.CRYPTO_CIPHERPROVIDER_CONF_KEY);
+ if (provider == null || provider.equals("")
+ || provider.equals(DefaultCipherProvider.class.getName())) {
+ return "aes-default";
+ } else if (provider.equals(CryptoCipherProvider.class.getName())) {
+ return "aes-commons";
+ }
+ }
+ return cipherName;
+ }
}