You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by wi...@apache.org on 2022/07/27 06:33:33 UTC
[orc] branch main updated: ORC-1230: Move encryption utility functions to `HadoopShimsCurrent`
This is an automated email from the ASF dual-hosted git repository.
william pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/orc.git
The following commit(s) were added to refs/heads/main by this push:
new 5378cc9b1 ORC-1230: Move encryption utility functions to `HadoopShimsCurrent`
5378cc9b1 is described below
commit 5378cc9b19d7e9213eb7b48fb1a42eb1259e6b1b
Author: William Hyun <wi...@apache.org>
AuthorDate: Tue Jul 26 23:33:25 2022 -0700
ORC-1230: Move encryption utility functions to `HadoopShimsCurrent`
### What changes were proposed in this pull request?
This PR aims to move encryption utility functions to `HadoopShimsCurrent`.
### Why are the changes needed?
To clean up old Hadoop dependencies.
### How was this patch tested?
Pass the CIs.
Closes #1199 from williamhyun/move3.
Authored-by: William Hyun <wi...@apache.org>
Signed-off-by: William Hyun <wi...@apache.org>
---
.../org/apache/orc/impl/HadoopShimsCurrent.java | 56 ++++++++++++++++++++--
.../org/apache/orc/impl/HadoopShimsPre2_7.java | 46 +-----------------
.../java/org/apache/orc/impl/KeyProviderImpl.java | 6 +--
.../org/apache/orc/impl/TestHadoopShimsPre2_7.java | 8 ++--
4 files changed, 60 insertions(+), 56 deletions(-)
diff --git a/java/shims/src/java/org/apache/orc/impl/HadoopShimsCurrent.java b/java/shims/src/java/org/apache/orc/impl/HadoopShimsCurrent.java
index a80245702..1eed63b8e 100644
--- a/java/shims/src/java/org/apache/orc/impl/HadoopShimsCurrent.java
+++ b/java/shims/src/java/org/apache/orc/impl/HadoopShimsCurrent.java
@@ -19,27 +19,34 @@
package org.apache.orc.impl;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.crypto.key.KeyProviderFactory;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.io.compress.snappy.SnappyDecompressor;
import org.apache.hadoop.io.compress.zlib.ZlibDecompressor;
+import org.apache.orc.EncryptionAlgorithm;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.OutputStream;
import java.util.EnumSet;
+import java.util.List;
import java.util.Random;
/**
* Shims for recent versions of Hadoop
- *
+ * <p>
* Adds support for:
* <ul>
* <li>Variable length HDFS blocks</li>
* </ul>
*/
public class HadoopShimsCurrent implements HadoopShims {
+ private static final Logger LOG = LoggerFactory.getLogger(HadoopShimsCurrent.class);
- static DirectDecompressor getDecompressor( DirectCompressionType codec) {
+ static DirectDecompressor getDecompressor(DirectCompressionType codec) {
switch (codec) {
case ZLIB:
return new ZlibDirectDecompressWrapper
@@ -56,6 +63,47 @@ public class HadoopShimsCurrent implements HadoopShims {
}
}
+ /**
+ * Find the correct algorithm based on the key's metadata.
+ *
+ * @param meta the key's metadata
+ * @return the correct algorithm
+ */
+ static EncryptionAlgorithm findAlgorithm(KeyProviderCryptoExtension.Metadata meta) {
+ String cipher = meta.getCipher();
+ if (cipher.startsWith("AES/")) {
+ int bitLength = meta.getBitLength();
+ if (bitLength == 128) {
+ return EncryptionAlgorithm.AES_CTR_128;
+ } else {
+ if (bitLength != 256) {
+ LOG.info("ORC column encryption does not support " + bitLength +
+ " bit keys. Using 256 bits instead.");
+ }
+ return EncryptionAlgorithm.AES_CTR_256;
+ }
+ }
+ throw new IllegalArgumentException("ORC column encryption only supports" +
+ " AES and not " + cipher);
+ }
+
+ static String buildKeyVersionName(KeyMetadata key) {
+ return key.getKeyName() + "@" + key.getVersion();
+ }
+
+ static KeyProvider createKeyProvider(Configuration conf,
+ Random random) throws IOException {
+ List<org.apache.hadoop.crypto.key.KeyProvider> result =
+ KeyProviderFactory.getProviders(conf);
+ if (result.size() == 0) {
+ LOG.info("Can't get KeyProvider for ORC encryption from" +
+ " hadoop.security.key.provider.path.");
+ return new NullKeyProvider();
+ } else {
+ return new KeyProviderImpl(result.get(0), random);
+ }
+ }
+
@Override
public DirectDecompressor getDirectDecompressor(DirectCompressionType codec) {
return getDecompressor(codec);
@@ -64,7 +112,7 @@ public class HadoopShimsCurrent implements HadoopShims {
@Override
public ZeroCopyReaderShim getZeroCopyReader(FSDataInputStream in,
ByteBufferPoolShim pool
- ) throws IOException {
+ ) throws IOException {
return ZeroCopyShims.getZeroCopyReader(in, pool);
}
@@ -81,6 +129,6 @@ public class HadoopShimsCurrent implements HadoopShims {
@Override
public KeyProvider getHadoopKeyProvider(Configuration conf,
Random random) throws IOException {
- return HadoopShimsPre2_7.createKeyProvider(conf, random);
+ return createKeyProvider(conf, random);
}
}
diff --git a/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_7.java b/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_7.java
index a7542d30b..00210046a 100644
--- a/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_7.java
+++ b/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_7.java
@@ -19,16 +19,12 @@
package org.apache.orc.impl;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
-import org.apache.hadoop.crypto.key.KeyProviderFactory;
import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.orc.EncryptionAlgorithm;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.OutputStream;
-import java.util.List;
import java.util.Random;
/**
@@ -62,49 +58,9 @@ public class HadoopShimsPre2_7 implements HadoopShims {
return false;
}
- static String buildKeyVersionName(KeyMetadata key) {
- return key.getKeyName() + "@" + key.getVersion();
- }
-
- static KeyProvider createKeyProvider(Configuration conf,
- Random random) throws IOException {
- List<org.apache.hadoop.crypto.key.KeyProvider> result =
- KeyProviderFactory.getProviders(conf);
- if (result.size() == 0) {
- LOG.info("Can't get KeyProvider for ORC encryption from" +
- " hadoop.security.key.provider.path.");
- return new NullKeyProvider();
- } else {
- return new KeyProviderImpl(result.get(0), random);
- }
- }
-
- /**
- * Find the correct algorithm based on the key's metadata.
- * @param meta the key's metadata
- * @return the correct algorithm
- */
- static EncryptionAlgorithm findAlgorithm(KeyProviderCryptoExtension.Metadata meta) {
- String cipher = meta.getCipher();
- if (cipher.startsWith("AES/")) {
- int bitLength = meta.getBitLength();
- if (bitLength == 128) {
- return EncryptionAlgorithm.AES_CTR_128;
- } else {
- if (bitLength != 256) {
- LOG.info("ORC column encryption does not support " + bitLength +
- " bit keys. Using 256 bits instead.");
- }
- return EncryptionAlgorithm.AES_CTR_256;
- }
- }
- throw new IllegalArgumentException("ORC column encryption only supports" +
- " AES and not " + cipher);
- }
-
@Override
public KeyProvider getHadoopKeyProvider(Configuration conf,
Random random) throws IOException {
- return createKeyProvider(conf, random);
+ return HadoopShimsCurrent.createKeyProvider(conf, random);
}
}
diff --git a/java/shims/src/java/org/apache/orc/impl/KeyProviderImpl.java b/java/shims/src/java/org/apache/orc/impl/KeyProviderImpl.java
index c1fda7972..c6596eb71 100644
--- a/java/shims/src/java/org/apache/orc/impl/KeyProviderImpl.java
+++ b/java/shims/src/java/org/apache/orc/impl/KeyProviderImpl.java
@@ -100,7 +100,7 @@ class KeyProviderImpl implements KeyProvider {
org.apache.hadoop.crypto.key.KeyProvider.Metadata meta =
provider.getMetadata(keyName);
return new HadoopShims.KeyMetadata(keyName, meta.getVersions() - 1,
- HadoopShimsPre2_7.findAlgorithm(meta));
+ HadoopShimsCurrent.findAlgorithm(meta));
}
/**
@@ -125,7 +125,7 @@ class KeyProviderImpl implements KeyProvider {
byte[] iv = new byte[algorithm.getIvLength()];
unmangleIv(encryptedKey, iv);
EncryptedKeyVersion param = EncryptedKeyVersion.createForDecryption(
- key.getKeyName(), HadoopShimsPre2_7.buildKeyVersionName(key), iv, encryptedKey);
+ key.getKeyName(), HadoopShimsCurrent.buildKeyVersionName(key), iv, encryptedKey);
try {
KeyProviderCryptoExtension.KeyVersion decryptedKey;
if (provider instanceof KeyProviderCryptoExtension) {
@@ -150,7 +150,7 @@ class KeyProviderImpl implements KeyProvider {
byte[] iv = new byte[algorithm.getIvLength()];
unmangleIv(encryptedKey, iv);
EncryptedKeyVersion param = EncryptedKeyVersion.createForDecryption(
- key.getKeyName(), HadoopShimsPre2_7.buildKeyVersionName(key), iv, encryptedKey);
+ key.getKeyName(), HadoopShimsCurrent.buildKeyVersionName(key), iv, encryptedKey);
try {
KeyProviderCryptoExtension.KeyVersion decryptedKey;
if (provider instanceof KeyProviderCryptoExtension) {
diff --git a/java/shims/src/test/org/apache/orc/impl/TestHadoopShimsPre2_7.java b/java/shims/src/test/org/apache/orc/impl/TestHadoopShimsPre2_7.java
index 3a86b2366..3dbd01864 100644
--- a/java/shims/src/test/org/apache/orc/impl/TestHadoopShimsPre2_7.java
+++ b/java/shims/src/test/org/apache/orc/impl/TestHadoopShimsPre2_7.java
@@ -37,7 +37,7 @@ public class TestHadoopShimsPre2_7 {
KeyProvider.Metadata meta = new KMSClientProvider.KMSMetadata(
"XXX/CTR/NoPadding", 128, "", new HashMap<String, String>(),
new Date(0), 1);
- HadoopShimsPre2_7.findAlgorithm(meta);
+ HadoopShimsCurrent.findAlgorithm(meta);
});
}
@@ -47,16 +47,16 @@ public class TestHadoopShimsPre2_7 {
"AES/CTR/NoPadding", 128, "", new HashMap<String, String>(),
new Date(0), 1);
assertEquals(EncryptionAlgorithm.AES_CTR_128,
- HadoopShimsPre2_7.findAlgorithm(meta));
+ HadoopShimsCurrent.findAlgorithm(meta));
meta = new KMSClientProvider.KMSMetadata(
"AES/CTR/NoPadding", 256, "", new HashMap<String, String>(),
new Date(0), 1);
assertEquals(EncryptionAlgorithm.AES_CTR_256,
- HadoopShimsPre2_7.findAlgorithm(meta));
+ HadoopShimsCurrent.findAlgorithm(meta));
meta = new KMSClientProvider.KMSMetadata(
"AES/CTR/NoPadding", 512, "", new HashMap<String, String>(),
new Date(0), 1);
assertEquals(EncryptionAlgorithm.AES_CTR_256,
- HadoopShimsPre2_7.findAlgorithm(meta));
+ HadoopShimsCurrent.findAlgorithm(meta));
}
}