You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ve...@apache.org on 2015/03/24 18:27:42 UTC
[3/4] incubator-ranger git commit: (RANGER-247)Development of Ranger
Key Storage Provider
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java
new file mode 100644
index 0000000..274b5f8
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java
@@ -0,0 +1,341 @@
+package org.apache.hadoop.crypto.key;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URL;
+import java.security.Key;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.crypto.spec.SecretKeySpec;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderFactory;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.apache.hadoop.fs.Path;
+import org.apache.ranger.kms.dao.DaoManager;
+
+public class RangerKeyStoreProvider extends KeyProvider{
+
+ public static final String SCHEME_NAME = "dbks";
+ public static final String KMS_CONFIG_DIR = "kms.config.dir";
+ public static final String DBKS_SITE_XML = "dbks-site.xml";
+ public static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password";
+ private static final String KEY_METADATA = "KeyMetadata";
+ private final RangerKeyStore dbStore;
+ private char[] masterKey;
+ private boolean changed = false;
+ private final Map<String, Metadata> cache = new HashMap<String, Metadata>();
+ private DaoManager daoManager;
+
+ public RangerKeyStoreProvider(Configuration conf) throws Throwable {
+ super(conf);
+ conf = getDBKSConf();
+ RangerKMSDB rangerKMSDB = new RangerKMSDB(conf);
+ daoManager = rangerKMSDB.getDaoManager();
+ RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager);
+ dbStore = new RangerKeyStore(daoManager);
+ String password = conf.get(ENCRYPTION_KEY);
+ rangerMasterKey.generateMasterKey(password);
+ //code to retrieve rangerMasterKey password
+ masterKey = rangerMasterKey.getMasterKey(password).toCharArray();
+ if(masterKey == null){
+ // Master Key does not exists
+ throw new IOException("Ranger MasterKey does not exists");
+ }
+ try {
+ loadKeys(masterKey);
+ } catch (NoSuchAlgorithmException e) {
+ e.printStackTrace();
+ throw new IOException("Can't load Keys");
+ }catch(CertificateException e){
+ e.printStackTrace();
+ throw new IOException("Can't load Keys");
+ }
+ }
+
+ public static Configuration getDBKSConf() {
+ return getConfiguration(true, DBKS_SITE_XML);
+ }
+
+ static Configuration getConfiguration(boolean loadHadoopDefaults,
+ String ... resources) {
+ Configuration conf = new Configuration(loadHadoopDefaults);
+ String confDir = System.getProperty(KMS_CONFIG_DIR);
+ if (confDir != null) {
+ try {
+ Path confPath = new Path(confDir);
+ if (!confPath.isUriPathAbsolute()) {
+ throw new RuntimeException("System property '" + KMS_CONFIG_DIR +
+ "' must be an absolute path: " + confDir);
+ }
+ for (String resource : resources) {
+ conf.addResource(new URL("file://" + new Path(confDir, resource).toUri()));
+ }
+ } catch (MalformedURLException ex) {
+ ex.printStackTrace();
+ throw new RuntimeException(ex);
+ }
+ } else {
+ for (String resource : resources) {
+ conf.addResource(resource);
+ }
+ }
+ return conf;
+ }
+
+ private void loadKeys(char[] masterKey) throws NoSuchAlgorithmException, CertificateException, IOException {
+ dbStore.engineLoad(null, masterKey);
+ }
+
+ @Override
+ public KeyVersion createKey(String name, byte[] material, Options options)
+ throws IOException {
+ if (dbStore.engineContainsAlias(name) || cache.containsKey(name)) {
+ throw new IOException("Key " + name + " already exists");
+ }
+ Metadata meta = new Metadata(options.getCipher(), options.getBitLength(),
+ options.getDescription(), options.getAttributes(), new Date(), 1);
+ if (options.getBitLength() != 8 * material.length) {
+ throw new IOException("Wrong key length. Required " +
+ options.getBitLength() + ", but got " + (8 * material.length));
+ }
+ cache.put(name, meta);
+ String versionName = buildVersionName(name, 0);
+ return innerSetKeyVersion(name, versionName, material, meta.getCipher(), meta.getBitLength(), meta.getDescription(), meta.getVersions(), meta.getAttributes());
+ }
+
+ KeyVersion innerSetKeyVersion(String name, String versionName, byte[] material, String cipher, int bitLength, String description, int version, Map<String, String> attributes) throws IOException {
+ try {
+ ObjectMapper om = new ObjectMapper();
+ String attribute = om.writeValueAsString(attributes);
+ dbStore.engineSetKeyEntry(versionName, new SecretKeySpec(material, cipher), masterKey, cipher, bitLength, description, version, attribute);
+ } catch (KeyStoreException e) {
+ e.printStackTrace();
+ throw new IOException("Can't store key " + versionName,e);
+ }
+ changed = true;
+ return new KeyVersion(name, versionName, material);
+ }
+
+ @Override
+ public void deleteKey(String name) throws IOException {
+ Metadata meta = getMetadata(name);
+ if (meta == null) {
+ throw new IOException("Key " + name + " does not exist");
+ }
+ for(int v=0; v < meta.getVersions(); ++v) {
+ String versionName = buildVersionName(name, v);
+ try {
+ if (dbStore.engineContainsAlias(versionName)) {
+ dbStore.engineDeleteEntry(versionName);
+ }
+ } catch (KeyStoreException e) {
+ throw new IOException("Problem removing " + versionName, e);
+ }
+ }
+ try {
+ if (dbStore.engineContainsAlias(name)) {
+ dbStore.engineDeleteEntry(name);
+ }
+ } catch (KeyStoreException e) {
+ e.printStackTrace();
+ throw new IOException("Problem removing " + name + " from " + this, e);
+ }
+ cache.remove(name);
+ changed = true;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ try {
+ if (!changed) {
+ return;
+ }
+ // put all of the updates into the db
+ for(Map.Entry<String, Metadata> entry: cache.entrySet()) {
+ try {
+ Metadata metadata = entry.getValue();
+ ObjectMapper om = new ObjectMapper();
+ String attributes = om.writeValueAsString(metadata.getAttributes());
+ dbStore.engineSetKeyEntry(entry.getKey(), new KeyMetadata(metadata), masterKey, metadata.getAlgorithm(), metadata.getBitLength(), metadata.getDescription(), metadata.getVersions(), attributes);
+ } catch (KeyStoreException e) {
+ e.printStackTrace();
+ throw new IOException("Can't set metadata key " + entry.getKey(),e );
+ }
+ }
+ try {
+ dbStore.engineStore(null, masterKey);
+ } catch (NoSuchAlgorithmException e) {
+ e.printStackTrace();
+ throw new IOException("No such algorithm storing key", e);
+ } catch (CertificateException e) {
+ e.printStackTrace();
+ throw new IOException("Certificate exception storing key", e);
+ }
+ changed = false;
+ }catch (IOException ioe) {
+ ioe.printStackTrace();
+ throw ioe;
+ }
+ }
+
+ @Override
+ public KeyVersion getKeyVersion(String versionName) throws IOException {
+ SecretKeySpec key = null;
+ try {
+ if (!dbStore.engineContainsAlias(versionName)) {
+ return null;
+ }
+ key = (SecretKeySpec) dbStore.engineGetKey(versionName, masterKey);
+ } catch (NoSuchAlgorithmException e) {
+ e.printStackTrace();
+ throw new IOException("Can't get algorithm for key " + key, e);
+ } catch (UnrecoverableKeyException e) {
+ e.printStackTrace();
+ throw new IOException("Can't recover key " + key, e);
+ }
+ return new KeyVersion(getBaseName(versionName), versionName, key.getEncoded());
+ }
+
+ @Override
+ public List<KeyVersion> getKeyVersions(String name) throws IOException {
+ List<KeyVersion> list = new ArrayList<KeyVersion>();
+ Metadata km = getMetadata(name);
+ if (km != null) {
+ int latestVersion = km.getVersions();
+ KeyVersion v = null;
+ String versionName = null;
+ for (int i = 0; i < latestVersion; i++) {
+ versionName = buildVersionName(name, i);
+ v = getKeyVersion(versionName);
+ if (v != null) {
+ list.add(v);
+ }
+ }
+ }
+ return list;
+ }
+
+ @Override
+ public List<String> getKeys() throws IOException {
+ ArrayList<String> list = new ArrayList<String>();
+ String alias = null;
+ Enumeration<String> e = dbStore.engineAliases();
+ while (e.hasMoreElements()) {
+ alias = e.nextElement();
+ // only include the metadata key names in the list of names
+ if (!alias.contains("@")) {
+ list.add(alias);
+ }
+ }
+ return list;
+ }
+
+ @Override
+ public Metadata getMetadata(String name) throws IOException {
+ if (cache.containsKey(name)) {
+ return cache.get(name);
+ }
+ try {
+ if (!dbStore.engineContainsAlias(name)) {
+ return null;
+ }
+ Metadata meta = ((KeyMetadata) dbStore.engineGetKey(name, masterKey)).metadata;
+ cache.put(name, meta);
+ return meta;
+ } catch (NoSuchAlgorithmException e) {
+ e.printStackTrace();
+ throw new IOException("Can't get algorithm for " + name, e);
+ } catch (UnrecoverableKeyException e) {
+ e.printStackTrace();
+ throw new IOException("Can't recover key for " + name, e);
+ }
+ }
+
+ @Override
+ public KeyVersion rollNewVersion(String name, byte[] material)throws IOException {
+ Metadata meta = getMetadata(name);
+ if (meta == null) {
+ throw new IOException("Key " + name + " not found");
+ }
+ if (meta.getBitLength() != 8 * material.length) {
+ throw new IOException("Wrong key length. Required " + meta.getBitLength() + ", but got " + (8 * material.length));
+ }
+ int nextVersion = meta.addVersion();
+ String versionName = buildVersionName(name, nextVersion);
+ return innerSetKeyVersion(name, versionName, material, meta.getCipher(), meta.getBitLength(), meta.getDescription(), meta.getVersions(), meta.getAttributes());
+ }
+
+ /**
+ * The factory to create JksProviders, which is used by the ServiceLoader.
+ */
+ public static class Factory extends KeyProviderFactory {
+ @Override
+ public KeyProvider createProvider(URI providerName,
+ Configuration conf) throws IOException {
+ try {
+ if (SCHEME_NAME.equals(providerName.getScheme())) {
+ return new RangerKeyStoreProvider(conf);
+ }
+ } catch (Throwable e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+ }
+
+ /**
+ * An adapter between a KeyStore Key and our Metadata. This is used to store
+ * the metadata in a KeyStore even though isn't really a key.
+ */
+ public static class KeyMetadata implements Key, Serializable {
+ private Metadata metadata;
+ private final static long serialVersionUID = 8405872419967874451L;
+
+ private KeyMetadata(Metadata meta) {
+ this.metadata = meta;
+ }
+
+ @Override
+ public String getAlgorithm() {
+ return metadata.getCipher();
+ }
+
+ @Override
+ public String getFormat() {
+ return KEY_METADATA;
+ }
+
+ @Override
+ public byte[] getEncoded() {
+ return new byte[0];
+ }
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ byte[] serialized = metadata.serialize();
+ out.writeInt(serialized.length);
+ out.write(serialized);
+ }
+
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ byte[] buf = new byte[in.readInt()];
+ in.readFully(buf);
+ metadata = new Metadata(buf);
+ }
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java
new file mode 100644
index 0000000..6102bfc
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java
@@ -0,0 +1,210 @@
+package org.apache.hadoop.crypto.key;
+
+import java.security.Key;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import javax.crypto.Cipher;
+import javax.crypto.KeyGenerator;
+import javax.crypto.SecretKey;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.PBEKeySpec;
+import javax.crypto.spec.PBEParameterSpec;
+import javax.crypto.spec.SecretKeySpec;
+import org.apache.log4j.Logger;
+import org.apache.ranger.kms.dao.DaoManager;
+import org.apache.ranger.kms.dao.RangerMasterKeyDao;
+import org.apache.ranger.entity.XXRangerMasterKey;
+
+import com.sun.org.apache.xml.internal.security.exceptions.Base64DecodingException;
+import com.sun.org.apache.xml.internal.security.utils.Base64;
+
+public class RangerMasterKey {
+
+ static final Logger logger = Logger.getLogger(RangerMasterKey.class);
+
+ private static final String MK_CIPHER = "AES";
+ private static final int MK_KeySize = 256;
+ private static final int SALT_SIZE = 8;
+ private static final String PBE_ALGO = "PBEWithMD5AndTripleDES";
+ private static final String MD_ALGO = "MD5";
+
+ public static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password";
+
+ private DaoManager daoManager;
+
+ public RangerMasterKey() {
+ }
+
+ public RangerMasterKey(DaoManager daoManager) {
+ this.daoManager = daoManager;
+ }
+
+ /**
+ * To get Master Key
+ * @param password password to be used for decryption
+ * @return Decrypted Master Key
+ * @throws Throwable
+ */
+ public String getMasterKey(String password) throws Throwable{
+ logger.info("Getting Master Key");
+ byte masterKeyByte[] = getEncryptedMK();
+ if(masterKeyByte != null && masterKeyByte.length > 0){
+ String masterKey = decryptMasterKey(masterKeyByte, password);
+ return masterKey;
+ }else{
+ throw new Exception("No Master Key Found");
+ }
+ }
+
+ /**
+ * Generate the master key encrypt's it and save it in database
+ * @param password password to be used for encryption
+ * @return true if successfully created the master key
+ * false if master key generation was unsuccessful or already master key exists
+ * @throws Throwable
+ */
+ public boolean generateMasterKey(String password) throws Throwable{
+ logger.info("Generating Master Key");
+ String encryptedMasterKey = encryptMasterKey(password);
+ String savedKey = saveEncryptedMK(encryptedMasterKey, daoManager);
+ if(savedKey != null && !savedKey.trim().equals("")){
+ logger.debug("Master Key Created with id = "+savedKey);
+ return true;
+ }
+ return false;
+ }
+
+ private String decryptMasterKey(byte masterKey[], String password) throws Throwable {
+ logger.debug("Decrypting Master Key");
+ PBEKeySpec pbeKeyspec = getPBEParameterSpec(password) ;
+ byte[] masterKeyFromDBDecrypted = decryptKey(masterKey, pbeKeyspec) ;
+ SecretKey masterKeyFromDB = getMasterKeyFromBytes(masterKeyFromDBDecrypted) ;
+ return Base64.encode(masterKeyFromDB.getEncoded());
+ }
+
+ private byte[] getEncryptedMK() throws Base64DecodingException {
+ logger.debug("Retrieving Encrypted Master Key from database");
+ try{
+ if(daoManager != null){
+ RangerMasterKeyDao rangerKMSDao = new RangerMasterKeyDao(daoManager);
+ List<XXRangerMasterKey> lstRangerMasterKey = rangerKMSDao.getAll();
+ if(lstRangerMasterKey.size() < 1){
+ throw new Exception("No Master Key exists");
+ }else if(lstRangerMasterKey.size() > 1){
+ throw new Exception("More than one Master Key exists");
+ }else {
+ XXRangerMasterKey rangerMasterKey = rangerKMSDao.getById(lstRangerMasterKey.get(0).getId());
+ String masterKeyStr = rangerMasterKey.getMasterKey();
+ byte[] masterKeyFromDBEncrypted = Base64.decode(masterKeyStr) ;
+ return masterKeyFromDBEncrypted;
+ }
+ }
+ }catch(Exception e){
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ private String saveEncryptedMK(String encryptedMasterKey, DaoManager daoManager) {
+ logger.debug("Saving Encrypted Master Key to database");
+ XXRangerMasterKey xxRangerMasterKey = new XXRangerMasterKey();
+ xxRangerMasterKey.setCipher(MK_CIPHER);
+ xxRangerMasterKey.setBitLength(MK_KeySize);
+ xxRangerMasterKey.setMasterKey(encryptedMasterKey);
+ try{
+ if(daoManager != null){
+ RangerMasterKeyDao rangerKMSDao = new RangerMasterKeyDao(daoManager);
+ Long l = rangerKMSDao.getAllCount();
+ if(l < 1){
+ XXRangerMasterKey rangerMasterKey = rangerKMSDao.create(xxRangerMasterKey);
+ return rangerMasterKey.getId().toString();
+ }
+ }
+ }catch(Exception e){
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ private String encryptMasterKey(String password) throws Throwable {
+ logger.debug("Encrypting Master Key");
+ Key secretKey = generateMasterKey();
+ PBEKeySpec pbeKeySpec = getPBEParameterSpec(password);
+ byte[] masterKeyToDB = encryptKey(secretKey.getEncoded(), pbeKeySpec);
+ String masterKey = Base64.encode(masterKeyToDB) ;
+ return masterKey;
+ }
+
+ private Key generateMasterKey() throws NoSuchAlgorithmException{
+ KeyGenerator kg = KeyGenerator.getInstance(MK_CIPHER);
+ kg.init(MK_KeySize);
+ return kg.generateKey();
+ }
+
+ private PBEKeySpec getPBEParameterSpec(String password) throws Throwable {
+ MessageDigest md = MessageDigest.getInstance(MD_ALGO) ;
+ byte[] saltGen = md.digest(password.getBytes()) ;
+ byte[] salt = new byte[SALT_SIZE] ;
+ System.arraycopy(saltGen, 0, salt, 0, SALT_SIZE);
+ int iteration = password.toCharArray().length + 1 ;
+ PBEKeySpec spec = new PBEKeySpec(password.toCharArray(), salt, iteration) ;
+ return spec ;
+ }
+ private byte[] encryptKey(byte[] data, PBEKeySpec keyspec) throws Throwable {
+ SecretKey key = getPasswordKey(keyspec) ;
+ PBEParameterSpec paramSpec = new PBEParameterSpec(keyspec.getSalt(), keyspec.getIterationCount()) ;
+ Cipher c = Cipher.getInstance(key.getAlgorithm()) ;
+ c.init(Cipher.ENCRYPT_MODE, key,paramSpec);
+ byte[] encrypted = c.doFinal(data) ;
+
+ return encrypted ;
+ }
+ private SecretKey getPasswordKey(PBEKeySpec keyspec) throws Throwable {
+ SecretKeyFactory factory = SecretKeyFactory.getInstance(PBE_ALGO) ;
+ SecretKey PbKey = factory.generateSecret(keyspec) ;
+ return PbKey ;
+ }
+ private byte[] decryptKey(byte[] encrypted, PBEKeySpec keyspec) throws Throwable {
+ SecretKey key = getPasswordKey(keyspec) ;
+ PBEParameterSpec paramSpec = new PBEParameterSpec(keyspec.getSalt(), keyspec.getIterationCount()) ;
+ Cipher c = Cipher.getInstance(key.getAlgorithm()) ;
+ c.init(Cipher.DECRYPT_MODE, key, paramSpec);
+ byte[] data = c.doFinal(encrypted) ;
+ return data ;
+ }
+ private SecretKey getMasterKeyFromBytes(byte[] keyData) throws Throwable {
+ SecretKeySpec sks = new SecretKeySpec(keyData, MK_CIPHER) ;
+ return sks ;
+ }
+
+ public Map<String, String> getPropertiesWithPrefix(Properties props, String prefix) {
+ Map<String, String> prefixedProperties = new HashMap<String, String>();
+
+ if(props != null && prefix != null) {
+ for(String key : props.stringPropertyNames()) {
+ if(key == null) {
+ continue;
+ }
+
+ String val = props.getProperty(key);
+
+ if(key.startsWith(prefix)) {
+ key = key.substring(prefix.length());
+
+ if(key == null) {
+ continue;
+ }
+
+ prefixedProperties.put(key, val);
+ }
+ }
+ }
+
+ return prefixedProperties;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
new file mode 100644
index 0000000..64af2b6
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.crypto.key.kms.server;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.NoSuchAlgorithmException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.crypto.key.kms.ValueQueue;
+import org.apache.hadoop.crypto.key.kms.ValueQueue.SyncGenerationPolicy;
+
+/**
+ * A {@link KeyProviderCryptoExtension} that pre-generates and caches encrypted
+ * keys.
+ */
+@InterfaceAudience.Private
+public class EagerKeyGeneratorKeyProviderCryptoExtension
+ extends KeyProviderCryptoExtension {
+
+ private static final String KEY_CACHE_PREFIX =
+ "hadoop.security.kms.encrypted.key.cache.";
+
+ public static final String KMS_KEY_CACHE_SIZE =
+ KEY_CACHE_PREFIX + "size";
+ public static final int KMS_KEY_CACHE_SIZE_DEFAULT = 100;
+
+ public static final String KMS_KEY_CACHE_LOW_WATERMARK =
+ KEY_CACHE_PREFIX + "low.watermark";
+ public static final float KMS_KEY_CACHE_LOW_WATERMARK_DEFAULT = 0.30f;
+
+ public static final String KMS_KEY_CACHE_EXPIRY_MS =
+ KEY_CACHE_PREFIX + "expiry";
+ public static final int KMS_KEY_CACHE_EXPIRY_DEFAULT = 43200000;
+
+ public static final String KMS_KEY_CACHE_NUM_REFILL_THREADS =
+ KEY_CACHE_PREFIX + "num.fill.threads";
+ public static final int KMS_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT = 2;
+
+
+ private static class CryptoExtension
+ implements KeyProviderCryptoExtension.CryptoExtension {
+
+ private class EncryptedQueueRefiller implements
+ ValueQueue.QueueRefiller<EncryptedKeyVersion> {
+
+ @Override
+ public void fillQueueForKey(String keyName,
+ Queue<EncryptedKeyVersion> keyQueue, int numKeys) throws IOException {
+ List<EncryptedKeyVersion> retEdeks =
+ new LinkedList<EncryptedKeyVersion>();
+ for (int i = 0; i < numKeys; i++) {
+ try {
+ retEdeks.add(keyProviderCryptoExtension.generateEncryptedKey(
+ keyName));
+ } catch (GeneralSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+ keyQueue.addAll(retEdeks);
+ }
+ }
+
+ private KeyProviderCryptoExtension keyProviderCryptoExtension;
+ private final ValueQueue<EncryptedKeyVersion> encKeyVersionQueue;
+
+ public CryptoExtension(Configuration conf,
+ KeyProviderCryptoExtension keyProviderCryptoExtension) {
+ this.keyProviderCryptoExtension = keyProviderCryptoExtension;
+ encKeyVersionQueue =
+ new ValueQueue<KeyProviderCryptoExtension.EncryptedKeyVersion>(
+ conf.getInt(KMS_KEY_CACHE_SIZE,
+ KMS_KEY_CACHE_SIZE_DEFAULT),
+ conf.getFloat(KMS_KEY_CACHE_LOW_WATERMARK,
+ KMS_KEY_CACHE_LOW_WATERMARK_DEFAULT),
+ conf.getInt(KMS_KEY_CACHE_EXPIRY_MS,
+ KMS_KEY_CACHE_EXPIRY_DEFAULT),
+ conf.getInt(KMS_KEY_CACHE_NUM_REFILL_THREADS,
+ KMS_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT),
+ SyncGenerationPolicy.LOW_WATERMARK, new EncryptedQueueRefiller()
+ );
+ }
+
+ @Override
+ public void warmUpEncryptedKeys(String... keyNames) throws
+ IOException {
+ try {
+ encKeyVersionQueue.initializeQueuesForKeys(keyNames);
+ } catch (ExecutionException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public void drain(String keyName) {
+ encKeyVersionQueue.drain(keyName);
+ }
+
+ @Override
+ public EncryptedKeyVersion generateEncryptedKey(String encryptionKeyName)
+ throws IOException, GeneralSecurityException {
+ try {
+ return encKeyVersionQueue.getNext(encryptionKeyName);
+ } catch (ExecutionException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public KeyVersion
+ decryptEncryptedKey(EncryptedKeyVersion encryptedKeyVersion)
+ throws IOException, GeneralSecurityException {
+ return keyProviderCryptoExtension.decryptEncryptedKey(
+ encryptedKeyVersion);
+ }
+ }
+
+ /**
+ * This class is a proxy for a <code>KeyProviderCryptoExtension</code> that
+ * decorates the underlying <code>CryptoExtension</code> with one that eagerly
+ * caches pre-generated Encrypted Keys using a <code>ValueQueue</code>
+ *
+ * @param conf Configuration object to load parameters from
+ * @param keyProviderCryptoExtension <code>KeyProviderCryptoExtension</code>
+ * to delegate calls to.
+ */
+ public EagerKeyGeneratorKeyProviderCryptoExtension(Configuration conf,
+ KeyProviderCryptoExtension keyProviderCryptoExtension) {
+ super(keyProviderCryptoExtension,
+ new CryptoExtension(conf, keyProviderCryptoExtension));
+ }
+
+ @Override
+ public KeyVersion rollNewVersion(String name)
+ throws NoSuchAlgorithmException, IOException {
+ KeyVersion keyVersion = super.rollNewVersion(name);
+ getExtension().drain(name);
+ return keyVersion;
+ }
+
+ @Override
+ public KeyVersion rollNewVersion(String name, byte[] material)
+ throws IOException {
+ KeyVersion keyVersion = super.rollNewVersion(name, material);
+ getExtension().drain(name);
+ return keyVersion;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
new file mode 100644
index 0000000..9b5b5a1
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
@@ -0,0 +1,482 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
+import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
+import org.apache.hadoop.crypto.key.kms.server.KMSACLsType.Type;
+import org.apache.hadoop.security.token.delegation.web.HttpUserGroupInformation;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Class providing the REST bindings, via Jersey, for the KMS.
+ */
+@Path(KMSRESTConstants.SERVICE_VERSION)
+@InterfaceAudience.Private
+public class KMS {
+
+ public static enum KMSOp {
+ CREATE_KEY, DELETE_KEY, ROLL_NEW_VERSION,
+ GET_KEYS, GET_KEYS_METADATA,
+ GET_KEY_VERSIONS, GET_METADATA, GET_KEY_VERSION, GET_CURRENT_KEY,
+ GENERATE_EEK, DECRYPT_EEK
+ }
+
+ private KeyProviderCryptoExtension provider;
+ private KMSAudit kmsAudit;
+
+ public KMS() throws Exception {
+ provider = KMSWebApp.getKeyProvider();
+ kmsAudit= KMSWebApp.getKMSAudit();
+ }
+
+ private void assertAccess(Type aclType, UserGroupInformation ugi,
+ KMSOp operation) throws AccessControlException {
+ KMSWebApp.getACLs().assertAccess(aclType, ugi, operation, null);
+ }
+
+ private void assertAccess(Type aclType, UserGroupInformation ugi,
+ KMSOp operation, String key) throws AccessControlException {
+ KMSWebApp.getACLs().assertAccess(aclType, ugi, operation, key);
+ }
+
+ private static KeyProvider.KeyVersion removeKeyMaterial(
+ KeyProvider.KeyVersion keyVersion) {
+ return new KMSClientProvider.KMSKeyVersion(keyVersion.getName(),
+ keyVersion.getVersionName(), null);
+ }
+
+ private static URI getKeyURI(String name) throws URISyntaxException {
+ return new URI(KMSRESTConstants.SERVICE_VERSION + "/" +
+ KMSRESTConstants.KEY_RESOURCE + "/" + name);
+ }
+
+ @POST
+ @Path(KMSRESTConstants.KEYS_RESOURCE)
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ @SuppressWarnings("unchecked")
+ public Response createKey(Map jsonKey) throws Exception {
+ KMSWebApp.getAdminCallsMeter().mark();
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ final String name = (String) jsonKey.get(KMSRESTConstants.NAME_FIELD);
+ KMSClientProvider.checkNotEmpty(name, KMSRESTConstants.NAME_FIELD);
+ assertAccess(Type.CREATE, user, KMSOp.CREATE_KEY, name);
+ String cipher = (String) jsonKey.get(KMSRESTConstants.CIPHER_FIELD);
+ final String material = (String) jsonKey.get(KMSRESTConstants.MATERIAL_FIELD);
+ int length = (jsonKey.containsKey(KMSRESTConstants.LENGTH_FIELD))
+ ? (Integer) jsonKey.get(KMSRESTConstants.LENGTH_FIELD) : 0;
+ String description = (String)
+ jsonKey.get(KMSRESTConstants.DESCRIPTION_FIELD);
+ Map<String, String> attributes = (Map<String, String>)
+ jsonKey.get(KMSRESTConstants.ATTRIBUTES_FIELD);
+ if (material != null) {
+ assertAccess(Type.SET_KEY_MATERIAL, user,
+ KMSOp.CREATE_KEY, name);
+ }
+ final KeyProvider.Options options = new KeyProvider.Options(
+ KMSWebApp.getConfiguration());
+ if (cipher != null) {
+ options.setCipher(cipher);
+ }
+ if (length != 0) {
+ options.setBitLength(length);
+ }
+ options.setDescription(description);
+ options.setAttributes(attributes);
+
+ KeyProvider.KeyVersion keyVersion = user.doAs(
+ new PrivilegedExceptionAction<KeyVersion>() {
+ @Override
+ public KeyVersion run() throws Exception {
+ KeyProvider.KeyVersion keyVersion = (material != null)
+ ? provider.createKey(name, Base64.decodeBase64(material), options)
+ : provider.createKey(name, options);
+ provider.flush();
+ return keyVersion;
+ }
+ }
+ );
+
+ kmsAudit.ok(user, KMSOp.CREATE_KEY, name, "UserProvidedMaterial:" +
+ (material != null) + " Description:" + description);
+
+ if (!KMSWebApp.getACLs().hasAccess(Type.GET, user)) {
+ keyVersion = removeKeyMaterial(keyVersion);
+ }
+ Map json = KMSServerJSONUtils.toJSON(keyVersion);
+ String requestURL = KMSMDCFilter.getURL();
+ int idx = requestURL.lastIndexOf(KMSRESTConstants.KEYS_RESOURCE);
+ requestURL = requestURL.substring(0, idx);
+ String keyURL = requestURL + KMSRESTConstants.KEY_RESOURCE + "/" + name;
+ return Response.created(getKeyURI(name)).type(MediaType.APPLICATION_JSON).
+ header("Location", keyURL).entity(json).build();
+ }
+
+ @DELETE
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}")
+ public Response deleteKey(@PathParam("name") final String name)
+ throws Exception {
+ KMSWebApp.getAdminCallsMeter().mark();
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ assertAccess(Type.DELETE, user, KMSOp.DELETE_KEY, name);
+ KMSClientProvider.checkNotEmpty(name, "name");
+
+ user.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ provider.deleteKey(name);
+ provider.flush();
+ return null;
+ }
+ });
+
+ kmsAudit.ok(user, KMSOp.DELETE_KEY, name, "");
+
+ return Response.ok().build();
+ }
+
+ @POST
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}")
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response rolloverKey(@PathParam("name") final String name,
+ Map jsonMaterial) throws Exception {
+ KMSWebApp.getAdminCallsMeter().mark();
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ assertAccess(Type.ROLLOVER, user, KMSOp.ROLL_NEW_VERSION, name);
+ KMSClientProvider.checkNotEmpty(name, "name");
+ final String material = (String)
+ jsonMaterial.get(KMSRESTConstants.MATERIAL_FIELD);
+ if (material != null) {
+ assertAccess(Type.SET_KEY_MATERIAL, user,
+ KMSOp.ROLL_NEW_VERSION, name);
+ }
+
+ KeyProvider.KeyVersion keyVersion = user.doAs(
+ new PrivilegedExceptionAction<KeyVersion>() {
+ @Override
+ public KeyVersion run() throws Exception {
+ KeyVersion keyVersion = (material != null)
+ ? provider.rollNewVersion(name, Base64.decodeBase64(material))
+ : provider.rollNewVersion(name);
+ provider.flush();
+ return keyVersion;
+ }
+ }
+ );
+
+ kmsAudit.ok(user, KMSOp.ROLL_NEW_VERSION, name, "UserProvidedMaterial:" +
+ (material != null) + " NewVersion:" + keyVersion.getVersionName());
+
+ if (!KMSWebApp.getACLs().hasAccess(Type.GET, user)) {
+ keyVersion = removeKeyMaterial(keyVersion);
+ }
+ Map json = KMSServerJSONUtils.toJSON(keyVersion);
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEYS_METADATA_RESOURCE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getKeysMetadata(@QueryParam(KMSRESTConstants.KEY)
+ List<String> keyNamesList) throws Exception {
+ KMSWebApp.getAdminCallsMeter().mark();
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ final String[] keyNames = keyNamesList.toArray(
+ new String[keyNamesList.size()]);
+ assertAccess(Type.GET_METADATA, user, KMSOp.GET_KEYS_METADATA);
+
+ KeyProvider.Metadata[] keysMeta = user.doAs(
+ new PrivilegedExceptionAction<KeyProvider.Metadata[]>() {
+ @Override
+ public KeyProvider.Metadata[] run() throws Exception {
+ return provider.getKeysMetadata(keyNames);
+ }
+ }
+ );
+
+ Object json = KMSServerJSONUtils.toJSON(keyNames, keysMeta);
+ kmsAudit.ok(user, KMSOp.GET_KEYS_METADATA, "");
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEYS_NAMES_RESOURCE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getKeyNames() throws Exception {
+ KMSWebApp.getAdminCallsMeter().mark();
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ assertAccess(Type.GET_KEYS, user, KMSOp.GET_KEYS);
+
+ List<String> json = user.doAs(
+ new PrivilegedExceptionAction<List<String>>() {
+ @Override
+ public List<String> run() throws Exception {
+ return provider.getKeys();
+ }
+ }
+ );
+
+ kmsAudit.ok(user, KMSOp.GET_KEYS, "");
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}")
+ public Response getKey(@PathParam("name") String name)
+ throws Exception {
+ return getMetadata(name);
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" +
+ KMSRESTConstants.METADATA_SUB_RESOURCE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getMetadata(@PathParam("name") final String name)
+ throws Exception {
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ KMSClientProvider.checkNotEmpty(name, "name");
+ KMSWebApp.getAdminCallsMeter().mark();
+ assertAccess(Type.GET_METADATA, user, KMSOp.GET_METADATA, name);
+
+ KeyProvider.Metadata metadata = user.doAs(
+ new PrivilegedExceptionAction<KeyProvider.Metadata>() {
+ @Override
+ public KeyProvider.Metadata run() throws Exception {
+ return provider.getMetadata(name);
+ }
+ }
+ );
+
+ Object json = KMSServerJSONUtils.toJSON(name, metadata);
+ kmsAudit.ok(user, KMSOp.GET_METADATA, name, "");
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" +
+ KMSRESTConstants.CURRENT_VERSION_SUB_RESOURCE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getCurrentVersion(@PathParam("name") final String name)
+ throws Exception {
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ KMSClientProvider.checkNotEmpty(name, "name");
+ KMSWebApp.getKeyCallsMeter().mark();
+ assertAccess(Type.GET, user, KMSOp.GET_CURRENT_KEY, name);
+
+ KeyVersion keyVersion = user.doAs(
+ new PrivilegedExceptionAction<KeyVersion>() {
+ @Override
+ public KeyVersion run() throws Exception {
+ return provider.getCurrentKey(name);
+ }
+ }
+ );
+
+ Object json = KMSServerJSONUtils.toJSON(keyVersion);
+ kmsAudit.ok(user, KMSOp.GET_CURRENT_KEY, name, "");
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEY_VERSION_RESOURCE + "/{versionName:.*}")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getKeyVersion(
+ @PathParam("versionName") final String versionName) throws Exception {
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ KMSClientProvider.checkNotEmpty(versionName, "versionName");
+ KMSWebApp.getKeyCallsMeter().mark();
+ assertAccess(Type.GET, user, KMSOp.GET_KEY_VERSION);
+
+ KeyVersion keyVersion = user.doAs(
+ new PrivilegedExceptionAction<KeyVersion>() {
+ @Override
+ public KeyVersion run() throws Exception {
+ return provider.getKeyVersion(versionName);
+ }
+ }
+ );
+
+ if (keyVersion != null) {
+ kmsAudit.ok(user, KMSOp.GET_KEY_VERSION, keyVersion.getName(), "");
+ }
+ Object json = KMSServerJSONUtils.toJSON(keyVersion);
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @GET
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" +
+ KMSRESTConstants.EEK_SUB_RESOURCE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response generateEncryptedKeys(
+ @PathParam("name") final String name,
+ @QueryParam(KMSRESTConstants.EEK_OP) String edekOp,
+ @DefaultValue("1")
+ @QueryParam(KMSRESTConstants.EEK_NUM_KEYS) final int numKeys)
+ throws Exception {
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ KMSClientProvider.checkNotEmpty(name, "name");
+ KMSClientProvider.checkNotNull(edekOp, "eekOp");
+
+ Object retJSON;
+ if (edekOp.equals(KMSRESTConstants.EEK_GENERATE)) {
+ assertAccess(Type.GENERATE_EEK, user, KMSOp.GENERATE_EEK, name);
+
+ final List<EncryptedKeyVersion> retEdeks =
+ new LinkedList<EncryptedKeyVersion>();
+ try {
+
+ user.doAs(
+ new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ for (int i = 0; i < numKeys; i++) {
+ retEdeks.add(provider.generateEncryptedKey(name));
+ }
+ return null;
+ }
+ }
+ );
+
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ kmsAudit.ok(user, KMSOp.GENERATE_EEK, name, "");
+ retJSON = new ArrayList();
+ for (EncryptedKeyVersion edek : retEdeks) {
+ ((ArrayList)retJSON).add(KMSServerJSONUtils.toJSON(edek));
+ }
+ } else {
+ throw new IllegalArgumentException("Wrong " + KMSRESTConstants.EEK_OP +
+ " value, it must be " + KMSRESTConstants.EEK_GENERATE + " or " +
+ KMSRESTConstants.EEK_DECRYPT);
+ }
+ KMSWebApp.getGenerateEEKCallsMeter().mark();
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(retJSON)
+ .build();
+ }
+
+ @SuppressWarnings("rawtypes")
+ @POST
+ @Path(KMSRESTConstants.KEY_VERSION_RESOURCE + "/{versionName:.*}/" +
+ KMSRESTConstants.EEK_SUB_RESOURCE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response decryptEncryptedKey(
+ @PathParam("versionName") final String versionName,
+ @QueryParam(KMSRESTConstants.EEK_OP) String eekOp,
+ Map jsonPayload)
+ throws Exception {
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ KMSClientProvider.checkNotEmpty(versionName, "versionName");
+ KMSClientProvider.checkNotNull(eekOp, "eekOp");
+
+ final String keyName = (String) jsonPayload.get(
+ KMSRESTConstants.NAME_FIELD);
+ String ivStr = (String) jsonPayload.get(KMSRESTConstants.IV_FIELD);
+ String encMaterialStr =
+ (String) jsonPayload.get(KMSRESTConstants.MATERIAL_FIELD);
+ Object retJSON;
+ if (eekOp.equals(KMSRESTConstants.EEK_DECRYPT)) {
+ assertAccess(Type.DECRYPT_EEK, user, KMSOp.DECRYPT_EEK, keyName);
+ KMSClientProvider.checkNotNull(ivStr, KMSRESTConstants.IV_FIELD);
+ final byte[] iv = Base64.decodeBase64(ivStr);
+ KMSClientProvider.checkNotNull(encMaterialStr,
+ KMSRESTConstants.MATERIAL_FIELD);
+ final byte[] encMaterial = Base64.decodeBase64(encMaterialStr);
+
+ KeyProvider.KeyVersion retKeyVersion = user.doAs(
+ new PrivilegedExceptionAction<KeyVersion>() {
+ @Override
+ public KeyVersion run() throws Exception {
+ return provider.decryptEncryptedKey(
+ new KMSClientProvider.KMSEncryptedKeyVersion(keyName,
+ versionName, iv, KeyProviderCryptoExtension.EEK,
+ encMaterial)
+ );
+ }
+ }
+ );
+
+ retJSON = KMSServerJSONUtils.toJSON(retKeyVersion);
+ kmsAudit.ok(user, KMSOp.DECRYPT_EEK, keyName, "");
+ } else {
+ throw new IllegalArgumentException("Wrong " + KMSRESTConstants.EEK_OP +
+ " value, it must be " + KMSRESTConstants.EEK_GENERATE + " or " +
+ KMSRESTConstants.EEK_DECRYPT);
+ }
+ KMSWebApp.getDecryptEEKCallsMeter().mark();
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(retJSON)
+ .build();
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" +
+ KMSRESTConstants.VERSIONS_SUB_RESOURCE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getKeyVersions(@PathParam("name") final String name)
+ throws Exception {
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ KMSClientProvider.checkNotEmpty(name, "name");
+ KMSWebApp.getKeyCallsMeter().mark();
+ assertAccess(Type.GET, user, KMSOp.GET_KEY_VERSIONS, name);
+
+ List<KeyVersion> ret = user.doAs(
+ new PrivilegedExceptionAction<List<KeyVersion>>() {
+ @Override
+ public List<KeyVersion> run() throws Exception {
+ return provider.getKeyVersions(name);
+ }
+ }
+ );
+
+ Object json = KMSServerJSONUtils.toJSON(ret);
+ kmsAudit.ok(user, KMSOp.GET_KEY_VERSIONS, name, "");
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
new file mode 100644
index 0000000..f2298c0
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
@@ -0,0 +1,253 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.kms.server.KMS.KMSOp;
+import org.apache.hadoop.crypto.key.kms.server.KMSACLsType.Type;
+import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyACLs;
+import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyOpType;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+
+/**
+ * Provides access to the <code>AccessControlList</code>s used by KMS,
+ * hot-reloading them if the <code>kms-acls.xml</code> file where the ACLs
+ * are defined has been updated.
+ */
+@InterfaceAudience.Private
+public class KMSACLs implements Runnable, KeyACLs {
+ private static final Logger LOG = LoggerFactory.getLogger(KMSACLs.class);
+
+ private static final String UNAUTHORIZED_MSG_WITH_KEY =
+ "User:%s not allowed to do '%s' on '%s'";
+
+ private static final String UNAUTHORIZED_MSG_WITHOUT_KEY =
+ "User:%s not allowed to do '%s'";
+
+ public static final String ACL_DEFAULT = AccessControlList.WILDCARD_ACL_VALUE;
+
+ public static final int RELOADER_SLEEP_MILLIS = 1000;
+
+ private volatile Map<Type, AccessControlList> acls;
+ private volatile Map<Type, AccessControlList> blacklistedAcls;
+ private volatile Map<String, HashMap<KeyOpType, AccessControlList>> keyAcls;
+ private final Map<KeyOpType, AccessControlList> defaultKeyAcls =
+ new HashMap<KeyOpType, AccessControlList>();
+ private ScheduledExecutorService executorService;
+ private long lastReload;
+
+ KMSACLs(Configuration conf) {
+ if (conf == null) {
+ conf = loadACLs();
+ }
+ setKMSACLs(conf);
+ setKeyACLs(conf);
+ }
+
+ public KMSACLs() {
+ this(null);
+ }
+
+ private void setKMSACLs(Configuration conf) {
+ Map<Type, AccessControlList> tempAcls = new HashMap<Type, AccessControlList>();
+ Map<Type, AccessControlList> tempBlacklist = new HashMap<Type, AccessControlList>();
+ for (Type aclType : Type.values()) {
+ String aclStr = conf.get(aclType.getAclConfigKey(), ACL_DEFAULT);
+ tempAcls.put(aclType, new AccessControlList(aclStr));
+ String blacklistStr = conf.get(aclType.getBlacklistConfigKey());
+ if (blacklistStr != null) {
+ // Only add if blacklist is present
+ tempBlacklist.put(aclType, new AccessControlList(blacklistStr));
+ LOG.info("'{}' Blacklist '{}'", aclType, blacklistStr);
+ }
+ LOG.info("'{}' ACL '{}'", aclType, aclStr);
+ }
+ acls = tempAcls;
+ blacklistedAcls = tempBlacklist;
+ }
+
+ private void setKeyACLs(Configuration conf) {
+ Map<String, HashMap<KeyOpType, AccessControlList>> tempKeyAcls =
+ new HashMap<String, HashMap<KeyOpType,AccessControlList>>();
+ Map<String, String> allKeyACLS =
+ conf.getValByRegex(Pattern.quote(KMSConfiguration.KEY_ACL_PREFIX));
+ for (Map.Entry<String, String> keyAcl : allKeyACLS.entrySet()) {
+ String k = keyAcl.getKey();
+ // this should be of type "key.acl.<KEY_NAME>.<OP_TYPE>"
+ int keyNameStarts = KMSConfiguration.KEY_ACL_PREFIX.length();
+ int keyNameEnds = k.lastIndexOf(".");
+ if (keyNameStarts >= keyNameEnds) {
+ LOG.warn("Invalid key name '{}'", k);
+ } else {
+ String aclStr = keyAcl.getValue();
+ String keyName = k.substring(keyNameStarts, keyNameEnds);
+ String keyOp = k.substring(keyNameEnds + 1);
+ KeyOpType aclType = null;
+ try {
+ aclType = KeyOpType.valueOf(keyOp);
+ } catch (IllegalArgumentException e) {
+ LOG.warn("Invalid key Operation '{}'", keyOp);
+ }
+ if (aclType != null) {
+ // On the assumption this will be single threaded.. else we need to
+ // ConcurrentHashMap
+ HashMap<KeyOpType,AccessControlList> aclMap =
+ tempKeyAcls.get(keyName);
+ if (aclMap == null) {
+ aclMap = new HashMap<KeyOpType, AccessControlList>();
+ tempKeyAcls.put(keyName, aclMap);
+ }
+ aclMap.put(aclType, new AccessControlList(aclStr));
+ LOG.info("KEY_NAME '{}' KEY_OP '{}' ACL '{}'",
+ keyName, aclType, aclStr);
+ }
+ }
+ }
+
+ keyAcls = tempKeyAcls;
+ for (KeyOpType keyOp : KeyOpType.values()) {
+ if (!defaultKeyAcls.containsKey(keyOp)) {
+ String confKey = KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + keyOp;
+ String aclStr = conf.get(confKey);
+ if (aclStr != null) {
+ if (aclStr.equals("*")) {
+ LOG.info("Default Key ACL for KEY_OP '{}' is set to '*'", keyOp);
+ }
+ defaultKeyAcls.put(keyOp, new AccessControlList(aclStr));
+ }
+ }
+ }
+ }
+
+ @Override
+ public void run() {
+ try {
+ if (KMSConfiguration.isACLsFileNewer(lastReload)) {
+ setKMSACLs(loadACLs());
+ setKeyACLs(loadACLs());
+ }
+ } catch (Exception ex) {
+ LOG.warn(
+ String.format("Could not reload ACLs file: '%s'", ex.toString()), ex);
+ }
+ }
+
+ public synchronized void startReloader() {
+ if (executorService == null) {
+ executorService = Executors.newScheduledThreadPool(1);
+ executorService.scheduleAtFixedRate(this, RELOADER_SLEEP_MILLIS,
+ RELOADER_SLEEP_MILLIS, TimeUnit.MILLISECONDS);
+ }
+ }
+
+ public synchronized void stopReloader() {
+ if (executorService != null) {
+ executorService.shutdownNow();
+ executorService = null;
+ }
+ }
+
+ private Configuration loadACLs() {
+ LOG.debug("Loading ACLs file");
+ lastReload = System.currentTimeMillis();
+ Configuration conf = KMSConfiguration.getACLsConf();
+ // triggering the resource loading.
+ conf.get(Type.CREATE.getAclConfigKey());
+ return conf;
+ }
+
+ /**
+ * First Check if user is in ACL for the KMS operation, if yes, then
+ * return true if user is not present in any configured blacklist for
+ * the operation
+ * @param type KMS Operation
+ * @param ugi UserGroupInformation of user
+ * @return true is user has access
+ */
+ @Override
+ public boolean hasAccess(Type type, UserGroupInformation ugi) {
+ boolean access = acls.get(type).isUserAllowed(ugi);
+ if (access) {
+ AccessControlList blacklist = blacklistedAcls.get(type);
+ access = (blacklist == null) || !blacklist.isUserInList(ugi);
+ }
+ return access;
+ }
+
+ @Override
+ public void assertAccess(Type aclType,
+ UserGroupInformation ugi, KMSOp operation, String key)
+ throws AccessControlException {
+ if (!KMSWebApp.getACLs().hasAccess(aclType, ugi)) {
+ KMSWebApp.getUnauthorizedCallsMeter().mark();
+ KMSWebApp.getKMSAudit().unauthorized(ugi, operation, key);
+ throw new AuthorizationException(String.format(
+ (key != null) ? UNAUTHORIZED_MSG_WITH_KEY
+ : UNAUTHORIZED_MSG_WITHOUT_KEY,
+ ugi.getShortUserName(), operation, key));
+ }
+ }
+
+ @Override
+ public boolean hasAccessToKey(String keyName, UserGroupInformation ugi,
+ KeyOpType opType) {
+ Map<KeyOpType, AccessControlList> keyAcl = keyAcls.get(keyName);
+ if (keyAcl == null) {
+ // Get KeyAcl map of DEFAULT KEY.
+ keyAcl = defaultKeyAcls;
+ }
+ // If No key acl defined for this key, check to see if
+ // there are key defaults configured for this operation
+ AccessControlList acl = keyAcl.get(opType);
+ if (acl == null) {
+ // If no acl is specified for this operation,
+ // deny access
+ return false;
+ } else {
+ return acl.isUserAllowed(ugi);
+ }
+ }
+
+ @Override
+ public boolean isACLPresent(String keyName, KeyOpType opType) {
+ return (keyAcls.containsKey(keyName) || defaultKeyAcls.containsKey(opType));
+ }
+
+ @Override
+ public void startACLReloader() {
+ this.startReloader();
+ }
+
+ @Override
+ public void stopACLReloader() {
+ this.stopReloader();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLsType.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLsType.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLsType.java
new file mode 100644
index 0000000..0fd3091
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLsType.java
@@ -0,0 +1,17 @@
+package org.apache.hadoop.crypto.key.kms.server;
+
+public class KMSACLsType {
+
+ public enum Type {
+ CREATE, DELETE, ROLLOVER, GET, GET_KEYS, GET_METADATA,
+ SET_KEY_MATERIAL, GENERATE_EEK, DECRYPT_EEK;
+
+ public String getAclConfigKey() {
+ return KMSConfiguration.CONFIG_PREFIX + "acl." + this.toString();
+ }
+
+ public String getBlacklistConfigKey() {
+ return KMSConfiguration.CONFIG_PREFIX + "blacklist." + this.toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
new file mode 100644
index 0000000..7ff76e5
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
@@ -0,0 +1,230 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.hadoop.security.UserGroupInformation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Provides convenience methods for audit logging consistently the different
+ * types of events.
+ */
+public class KMSAudit {
+
+ private static class AuditEvent {
+ private final AtomicLong accessCount = new AtomicLong(-1);
+ private final String keyName;
+ private final String user;
+ private final KMS.KMSOp op;
+ private final String extraMsg;
+ private final long startTime = System.currentTimeMillis();
+
+ private AuditEvent(String keyName, String user, KMS.KMSOp op, String msg) {
+ this.keyName = keyName;
+ this.user = user;
+ this.op = op;
+ this.extraMsg = msg;
+ }
+
+ public String getExtraMsg() {
+ return extraMsg;
+ }
+
+ public AtomicLong getAccessCount() {
+ return accessCount;
+ }
+
+ public String getKeyName() {
+ return keyName;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public KMS.KMSOp getOp() {
+ return op;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+ }
+
+ public static enum OpStatus {
+ OK, UNAUTHORIZED, UNAUTHENTICATED, ERROR;
+ }
+
+ private static Set<KMS.KMSOp> AGGREGATE_OPS_WHITELIST = Sets.newHashSet(
+ KMS.KMSOp.GET_KEY_VERSION, KMS.KMSOp.GET_CURRENT_KEY,
+ KMS.KMSOp.DECRYPT_EEK, KMS.KMSOp.GENERATE_EEK
+ );
+
+ private Cache<String, AuditEvent> cache;
+
+ private ScheduledExecutorService executor;
+
+ public static final String KMS_LOGGER_NAME = "kms-audit";
+
+ private static Logger AUDIT_LOG = LoggerFactory.getLogger(KMS_LOGGER_NAME);
+
+ /**
+ * Create a new KMSAudit.
+ *
+ * @param windowMs Duplicate events within the aggregation window are quashed
+ * to reduce log traffic. A single message for aggregated
+ * events is printed at the end of the window, along with a
+ * count of the number of aggregated events.
+ */
+ KMSAudit(long windowMs) {
+ cache = CacheBuilder.newBuilder()
+ .expireAfterWrite(windowMs, TimeUnit.MILLISECONDS)
+ .removalListener(
+ new RemovalListener<String, AuditEvent>() {
+ @Override
+ public void onRemoval(
+ RemovalNotification<String, AuditEvent> entry) {
+ AuditEvent event = entry.getValue();
+ if (event.getAccessCount().get() > 0) {
+ KMSAudit.this.logEvent(event);
+ event.getAccessCount().set(0);
+ KMSAudit.this.cache.put(entry.getKey(), event);
+ }
+ }
+ }).build();
+ executor = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder()
+ .setDaemon(true).setNameFormat(KMS_LOGGER_NAME + "_thread").build());
+ executor.scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ cache.cleanUp();
+ }
+ }, windowMs / 10, windowMs / 10, TimeUnit.MILLISECONDS);
+ }
+
+ private void logEvent(AuditEvent event) {
+ AUDIT_LOG.info(
+ "OK[op={}, key={}, user={}, accessCount={}, interval={}ms] {}",
+ event.getOp(), event.getKeyName(), event.getUser(),
+ event.getAccessCount().get(),
+ (System.currentTimeMillis() - event.getStartTime()),
+ event.getExtraMsg());
+ }
+
+ private void op(OpStatus opStatus, final KMS.KMSOp op, final String user,
+ final String key, final String extraMsg) {
+ if (!Strings.isNullOrEmpty(user) && !Strings.isNullOrEmpty(key)
+ && (op != null)
+ && AGGREGATE_OPS_WHITELIST.contains(op)) {
+ String cacheKey = createCacheKey(user, key, op);
+ if (opStatus == OpStatus.UNAUTHORIZED) {
+ cache.invalidate(cacheKey);
+ AUDIT_LOG.info("UNAUTHORIZED[op={}, key={}, user={}] {}", op, key, user,
+ extraMsg);
+ } else {
+ try {
+ AuditEvent event = cache.get(cacheKey, new Callable<AuditEvent>() {
+ @Override
+ public AuditEvent call() throws Exception {
+ return new AuditEvent(key, user, op, extraMsg);
+ }
+ });
+ // Log first access (initialized as -1 so
+ // incrementAndGet() == 0 implies first access)
+ if (event.getAccessCount().incrementAndGet() == 0) {
+ event.getAccessCount().incrementAndGet();
+ logEvent(event);
+ }
+ } catch (ExecutionException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ } else {
+ List<String> kvs = new LinkedList<String>();
+ if (op != null) {
+ kvs.add("op=" + op);
+ }
+ if (!Strings.isNullOrEmpty(key)) {
+ kvs.add("key=" + key);
+ }
+ if (!Strings.isNullOrEmpty(user)) {
+ kvs.add("user=" + user);
+ }
+ if (kvs.size() == 0) {
+ AUDIT_LOG.info("{} {}", opStatus.toString(), extraMsg);
+ } else {
+ String join = Joiner.on(", ").join(kvs);
+ AUDIT_LOG.info("{}[{}] {}", opStatus.toString(), join, extraMsg);
+ }
+ }
+ }
+
+ public void ok(UserGroupInformation user, KMS.KMSOp op, String key,
+ String extraMsg) {
+ op(OpStatus.OK, op, user.getShortUserName(), key, extraMsg);
+ }
+
+ public void ok(UserGroupInformation user, KMS.KMSOp op, String extraMsg) {
+ op(OpStatus.OK, op, user.getShortUserName(), null, extraMsg);
+ }
+
+ public void unauthorized(UserGroupInformation user, KMS.KMSOp op, String key) {
+ op(OpStatus.UNAUTHORIZED, op, user.getShortUserName(), key, "");
+ }
+
+ public void error(UserGroupInformation user, String method, String url,
+ String extraMsg) {
+ op(OpStatus.ERROR, null, user.getShortUserName(), null, "Method:'" + method
+ + "' Exception:'" + extraMsg + "'");
+ }
+
+ public void unauthenticated(String remoteHost, String method,
+ String url, String extraMsg) {
+ op(OpStatus.UNAUTHENTICATED, null, null, null, "RemoteHost:"
+ + remoteHost + " Method:" + method
+ + " URL:" + url + " ErrorMsg:'" + extraMsg + "'");
+ }
+
+ private static String createCacheKey(String user, String key, KMS.KMSOp op) {
+ return user + "#" + key + "#" + op;
+ }
+
+ public void shutdown() {
+ executor.shutdownNow();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
new file mode 100644
index 0000000..79652f3
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
+import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler;
+import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler;
+import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticationHandler;
+
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.servlet.http.HttpServletResponseWrapper;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Authentication filter that takes the configuration from the KMS configuration
+ * file.
+ */
+@InterfaceAudience.Private
+public class KMSAuthenticationFilter
+ extends DelegationTokenAuthenticationFilter {
+
+ public static final String CONFIG_PREFIX = KMSConfiguration.CONFIG_PREFIX +
+ "authentication.";
+
+ @Override
+ protected Properties getConfiguration(String configPrefix,
+ FilterConfig filterConfig) {
+ Properties props = new Properties();
+ Configuration conf = KMSWebApp.getConfiguration();
+ for (Map.Entry<String, String> entry : conf) {
+ String name = entry.getKey();
+ if (name.startsWith(CONFIG_PREFIX)) {
+ String value = conf.get(name);
+ name = name.substring(CONFIG_PREFIX.length());
+ props.setProperty(name, value);
+ }
+ }
+ String authType = props.getProperty(AUTH_TYPE);
+ if (authType.equals(PseudoAuthenticationHandler.TYPE)) {
+ props.setProperty(AUTH_TYPE,
+ PseudoDelegationTokenAuthenticationHandler.class.getName());
+ } else if (authType.equals(KerberosAuthenticationHandler.TYPE)) {
+ props.setProperty(AUTH_TYPE,
+ KerberosDelegationTokenAuthenticationHandler.class.getName());
+ }
+ props.setProperty(DelegationTokenAuthenticationHandler.TOKEN_KIND,
+ KMSClientProvider.TOKEN_KIND);
+ return props;
+ }
+
+ protected Configuration getProxyuserConfiguration(FilterConfig filterConfig) {
+ Map<String, String> proxyuserConf = KMSWebApp.getConfiguration().
+ getValByRegex("hadoop\\.kms\\.proxyuser\\.");
+ Configuration conf = new Configuration(false);
+ for (Map.Entry<String, String> entry : proxyuserConf.entrySet()) {
+ conf.set(entry.getKey().substring("hadoop.kms.".length()),
+ entry.getValue());
+ }
+ return conf;
+ }
+
+ private static class KMSResponse extends HttpServletResponseWrapper {
+ public int statusCode;
+ public String msg;
+
+ public KMSResponse(ServletResponse response) {
+ super((HttpServletResponse)response);
+ }
+
+ @Override
+ public void setStatus(int sc) {
+ statusCode = sc;
+ super.setStatus(sc);
+ }
+
+ @Override
+ public void sendError(int sc, String msg) throws IOException {
+ statusCode = sc;
+ this.msg = msg;
+ super.sendError(sc, msg);
+ }
+
+ @Override
+ public void sendError(int sc) throws IOException {
+ statusCode = sc;
+ super.sendError(sc);
+ }
+
+ @Override
+ public void setStatus(int sc, String sm) {
+ statusCode = sc;
+ msg = sm;
+ super.setStatus(sc, sm);
+ }
+ }
+
+ @Override
+ public void doFilter(ServletRequest request, ServletResponse response,
+ FilterChain filterChain) throws IOException, ServletException {
+ KMSResponse kmsResponse = new KMSResponse(response);
+ super.doFilter(request, kmsResponse, filterChain);
+
+ if (kmsResponse.statusCode != HttpServletResponse.SC_OK &&
+ kmsResponse.statusCode != HttpServletResponse.SC_CREATED &&
+ kmsResponse.statusCode != HttpServletResponse.SC_UNAUTHORIZED) {
+ KMSWebApp.getInvalidCallsMeter().mark();
+ }
+
+ // HttpServletResponse.SC_UNAUTHORIZED is because the request does not
+ // belong to an authenticated user.
+ if (kmsResponse.statusCode == HttpServletResponse.SC_UNAUTHORIZED) {
+ KMSWebApp.getUnauthenticatedCallsMeter().mark();
+ String method = ((HttpServletRequest) request).getMethod();
+ StringBuffer requestURL = ((HttpServletRequest) request).getRequestURL();
+ String queryString = ((HttpServletRequest) request).getQueryString();
+ if (queryString != null) {
+ requestURL.append("?").append(queryString);
+ }
+
+ KMSWebApp.getKMSAudit().unauthenticated(
+ request.getRemoteHost(), method, requestURL.toString(),
+ kmsResponse.msg);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
new file mode 100644
index 0000000..d6c77ea
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import java.io.File;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+/**
+ * Utility class to load KMS configuration files.
+ */
+@InterfaceAudience.Private
+public class KMSConfiguration {
+
+ public static final String KMS_CONFIG_DIR = "kms.config.dir";
+ public static final String KMS_SITE_XML = "kms-site.xml";
+ public static final String KMS_ACLS_XML = "kms-acls.xml";
+
+ public static final String CONFIG_PREFIX = "hadoop.kms.";
+
+ public static final String KEY_ACL_PREFIX = "key.acl.";
+ public static final String DEFAULT_KEY_ACL_PREFIX = "default.key.acl.";
+
+ // Property to set the backing KeyProvider
+ public static final String KEY_PROVIDER_URI = CONFIG_PREFIX +
+ "key.provider.uri";
+
+ // Property to Enable/Disable Caching
+ public static final String KEY_CACHE_ENABLE = CONFIG_PREFIX +
+ "cache.enable";
+ // Timeout for the Key and Metadata Cache
+ public static final String KEY_CACHE_TIMEOUT_KEY = CONFIG_PREFIX +
+ "cache.timeout.ms";
+ // TImeout for the Current Key cache
+ public static final String CURR_KEY_CACHE_TIMEOUT_KEY = CONFIG_PREFIX +
+ "current.key.cache.timeout.ms";
+ // Delay for Audit logs that need aggregation
+ public static final String KMS_AUDIT_AGGREGATION_WINDOW = CONFIG_PREFIX +
+ "audit.aggregation.window.ms";
+
+ //for authorizer
+ public static final String KMS_SECURITY_AUTHORIZER = CONFIG_PREFIX + "security.authorization.manager";
+
+ public static final boolean KEY_CACHE_ENABLE_DEFAULT = true;
+ // 10 mins
+ public static final long KEY_CACHE_TIMEOUT_DEFAULT = 10 * 60 * 1000;
+ // 30 secs
+ public static final long CURR_KEY_CACHE_TIMEOUT_DEFAULT = 30 * 1000;
+ // 10 secs
+ public static final long KMS_AUDIT_AGGREGATION_WINDOW_DEFAULT = 10000;
+
+ // Property to Enable/Disable per Key authorization
+ public static final String KEY_AUTHORIZATION_ENABLE = CONFIG_PREFIX +
+ "key.authorization.enable";
+
+ public static final boolean KEY_AUTHORIZATION_ENABLE_DEFAULT = true;
+
+ static Configuration getConfiguration(boolean loadHadoopDefaults,
+ String ... resources) {
+ Configuration conf = new Configuration(loadHadoopDefaults);
+ String confDir = System.getProperty(KMS_CONFIG_DIR);
+ if (confDir != null) {
+ try {
+ Path confPath = new Path(confDir);
+ if (!confPath.isUriPathAbsolute()) {
+ throw new RuntimeException("System property '" + KMS_CONFIG_DIR +
+ "' must be an absolute path: " + confDir);
+ }
+ for (String resource : resources) {
+ conf.addResource(new URL("file://" + new Path(confDir, resource).toUri()));
+ }
+ } catch (MalformedURLException ex) {
+ throw new RuntimeException(ex);
+ }
+ } else {
+ for (String resource : resources) {
+ conf.addResource(resource);
+ }
+ }
+ return conf;
+ }
+
+ public static Configuration getKMSConf() {
+ return getConfiguration(true, "core-site.xml", KMS_SITE_XML);
+ }
+
+ public static Configuration getACLsConf() {
+ return getConfiguration(false, KMS_ACLS_XML);
+ }
+
+ public static boolean isACLsFileNewer(long time) {
+ boolean newer = false;
+ String confDir = System.getProperty(KMS_CONFIG_DIR);
+ if (confDir != null) {
+ Path confPath = new Path(confDir);
+ if (!confPath.isUriPathAbsolute()) {
+ throw new RuntimeException("System property '" + KMS_CONFIG_DIR +
+ "' must be an absolute path: " + confDir);
+ }
+ File f = new File(confDir, KMS_ACLS_XML);
+ // at least 100ms newer than time, we do this to ensure the file
+ // has been properly closed/flushed
+ newer = f.lastModified() - time > 100;
+ }
+ return newer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
new file mode 100644
index 0000000..5cb0885
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import com.sun.jersey.api.container.ContainerException;
+
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.util.HttpExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+import javax.ws.rs.ext.Provider;
+
+import java.io.IOException;
+
+/**
+ * Jersey provider that converts KMS exceptions into detailed HTTP errors.
+ */
+@Provider
+@InterfaceAudience.Private
+public class KMSExceptionsProvider implements ExceptionMapper<Exception> {
+ private static Logger LOG =
+ LoggerFactory.getLogger(KMSExceptionsProvider.class);
+
+ private static final String ENTER = System.getProperty("line.separator");
+
+ protected Response createResponse(Response.Status status, Throwable ex) {
+ return HttpExceptionUtils.createJerseyExceptionResponse(status, ex);
+ }
+
+ protected String getOneLineMessage(Throwable exception) {
+ String message = exception.getMessage();
+ if (message != null) {
+ int i = message.indexOf(ENTER);
+ if (i > -1) {
+ message = message.substring(0, i);
+ }
+ }
+ return message;
+ }
+
+ /**
+ * Maps different exceptions thrown by KMS to HTTP status codes.
+ */
+ @Override
+ public Response toResponse(Exception exception) {
+ Response.Status status;
+ boolean doAudit = true;
+ Throwable throwable = exception;
+ if (exception instanceof ContainerException) {
+ throwable = exception.getCause();
+ }
+ if (throwable instanceof SecurityException) {
+ status = Response.Status.FORBIDDEN;
+ } else if (throwable instanceof AuthenticationException) {
+ status = Response.Status.FORBIDDEN;
+ // we don't audit here because we did it already when checking access
+ doAudit = false;
+ } else if (throwable instanceof AuthorizationException) {
+ status = Response.Status.FORBIDDEN;
+ // we don't audit here because we did it already when checking access
+ doAudit = false;
+ } else if (throwable instanceof AccessControlException) {
+ status = Response.Status.FORBIDDEN;
+ } else if (exception instanceof IOException) {
+ status = Response.Status.INTERNAL_SERVER_ERROR;
+ } else if (exception instanceof UnsupportedOperationException) {
+ status = Response.Status.BAD_REQUEST;
+ } else if (exception instanceof IllegalArgumentException) {
+ status = Response.Status.BAD_REQUEST;
+ } else {
+ status = Response.Status.INTERNAL_SERVER_ERROR;
+ }
+ if (doAudit) {
+ KMSWebApp.getKMSAudit().error(KMSMDCFilter.getUgi(),
+ KMSMDCFilter.getMethod(),
+ KMSMDCFilter.getURL(), getOneLineMessage(exception));
+ }
+ return createResponse(status, throwable);
+ }
+
+ protected void log(Response.Status status, Throwable ex) {
+ UserGroupInformation ugi = KMSMDCFilter.getUgi();
+ String method = KMSMDCFilter.getMethod();
+ String url = KMSMDCFilter.getURL();
+ String msg = getOneLineMessage(ex);
+ LOG.warn("User:'{}' Method:{} URL:{} Response:{}-{}", ugi, method, url,
+ status, msg, ex);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java
new file mode 100644
index 0000000..6918015
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.jmx.JMXJsonServlet;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import java.io.IOException;
+
+@InterfaceAudience.Private
+public class KMSJMXServlet extends JMXJsonServlet {
+
+ @Override
+ protected boolean isInstrumentationAccessAllowed(HttpServletRequest request,
+ HttpServletResponse response) throws IOException {
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java
new file mode 100644
index 0000000..d3e0064
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.ext.MessageBodyReader;
+import javax.ws.rs.ext.Provider;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Type;
+import java.util.Map;
+
+@Provider
+@Consumes(MediaType.APPLICATION_JSON)
+@InterfaceAudience.Private
+public class KMSJSONReader implements MessageBodyReader<Map> {
+
+ @Override
+ public boolean isReadable(Class<?> type, Type genericType,
+ Annotation[] annotations, MediaType mediaType) {
+ return type.isAssignableFrom(Map.class);
+ }
+
+ @Override
+ public Map readFrom(Class<Map> type, Type genericType,
+ Annotation[] annotations, MediaType mediaType,
+ MultivaluedMap<String, String> httpHeaders, InputStream entityStream)
+ throws IOException, WebApplicationException {
+ ObjectMapper mapper = new ObjectMapper();
+ return mapper.readValue(entityStream, type);
+ }
+}