You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ve...@apache.org on 2015/03/24 18:27:40 UTC
[1/4] incubator-ranger git commit: (RANGER-247)Development of Ranger
Key Storage Provider
Repository: incubator-ranger
Updated Branches:
refs/heads/master ce139e013 -> bb0bdcede
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
----------------------------------------------------------------------
diff --git a/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java b/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
new file mode 100644
index 0000000..04daeee
--- /dev/null
+++ b/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import java.io.ByteArrayOutputStream;
+import java.io.FilterOutputStream;
+import java.io.OutputStream;
+import java.io.PrintStream;
+
+import org.apache.hadoop.crypto.key.kms.server.KMS.KMSOp;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.PropertyConfigurator;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class TestKMSAudit {
+
+ private PrintStream originalOut;
+ private ByteArrayOutputStream memOut;
+ private FilterOut filterOut;
+ private PrintStream capturedOut;
+
+ private KMSAudit kmsAudit;
+
+ private static class FilterOut extends FilterOutputStream {
+ public FilterOut(OutputStream out) {
+ super(out);
+ }
+
+ public void setOutputStream(OutputStream out) {
+ this.out = out;
+ }
+ }
+
+ @Before
+ public void setUp() {
+ originalOut = System.err;
+ memOut = new ByteArrayOutputStream();
+ filterOut = new FilterOut(memOut);
+ capturedOut = new PrintStream(filterOut);
+ System.setErr(capturedOut);
+ PropertyConfigurator.configure(Thread.currentThread().
+ getContextClassLoader()
+ .getResourceAsStream("log4j-kmsaudit.properties"));
+ this.kmsAudit = new KMSAudit(1000);
+ }
+
+ @After
+ public void cleanUp() {
+ System.setErr(originalOut);
+ LogManager.resetConfiguration();
+ kmsAudit.shutdown();
+ }
+
+ private String getAndResetLogOutput() {
+ capturedOut.flush();
+ String logOutput = new String(memOut.toByteArray());
+ memOut = new ByteArrayOutputStream();
+ filterOut.setOutputStream(memOut);
+ return logOutput;
+ }
+
+ @Test
+ public void testAggregation() throws Exception {
+ UserGroupInformation luser = Mockito.mock(UserGroupInformation.class);
+ Mockito.when(luser.getShortUserName()).thenReturn("luser");
+ kmsAudit.ok(luser, KMSOp.DECRYPT_EEK, "k1", "testmsg");
+ kmsAudit.ok(luser, KMSOp.DECRYPT_EEK, "k1", "testmsg");
+ kmsAudit.ok(luser, KMSOp.DECRYPT_EEK, "k1", "testmsg");
+ kmsAudit.ok(luser, KMSOp.DELETE_KEY, "k1", "testmsg");
+ kmsAudit.ok(luser, KMSOp.ROLL_NEW_VERSION, "k1", "testmsg");
+ kmsAudit.ok(luser, KMSOp.DECRYPT_EEK, "k1", "testmsg");
+ kmsAudit.ok(luser, KMSOp.DECRYPT_EEK, "k1", "testmsg");
+ kmsAudit.ok(luser, KMSOp.DECRYPT_EEK, "k1", "testmsg");
+ Thread.sleep(1500);
+ kmsAudit.ok(luser, KMSOp.DECRYPT_EEK, "k1", "testmsg");
+ Thread.sleep(1500);
+ String out = getAndResetLogOutput();
+ System.out.println(out);
+ Assert.assertTrue(
+ out.matches(
+ "OK\\[op=DECRYPT_EEK, key=k1, user=luser, accessCount=1, interval=[^m]{1,4}ms\\] testmsg"
+ // Not aggregated !!
+ + "OK\\[op=DELETE_KEY, key=k1, user=luser\\] testmsg"
+ + "OK\\[op=ROLL_NEW_VERSION, key=k1, user=luser\\] testmsg"
+ // Aggregated
+ + "OK\\[op=DECRYPT_EEK, key=k1, user=luser, accessCount=6, interval=[^m]{1,4}ms\\] testmsg"
+ + "OK\\[op=DECRYPT_EEK, key=k1, user=luser, accessCount=1, interval=[^m]{1,4}ms\\] testmsg"));
+ }
+
+ @Test
+ public void testAggregationUnauth() throws Exception {
+ UserGroupInformation luser = Mockito.mock(UserGroupInformation.class);
+ Mockito.when(luser.getShortUserName()).thenReturn("luser");
+ kmsAudit.unauthorized(luser, KMSOp.GENERATE_EEK, "k2");
+ Thread.sleep(1000);
+ kmsAudit.ok(luser, KMSOp.GENERATE_EEK, "k3", "testmsg");
+ kmsAudit.ok(luser, KMSOp.GENERATE_EEK, "k3", "testmsg");
+ kmsAudit.ok(luser, KMSOp.GENERATE_EEK, "k3", "testmsg");
+ kmsAudit.ok(luser, KMSOp.GENERATE_EEK, "k3", "testmsg");
+ kmsAudit.ok(luser, KMSOp.GENERATE_EEK, "k3", "testmsg");
+ kmsAudit.unauthorized(luser, KMSOp.GENERATE_EEK, "k3");
+ kmsAudit.ok(luser, KMSOp.GENERATE_EEK, "k3", "testmsg");
+ Thread.sleep(2000);
+ String out = getAndResetLogOutput();
+ System.out.println(out);
+ Assert.assertTrue(
+ out.matches(
+ "UNAUTHORIZED\\[op=GENERATE_EEK, key=k2, user=luser\\] "
+ + "OK\\[op=GENERATE_EEK, key=k3, user=luser, accessCount=1, interval=[^m]{1,4}ms\\] testmsg"
+ + "OK\\[op=GENERATE_EEK, key=k3, user=luser, accessCount=5, interval=[^m]{1,4}ms\\] testmsg"
+ + "UNAUTHORIZED\\[op=GENERATE_EEK, key=k3, user=luser\\] "
+ + "OK\\[op=GENERATE_EEK, key=k3, user=luser, accessCount=1, interval=[^m]{1,4}ms\\] testmsg"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKeyAuthorizationKeyProvider.java
----------------------------------------------------------------------
diff --git a/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKeyAuthorizationKeyProvider.java b/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKeyAuthorizationKeyProvider.java
new file mode 100644
index 0000000..1db3d70
--- /dev/null
+++ b/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKeyAuthorizationKeyProvider.java
@@ -0,0 +1,271 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.net.URI;
+import java.security.PrivilegedExceptionAction;
+import java.security.SecureRandom;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
+import org.apache.hadoop.crypto.key.KeyProvider.Options;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
+import org.apache.hadoop.crypto.key.UserProvider;
+import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyACLs;
+import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyOpType;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestKeyAuthorizationKeyProvider {
+
+ private static final String CIPHER = "AES";
+
+ @Test
+ public void testCreateKey() throws Exception {
+ final Configuration conf = new Configuration();
+ KeyProvider kp =
+ new UserProvider.Factory().createProvider(new URI("user:///"), conf);
+ KeyACLs mock = mock(KeyACLs.class);
+ when(mock.isACLPresent("foo", KeyOpType.MANAGEMENT)).thenReturn(true);
+ UserGroupInformation u1 = UserGroupInformation.createRemoteUser("u1");
+ when(mock.hasAccessToKey("foo", u1, KeyOpType.MANAGEMENT)).thenReturn(true);
+ final KeyProviderCryptoExtension kpExt =
+ new KeyAuthorizationKeyProvider(
+ KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp),
+ mock);
+
+ u1.doAs(
+ new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ try {
+ kpExt.createKey("foo", SecureRandom.getSeed(16),
+ newOptions(conf));
+ } catch (IOException ioe) {
+ Assert.fail("User should be Authorized !!");
+ }
+
+ // "bar" key not configured
+ try {
+ kpExt.createKey("bar", SecureRandom.getSeed(16),
+ newOptions(conf));
+ Assert.fail("User should NOT be Authorized !!");
+ } catch (IOException ioe) {
+ // Ignore
+ }
+ return null;
+ }
+ }
+ );
+
+ // Unauthorized User
+ UserGroupInformation.createRemoteUser("badGuy").doAs(
+ new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ try {
+ kpExt.createKey("foo", SecureRandom.getSeed(16),
+ newOptions(conf));
+ Assert.fail("User should NOT be Authorized !!");
+ } catch (IOException ioe) {
+ // Ignore
+ }
+ return null;
+ }
+ }
+ );
+ }
+
+ @Test
+ public void testOpsWhenACLAttributeExists() throws Exception {
+ final Configuration conf = new Configuration();
+ KeyProvider kp =
+ new UserProvider.Factory().createProvider(new URI("user:///"), conf);
+ KeyACLs mock = mock(KeyACLs.class);
+ when(mock.isACLPresent("testKey", KeyOpType.MANAGEMENT)).thenReturn(true);
+ when(mock.isACLPresent("testKey", KeyOpType.GENERATE_EEK)).thenReturn(true);
+ when(mock.isACLPresent("testKey", KeyOpType.DECRYPT_EEK)).thenReturn(true);
+ when(mock.isACLPresent("testKey", KeyOpType.ALL)).thenReturn(true);
+ UserGroupInformation u1 = UserGroupInformation.createRemoteUser("u1");
+ UserGroupInformation u2 = UserGroupInformation.createRemoteUser("u2");
+ UserGroupInformation u3 = UserGroupInformation.createRemoteUser("u3");
+ UserGroupInformation sudo = UserGroupInformation.createRemoteUser("sudo");
+ when(mock.hasAccessToKey("testKey", u1, KeyOpType.MANAGEMENT)).thenReturn(true);
+ when(mock.hasAccessToKey("testKey", u2, KeyOpType.GENERATE_EEK)).thenReturn(true);
+ when(mock.hasAccessToKey("testKey", u3, KeyOpType.DECRYPT_EEK)).thenReturn(true);
+ when(mock.hasAccessToKey("testKey", sudo, KeyOpType.ALL)).thenReturn(true);
+ final KeyProviderCryptoExtension kpExt =
+ new KeyAuthorizationKeyProvider(
+ KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp),
+ mock);
+
+ final KeyVersion barKv = u1.doAs(
+ new PrivilegedExceptionAction<KeyVersion>() {
+ @Override
+ public KeyVersion run() throws Exception {
+ Options opt = newOptions(conf);
+ Map<String, String> m = new HashMap<String, String>();
+ m.put("key.acl.name", "testKey");
+ opt.setAttributes(m);
+ try {
+ KeyVersion kv =
+ kpExt.createKey("foo", SecureRandom.getSeed(16), opt);
+ kpExt.rollNewVersion(kv.getName());
+ kpExt.rollNewVersion(kv.getName(), SecureRandom.getSeed(16));
+ kpExt.deleteKey(kv.getName());
+ } catch (IOException ioe) {
+ Assert.fail("User should be Authorized !!");
+ }
+
+ KeyVersion retkv = null;
+ try {
+ retkv = kpExt.createKey("bar", SecureRandom.getSeed(16), opt);
+ kpExt.generateEncryptedKey(retkv.getName());
+ Assert.fail("User should NOT be Authorized to generate EEK !!");
+ } catch (IOException ioe) {
+ }
+ Assert.assertNotNull(retkv);
+ return retkv;
+ }
+ }
+ );
+
+ final EncryptedKeyVersion barEKv =
+ u2.doAs(
+ new PrivilegedExceptionAction<EncryptedKeyVersion>() {
+ @Override
+ public EncryptedKeyVersion run() throws Exception {
+ try {
+ kpExt.deleteKey(barKv.getName());
+ Assert.fail("User should NOT be Authorized to "
+ + "perform any other operation !!");
+ } catch (IOException ioe) {
+ }
+ return kpExt.generateEncryptedKey(barKv.getName());
+ }
+ });
+
+ u3.doAs(
+ new PrivilegedExceptionAction<KeyVersion>() {
+ @Override
+ public KeyVersion run() throws Exception {
+ try {
+ kpExt.deleteKey(barKv.getName());
+ Assert.fail("User should NOT be Authorized to "
+ + "perform any other operation !!");
+ } catch (IOException ioe) {
+ }
+ return kpExt.decryptEncryptedKey(barEKv);
+ }
+ });
+
+ sudo.doAs(
+ new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ Options opt = newOptions(conf);
+ Map<String, String> m = new HashMap<String, String>();
+ m.put("key.acl.name", "testKey");
+ opt.setAttributes(m);
+ try {
+ KeyVersion kv =
+ kpExt.createKey("foo", SecureRandom.getSeed(16), opt);
+ kpExt.rollNewVersion(kv.getName());
+ kpExt.rollNewVersion(kv.getName(), SecureRandom.getSeed(16));
+ EncryptedKeyVersion ekv = kpExt.generateEncryptedKey(kv.getName());
+ kpExt.decryptEncryptedKey(ekv);
+ kpExt.deleteKey(kv.getName());
+ } catch (IOException ioe) {
+ Assert.fail("User should be Allowed to do everything !!");
+ }
+ return null;
+ }
+ }
+ );
+ }
+
+ private static KeyProvider.Options newOptions(Configuration conf) {
+ KeyProvider.Options options = new KeyProvider.Options(conf);
+ options.setCipher(CIPHER);
+ options.setBitLength(128);
+ return options;
+ }
+
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testDecryptWithKeyVersionNameKeyMismatch() throws Exception {
+ final Configuration conf = new Configuration();
+ KeyProvider kp =
+ new UserProvider.Factory().createProvider(new URI("user:///"), conf);
+ KeyACLs mock = mock(KeyACLs.class);
+ when(mock.isACLPresent("testKey", KeyOpType.MANAGEMENT)).thenReturn(true);
+ when(mock.isACLPresent("testKey", KeyOpType.GENERATE_EEK)).thenReturn(true);
+ when(mock.isACLPresent("testKey", KeyOpType.DECRYPT_EEK)).thenReturn(true);
+ when(mock.isACLPresent("testKey", KeyOpType.ALL)).thenReturn(true);
+ UserGroupInformation u1 = UserGroupInformation.createRemoteUser("u1");
+ UserGroupInformation u2 = UserGroupInformation.createRemoteUser("u2");
+ UserGroupInformation u3 = UserGroupInformation.createRemoteUser("u3");
+ UserGroupInformation sudo = UserGroupInformation.createRemoteUser("sudo");
+ when(mock.hasAccessToKey("testKey", u1,
+ KeyOpType.MANAGEMENT)).thenReturn(true);
+ when(mock.hasAccessToKey("testKey", u2,
+ KeyOpType.GENERATE_EEK)).thenReturn(true);
+ when(mock.hasAccessToKey("testKey", u3,
+ KeyOpType.DECRYPT_EEK)).thenReturn(true);
+ when(mock.hasAccessToKey("testKey", sudo,
+ KeyOpType.ALL)).thenReturn(true);
+ final KeyProviderCryptoExtension kpExt =
+ new KeyAuthorizationKeyProvider(
+ KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp),
+ mock);
+
+ sudo.doAs(
+ new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ Options opt = newOptions(conf);
+ Map<String, String> m = new HashMap<String, String>();
+ m.put("key.acl.name", "testKey");
+ opt.setAttributes(m);
+ KeyVersion kv =
+ kpExt.createKey("foo", SecureRandom.getSeed(16), opt);
+ kpExt.rollNewVersion(kv.getName());
+ kpExt.rollNewVersion(kv.getName(), SecureRandom.getSeed(16));
+ EncryptedKeyVersion ekv = kpExt.generateEncryptedKey(kv.getName());
+ ekv = EncryptedKeyVersion.createForDecryption(
+ ekv.getEncryptionKeyName() + "x",
+ ekv.getEncryptionKeyVersionName(),
+ ekv.getEncryptedKeyIv(),
+ ekv.getEncryptedKeyVersion().getMaterial());
+ kpExt.decryptEncryptedKey(ekv);
+ return null;
+ }
+ }
+ );
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 94e0e3c..0f9ccfb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -175,6 +175,28 @@
<powermock.version>1.5.6</powermock.version>
<aspectj.version>1.8.2</aspectj.version>
<findbugs.plugin.version>3.0.0</findbugs.plugin.version>
+ <hadoop.minikdc.version>2.3.0</hadoop.minikdc.version>
+ <jersey-server.version>1.9</jersey-server.version>
+ <asm.all.version>3.2</asm.all.version>
+ <mortbay.jetty.version>6.1.26</mortbay.jetty.version>
+ <metrics.core.version>3.0.2</metrics.core.version>
+ <curator.test.version>2.7.0</curator.test.version>
+ <servlet.api.version>2.5</servlet.api.version>
+ <httpcomponents.httpclient.version>4.2.5</httpcomponents.httpclient.version>
+ <xz.version>1.0</xz.version>
+ <xmlenc.version>0.52</xmlenc.version>
+ <snappy-java.version>1.0.4.1</snappy-java.version>
+ <protobuf-java.version>2.5.0</protobuf-java.version>
+ <paranamer.version>2.3</paranamer.version>
+ <netty.version>3.6.2.Final</netty.version>
+ <jsr305.version>1.3.9</jsr305.version>
+ <jsch.version>0.1.42</jsch.version>
+ <jline.version>0.9.94</jline.version>
+ <jettison.version>1.1</jettison.version>
+ <jaxb-impl.version>2.2.3-1</jaxb-impl.version>
+ <jaxb-api.version>2.2.2</jaxb-api.version>
+ <jackson.version>1.9.13</jackson.version>
+ <sun-jersey-bundle.version>1.19</sun-jersey-bundle.version>
<distMgmtStagingId>apache.staging.https</distMgmtStagingId>
<distMgmtStagingName>Apache Release Distribution Repository</distMgmtStagingName>
<distMgmtStagingUrl>https://repository.apache.org/service/local/staging/deploy/maven2</distMgmtStagingUrl>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/src/main/assembly/kms.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/kms.xml b/src/main/assembly/kms.xml
index 80f907c..0cd4673 100644
--- a/src/main/assembly/kms.xml
+++ b/src/main/assembly/kms.xml
@@ -23,27 +23,84 @@
</formats>
<baseDirectory>${project.name}-${project.version}-kms</baseDirectory>
<includeBaseDirectory>true</includeBaseDirectory>
- <moduleSets>
-
- <moduleSet>
+
+ <moduleSets>
+ <moduleSet>
<binaries>
<includeDependencies>false</includeDependencies>
<unpack>false</unpack>
<outputDirectory>/ews/webapp/lib</outputDirectory>
<dependencySets>
<dependencySet>
- <outputDirectory>/ews/webapp/root</outputDirectory>
- <unpack>false</unpack>
- <includes>
- <include>org.apache.hadoop:hadoop-kms:war:2.6.0</include>
- </includes>
- </dependencySet>
- <dependencySet>
<outputDirectory>/ews/webapp/lib</outputDirectory>
<unpack>false</unpack>
<includes>
<include>org.apache.hadoop:hadoop-common:jar:${hadoop-common.version}</include>
- </includes>
+ <include>org.apache.hadoop:hadoop-auth:jar:${hadoop-auth.version}</include>
+ <include>org.eclipse.persistence:eclipselink</include>
+ <include>org.eclipse.persistence:javax.persistence</include>
+ <include>com.googlecode.log4jdbc:log4jdbc</include>
+ <include>log4j:log4j</include>
+ <include>org.slf4j:slf4j-api</include>
+ <include>org.slf4j:slf4j-log4j12</include>
+ <include>com.codahale.metrics:metrics-core</include>
+ <include>org.slf4j:jul-to-slf4j</include>
+ <include>commons-logging:commons-logging</include>
+ <include>com.google.guava:guava</include>
+ <include>com.sun.jersey:jersey-core</include>
+ <include>com.sun.jersey:jersey-server</include>
+ <include>javax.servlet:servlet-api</include>
+ <include>org.mortbay.jetty:jetty</include>
+ <include>org.mortbay.jetty:jetty-util</include>
+ <include>commons-collections:commons-collections</include>
+ <include>commons-lang:commons-lang</include>
+ <include>org.apache.zookeeper:zookeeper</include>
+ <include>org.apache.curator:curator-framework</include>
+ <include>org.apache.curator:curator-client</include>
+ <include>org.apache.curator:curator-test</include>
+ <include>asm:asm-all</include>
+ <include>com.sun.jersey:jersey-bundle</include>
+ <include>org.apache.httpcomponents:httpclient</include>
+ <include>javax.activation:activation</include>
+ <include>org.apache.directory.server:apacheds-i18n</include>
+ <include>org.apache.directory.server:apacheds-kerberos-codec</include>
+ <include>org.apache.directory.api:api-asn1-api</include>
+ <include>org.apache.directory.api:api-i18n</include>
+ <include>org.apache.directory.api:api-util</include>
+ <include>org.apache.avro:avro</include>
+ <include>commons-beanutils:commons-beanutils</include>
+ <include>commons-beanutils:commons-beanutils-core</include>
+ <include>commons-cli:commons-cli</include>
+ <include>commons-codec:commons-codec</include>
+ <include>org.apache.commons:commons-compress</include>
+ <include>commons-configuration:commons-configuration</include>
+ <include>commons-digester:commons-digester</include>
+ <include>commons-io:commons-io</include>
+ <include>org.apache.commons:commons-math3</include>
+ <include>commons-net:commons-net</include>
+ <include>org.apache.curator:curator-recipes</include>
+ <include>com.google.code.gson:gson</include>
+ <include>org.apache.hadoop:hadoop-annotations</include>
+ <include>org.htrace:htrace-core</include>
+ <include>org.apache.httpcomponents:httpcore</include>
+ <include>org.codehaus.jackson:jackson-core-asl</include>
+ <include>org.codehaus.jackson:jackson-jaxrs</include>
+ <include>org.codehaus.jackson:jackson-mapper-asl</include>
+ <include>org.codehaus.jackson:jackson-xc</include>
+ <include>javax.xml.bind:jaxb-api</include>
+ <include>com.sun.xml.bind:jaxb-impl</include>
+ <include>com.sun.jersey:jersey-json</include>
+ <include>org.codehaus.jettison:jettison</include>
+ <include>jline:jline</include>
+ <include>com.jcraft:jsch</include>
+ <include>com.google.code.findbugs:jsr305</include>
+ <include>io.netty:netty</include>
+ <include>com.thoughtworks.paranamer:paranamer</include>
+ <include>com.google.protobuf:protobuf-java</include>
+ <include>org.xerial.snappy:snappy-java</include>
+ <include>xmlenc:xmlenc</include>
+ <include>org.tukaani:xz</include>
+ </includes>
</dependencySet>
</dependencySets>
</binaries>
@@ -51,6 +108,17 @@
<include>org.apache.ranger:ranger-kms</include>
</includes>
</moduleSet>
+
+ <moduleSet>
+ <binaries>
+ <includeDependencies>false</includeDependencies>
+ <unpack>true</unpack>
+ <outputDirectory>/ews/webapp</outputDirectory>
+ </binaries>
+ <includes>
+ <include>org.apache.ranger:ranger-kms</include>
+ </includes>
+ </moduleSet>
<moduleSet>
<binaries>
@@ -75,37 +143,68 @@
</includes>
</moduleSet>
+ <moduleSet>
+ <binaries>
+ <includeDependencies>false</includeDependencies>
+ <outputDirectory>/jisql/lib</outputDirectory>
+ <unpack>false</unpack>
+ <directoryMode>755</directoryMode>
+ <fileMode>644</fileMode>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>/jisql/lib</outputDirectory>
+ <unpack>false</unpack>
+ <includes>
+ <include>net.sourceforge.javacsv:javacsv</include>
+ <include>net.sf.jopt-simple:jopt-simple</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+ </binaries>
+ <includes>
+ <include>org.apache.ranger:jisql</include>
+ </includes>
+ </moduleSet>
</moduleSets>
-
+
<fileSets>
-
- <fileSet>
- <outputDirectory>/ews/webapp/config</outputDirectory>
- <directoryMode>755</directoryMode>
- <directory>kms/config/kms-webapp</directory>
- <fileMode>400</fileMode>
- </fileSet>
-
- <fileSet>
- <outputDirectory>/ews/conf</outputDirectory>
+ <fileSet>
+ <outputDirectory>/ews/webapp/WEB-INF/classes/conf.dist</outputDirectory>
+ <directory>embeddedwebserver/conf</directory>
+ <fileMode>700</fileMode>
+ </fileSet>
+ <fileSet>
+ <outputDirectory>/scripts/db/mysql</outputDirectory>
<directoryMode>755</directoryMode>
- <directory>kms/config/webserver</directory>
- <fileMode>400</fileMode>
- </fileSet>
-
- <fileSet>
+ <directory>kms/scripts/db/mysql</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <fileMode>544</fileMode>
+ </fileSet>
+ <fileSet>
<outputDirectory>/scripts</outputDirectory>
- <directoryMode>755</directoryMode>
<directory>kms/scripts</directory>
<includes>
- <include>ranger-*</include>
+ <include>db_setup.py</include>
+ <include>ranger-kms</include>
+ <include>ranger-kms-services.sh</include>
+ <include>setup.sh</include>
+ <include>migrate-keystore-dbks.sh</include>
</includes>
<fileMode>544</fileMode>
+ </fileSet>
+ <fileSet>
+ <outputDirectory>/ews/webapp/config</outputDirectory>
+ <directoryMode>0500</directoryMode>
+ <directory>kms/config/kms-webapp</directory>
+ <fileMode>0400</fileMode>
+ </fileSet>
+ <fileSet>
+ <outputDirectory>/ews/conf</outputDirectory>
+ <directoryMode>0500</directoryMode>
+ <directory>kms/config/webserver</directory>
+ <fileMode>0400</fileMode>
</fileSet>
-
-
</fileSets>
-
-
-
</assembly>
[2/4] incubator-ranger git commit: (RANGER-247)Development of Ranger
Key Storage Provider
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
new file mode 100644
index 0000000..3674e7a
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import javax.ws.rs.Produces;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.ext.MessageBodyWriter;
+import javax.ws.rs.ext.Provider;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Type;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Jersey provider that converts <code>Map</code>s and <code>List</code>s
+ * to their JSON representation.
+ */
+@Provider
+@Produces(MediaType.APPLICATION_JSON)
+@InterfaceAudience.Private
+public class KMSJSONWriter implements MessageBodyWriter<Object> {
+
+ @Override
+ public boolean isWriteable(Class<?> aClass, Type type,
+ Annotation[] annotations, MediaType mediaType) {
+ return Map.class.isAssignableFrom(aClass) ||
+ List.class.isAssignableFrom(aClass);
+ }
+
+ @Override
+ public long getSize(Object obj, Class<?> aClass, Type type,
+ Annotation[] annotations, MediaType mediaType) {
+ return -1;
+ }
+
+ @Override
+ public void writeTo(Object obj, Class<?> aClass, Type type,
+ Annotation[] annotations, MediaType mediaType,
+ MultivaluedMap<String, Object> stringObjectMultivaluedMap,
+ OutputStream outputStream) throws IOException, WebApplicationException {
+ Writer writer = new OutputStreamWriter(outputStream);
+ ObjectMapper jsonMapper = new ObjectMapper();
+ jsonMapper.writerWithDefaultPrettyPrinter().writeValue(writer, obj);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSMDCFilter.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSMDCFilter.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSMDCFilter.java
new file mode 100644
index 0000000..2a3c149
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSMDCFilter.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.delegation.web.HttpUserGroupInformation;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import java.io.IOException;
+
+/**
+ * Servlet filter that captures context of the HTTP request to be use in the
+ * scope of KMS calls on the server side.
+ */
+@InterfaceAudience.Private
+public class KMSMDCFilter implements Filter {
+
+ private static class Data {
+ private UserGroupInformation ugi;
+ private String method;
+ private StringBuffer url;
+
+ private Data(UserGroupInformation ugi, String method, StringBuffer url) {
+ this.ugi = ugi;
+ this.method = method;
+ this.url = url;
+ }
+ }
+
+ private static ThreadLocal<Data> DATA_TL = new ThreadLocal<Data>();
+
+ public static UserGroupInformation getUgi() {
+ return DATA_TL.get().ugi;
+ }
+
+ public static String getMethod() {
+ return DATA_TL.get().method;
+ }
+
+ public static String getURL() {
+ return DATA_TL.get().url.toString();
+ }
+
+ @Override
+ public void init(FilterConfig config) throws ServletException {
+ }
+
+ @Override
+ public void doFilter(ServletRequest request, ServletResponse response,
+ FilterChain chain)
+ throws IOException, ServletException {
+ try {
+ DATA_TL.remove();
+ UserGroupInformation ugi = HttpUserGroupInformation.get();
+ String method = ((HttpServletRequest) request).getMethod();
+ StringBuffer requestURL = ((HttpServletRequest) request).getRequestURL();
+ String queryString = ((HttpServletRequest) request).getQueryString();
+ if (queryString != null) {
+ requestURL.append("?").append(queryString);
+ }
+ DATA_TL.set(new Data(ugi, method, requestURL));
+ chain.doFilter(request, response);
+ } finally {
+ DATA_TL.remove();
+ }
+ }
+
+ @Override
+ public void destroy() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
new file mode 100644
index 0000000..24af81b
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
+import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * JSON utility methods for the KMS.
+ */
+@InterfaceAudience.Private
+public class KMSServerJSONUtils {
+ @SuppressWarnings("unchecked")
+ public static Map toJSON(KeyProvider.KeyVersion keyVersion) {
+ Map json = new LinkedHashMap();
+ if (keyVersion != null) {
+ json.put(KMSRESTConstants.NAME_FIELD,
+ keyVersion.getName());
+ json.put(KMSRESTConstants.VERSION_NAME_FIELD,
+ keyVersion.getVersionName());
+ json.put(KMSRESTConstants.MATERIAL_FIELD,
+ Base64.encodeBase64URLSafeString(
+ keyVersion.getMaterial()));
+ }
+ return json;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static List toJSON(List<KeyProvider.KeyVersion> keyVersions) {
+ List json = new ArrayList();
+ if (keyVersions != null) {
+ for (KeyProvider.KeyVersion version : keyVersions) {
+ json.add(toJSON(version));
+ }
+ }
+ return json;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static Map toJSON(EncryptedKeyVersion encryptedKeyVersion) {
+ Map json = new LinkedHashMap();
+ if (encryptedKeyVersion != null) {
+ json.put(KMSRESTConstants.VERSION_NAME_FIELD,
+ encryptedKeyVersion.getEncryptionKeyVersionName());
+ json.put(KMSRESTConstants.IV_FIELD,
+ Base64.encodeBase64URLSafeString(
+ encryptedKeyVersion.getEncryptedKeyIv()));
+ json.put(KMSRESTConstants.ENCRYPTED_KEY_VERSION_FIELD,
+ toJSON(encryptedKeyVersion.getEncryptedKeyVersion()));
+ }
+ return json;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static Map toJSON(String keyName, KeyProvider.Metadata meta) {
+ Map json = new LinkedHashMap();
+ if (meta != null) {
+ json.put(KMSRESTConstants.NAME_FIELD, keyName);
+ json.put(KMSRESTConstants.CIPHER_FIELD, meta.getCipher());
+ json.put(KMSRESTConstants.LENGTH_FIELD, meta.getBitLength());
+ json.put(KMSRESTConstants.DESCRIPTION_FIELD, meta.getDescription());
+ json.put(KMSRESTConstants.ATTRIBUTES_FIELD, meta.getAttributes());
+ json.put(KMSRESTConstants.CREATED_FIELD,
+ meta.getCreated().getTime());
+ json.put(KMSRESTConstants.VERSIONS_FIELD,
+ (long) meta.getVersions());
+ }
+ return json;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static List toJSON(String[] keyNames, KeyProvider.Metadata[] metas) {
+ List json = new ArrayList();
+ for (int i = 0; i < keyNames.length; i++) {
+ json.add(toJSON(keyNames[i], metas[i]));
+ }
+ return json;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
new file mode 100644
index 0000000..67b9f88
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
@@ -0,0 +1,307 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import com.codahale.metrics.JmxReporter;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.CachingKeyProvider;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.crypto.key.KeyProviderFactory;
+import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyACLs;
+import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.log4j.PropertyConfigurator;
+import org.mortbay.log.Log;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.bridge.SLF4JBridgeHandler;
+
+import javax.servlet.ServletContextEvent;
+import javax.servlet.ServletContextListener;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URL;
+
+@InterfaceAudience.Private
+public class KMSWebApp implements ServletContextListener {
+
+ private static final String LOG4J_PROPERTIES = "kms-log4j.properties";
+
+ private static final String METRICS_PREFIX = "hadoop.kms.";
+ private static final String ADMIN_CALLS_METER = METRICS_PREFIX +
+ "admin.calls.meter";
+ private static final String KEY_CALLS_METER = METRICS_PREFIX +
+ "key.calls.meter";
+ private static final String INVALID_CALLS_METER = METRICS_PREFIX +
+ "invalid.calls.meter";
+ private static final String UNAUTHORIZED_CALLS_METER = METRICS_PREFIX +
+ "unauthorized.calls.meter";
+ private static final String UNAUTHENTICATED_CALLS_METER = METRICS_PREFIX +
+ "unauthenticated.calls.meter";
+ private static final String GENERATE_EEK_METER = METRICS_PREFIX +
+ "generate_eek.calls.meter";
+ private static final String DECRYPT_EEK_METER = METRICS_PREFIX +
+ "decrypt_eek.calls.meter";
+
+ private static Logger LOG;
+ private static MetricRegistry metricRegistry;
+
+ private JmxReporter jmxReporter;
+ private static Configuration kmsConf;
+ private static KeyACLs kmsAcls;
+ private static Meter adminCallsMeter;
+ private static Meter keyCallsMeter;
+ private static Meter unauthorizedCallsMeter;
+ private static Meter unauthenticatedCallsMeter;
+ private static Meter decryptEEKCallsMeter;
+ private static Meter generateEEKCallsMeter;
+ private static Meter invalidCallsMeter;
+ private static KMSAudit kmsAudit;
+ private static KeyProviderCryptoExtension keyProviderCryptoExtension;
+
+ static {
+ SLF4JBridgeHandler.removeHandlersForRootLogger();
+ SLF4JBridgeHandler.install();
+ }
+
+ private void initLogging(String confDir) {
+ if (System.getProperty("log4j.configuration") == null) {
+ System.setProperty("log4j.defaultInitOverride", "true");
+ boolean fromClasspath = true;
+ File log4jConf = new File(confDir, LOG4J_PROPERTIES).getAbsoluteFile();
+ if (log4jConf.exists()) {
+ PropertyConfigurator.configureAndWatch(log4jConf.getPath(), 1000);
+ fromClasspath = false;
+ } else {
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+ URL log4jUrl = cl.getResource(LOG4J_PROPERTIES);
+ if (log4jUrl != null) {
+ PropertyConfigurator.configure(log4jUrl);
+ }
+ }
+ LOG = LoggerFactory.getLogger(KMSWebApp.class);
+ LOG.debug("KMS log starting");
+ if (fromClasspath) {
+ LOG.warn("Log4j configuration file '{}' not found", LOG4J_PROPERTIES);
+ LOG.warn("Logging with INFO level to standard output");
+ }
+ } else {
+ LOG = LoggerFactory.getLogger(KMSWebApp.class);
+ }
+ }
+
+ @Override
+ public void contextInitialized(ServletContextEvent sce) {
+ try {
+ String confDir = System.getProperty(KMSConfiguration.KMS_CONFIG_DIR);
+ if (confDir == null) {
+ throw new RuntimeException("System property '" +
+ KMSConfiguration.KMS_CONFIG_DIR + "' not defined");
+ }
+ kmsConf = KMSConfiguration.getKMSConf();
+ initLogging(confDir);
+ LOG.info("-------------------------------------------------------------");
+ LOG.info(" Java runtime version : {}", System.getProperty(
+ "java.runtime.version"));
+ LOG.info(" KMS Hadoop Version: " + VersionInfo.getVersion());
+ LOG.info("-------------------------------------------------------------");
+
+
+ kmsAcls = getAcls(kmsConf.get(KMSConfiguration.KMS_SECURITY_AUTHORIZER));
+
+ //kmsAcls = new KMSACLs();
+ kmsAcls.startACLReloader();
+
+ metricRegistry = new MetricRegistry();
+ jmxReporter = JmxReporter.forRegistry(metricRegistry).build();
+ jmxReporter.start();
+ generateEEKCallsMeter = metricRegistry.register(GENERATE_EEK_METER,
+ new Meter());
+ decryptEEKCallsMeter = metricRegistry.register(DECRYPT_EEK_METER,
+ new Meter());
+ adminCallsMeter = metricRegistry.register(ADMIN_CALLS_METER, new Meter());
+ keyCallsMeter = metricRegistry.register(KEY_CALLS_METER, new Meter());
+ invalidCallsMeter = metricRegistry.register(INVALID_CALLS_METER,
+ new Meter());
+ unauthorizedCallsMeter = metricRegistry.register(UNAUTHORIZED_CALLS_METER,
+ new Meter());
+ unauthenticatedCallsMeter = metricRegistry.register(
+ UNAUTHENTICATED_CALLS_METER, new Meter());
+
+ kmsAudit =
+ new KMSAudit(kmsConf.getLong(
+ KMSConfiguration.KMS_AUDIT_AGGREGATION_WINDOW,
+ KMSConfiguration.KMS_AUDIT_AGGREGATION_WINDOW_DEFAULT));
+
+ // this is required for the the JMXJsonServlet to work properly.
+ // the JMXJsonServlet is behind the authentication filter,
+ // thus the '*' ACL.
+ sce.getServletContext().setAttribute(HttpServer2.CONF_CONTEXT_ATTRIBUTE,
+ kmsConf);
+ sce.getServletContext().setAttribute(HttpServer2.ADMINS_ACL,
+ new AccessControlList(AccessControlList.WILDCARD_ACL_VALUE));
+
+ // intializing the KeyProvider
+ String providerString = kmsConf.get(KMSConfiguration.KEY_PROVIDER_URI);
+ if (providerString == null) {
+ throw new IllegalStateException("No KeyProvider has been defined");
+ }
+ Log.info("------------------ Ranger KMSWEbApp---------------------");
+ Log.info("provider string = "+providerString);
+ Log.info("URI = "+new URI(providerString).toString()+" scheme = "+new URI(providerString).getScheme());
+ Log.info("kmsconf size= "+kmsConf.size() + " kms classname="+kmsConf.getClass().getName());
+ Log.info("----------------INstantiating key provider ---------------");
+ KeyProvider keyProvider =
+ KeyProviderFactory.get(new URI(providerString), kmsConf);
+ Log.info("keyProvider = "+keyProvider.toString());
+ if (kmsConf.getBoolean(KMSConfiguration.KEY_CACHE_ENABLE,
+ KMSConfiguration.KEY_CACHE_ENABLE_DEFAULT)) {
+ long keyTimeOutMillis =
+ kmsConf.getLong(KMSConfiguration.KEY_CACHE_TIMEOUT_KEY,
+ KMSConfiguration.KEY_CACHE_TIMEOUT_DEFAULT);
+ long currKeyTimeOutMillis =
+ kmsConf.getLong(KMSConfiguration.CURR_KEY_CACHE_TIMEOUT_KEY,
+ KMSConfiguration.CURR_KEY_CACHE_TIMEOUT_DEFAULT);
+ keyProvider = new CachingKeyProvider(keyProvider, keyTimeOutMillis,
+ currKeyTimeOutMillis);
+ }
+ LOG.info("Initialized KeyProvider " + keyProvider);
+
+ keyProviderCryptoExtension = KeyProviderCryptoExtension.
+ createKeyProviderCryptoExtension(keyProvider);
+ keyProviderCryptoExtension =
+ new EagerKeyGeneratorKeyProviderCryptoExtension(kmsConf,
+ keyProviderCryptoExtension);
+ if (kmsConf.getBoolean(KMSConfiguration.KEY_AUTHORIZATION_ENABLE,
+ KMSConfiguration.KEY_AUTHORIZATION_ENABLE_DEFAULT)) {
+ keyProviderCryptoExtension =
+ new KeyAuthorizationKeyProvider(
+ keyProviderCryptoExtension, kmsAcls);
+ }
+
+ LOG.info("Initialized KeyProviderCryptoExtension "
+ + keyProviderCryptoExtension);
+ final int defaultBitlength = kmsConf
+ .getInt(KeyProvider.DEFAULT_BITLENGTH_NAME,
+ KeyProvider.DEFAULT_BITLENGTH);
+ LOG.info("Default key bitlength is {}", defaultBitlength);
+ LOG.info("Ranger KMS Started");
+ } catch (Throwable ex) {
+ System.out.println();
+ System.out.println("ERROR: Hadoop KMS could not be started");
+ System.out.println();
+ System.out.println("REASON: " + ex.toString());
+ System.out.println();
+ System.out.println("Stacktrace:");
+ System.out.println("---------------------------------------------------");
+ ex.printStackTrace(System.out);
+ System.out.println("---------------------------------------------------");
+ System.out.println();
+ System.exit(1);
+ }
+ }
+
+ private KeyACLs getAcls(String clsStr) throws IOException {
+ KeyACLs keyAcl = null;
+ try {
+ Class<? extends KeyACLs> cls = null;
+ if (clsStr == null || clsStr.trim().equals("")) {
+ cls = KMSACLs.class;
+ } else {
+ //Class<?> configClass = Class.forName(clsStr, true, JavaUtils.getClassLoader());
+ Class<?> configClass = Class.forName(clsStr);
+ //Class<?> configClass = Class.forName(clsStr, true, JavaUtils.getClassLoader());
+ if(!KeyACLs.class.isAssignableFrom(configClass) ){
+ //if it's not of type KeyACLs
+ //we can have default also "cls = KMSACLs.class;"
+ return null;
+ }
+ cls = (Class<? extends KeyACLs>)configClass;
+ }
+ if (cls != null) {
+ keyAcl = ReflectionUtils.newInstance(cls, kmsConf);
+ }
+ } catch (Exception e) {
+ throw new IOException(e.getMessage());
+ }
+ return keyAcl;
+ }
+
+@Override
+ public void contextDestroyed(ServletContextEvent sce) {
+ kmsAudit.shutdown();
+ kmsAcls.stopACLReloader();
+ jmxReporter.stop();
+ jmxReporter.close();
+ metricRegistry = null;
+ LOG.info("KMS Stopped");
+ }
+
+ public static Configuration getConfiguration() {
+ return new Configuration(kmsConf);
+ }
+
+ public static KeyACLs getACLs() {
+ return kmsAcls;
+ }
+
+ public static Meter getAdminCallsMeter() {
+ return adminCallsMeter;
+ }
+
+ public static Meter getKeyCallsMeter() {
+ return keyCallsMeter;
+ }
+
+ public static Meter getInvalidCallsMeter() {
+ return invalidCallsMeter;
+ }
+
+ public static Meter getGenerateEEKCallsMeter() {
+ return generateEEKCallsMeter;
+ }
+
+ public static Meter getDecryptEEKCallsMeter() {
+ return decryptEEKCallsMeter;
+ }
+
+ public static Meter getUnauthorizedCallsMeter() {
+ return unauthorizedCallsMeter;
+ }
+
+ public static Meter getUnauthenticatedCallsMeter() {
+ return unauthenticatedCallsMeter;
+ }
+
+ public static KeyProviderCryptoExtension getKeyProvider() {
+ return keyProviderCryptoExtension;
+ }
+
+ public static KMSAudit getKMSAudit() {
+ return kmsAudit;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java
new file mode 100644
index 0000000..5099daf
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java
@@ -0,0 +1,299 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.crypto.key.kms.server;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.NoSuchAlgorithmException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.crypto.key.kms.server.KMS.KMSOp;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AuthorizationException;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * A {@link KeyProvider} proxy that checks whether the current user derived via
+ * {@link UserGroupInformation}, is authorized to perform the following
+ * type of operations on a Key :
+ * <ol>
+ * <li>MANAGEMENT operations : createKey, rollNewVersion, deleteKey</li>
+ * <li>GENERATE_EEK operations : generateEncryptedKey, warmUpEncryptedKeys</li>
+ * <li>DECRYPT_EEK operation : decryptEncryptedKey</li>
+ * <li>READ operations : getKeyVersion, getKeyVersions, getMetadata,
+ * getKeysMetadata, getCurrentKey</li>
+ * </ol>
+ * The read operations (getCurrentKeyVersion / getMetadata) etc are not checked.
+ */
+public class KeyAuthorizationKeyProvider extends KeyProviderCryptoExtension {
+
+ public static final String KEY_ACL = "key.acl.";
+ private static final String KEY_ACL_NAME = KEY_ACL + "name";
+
+ public enum KeyOpType {
+ ALL, READ, MANAGEMENT, GENERATE_EEK, DECRYPT_EEK;
+ }
+
+ /**
+ * Interface that needs to be implemented by a client of the
+ * <code>KeyAuthorizationKeyProvider</code>.
+ */
+ public static interface KeyACLs {
+
+ /**
+ * This is called by the KeyProvider to check if the given user is
+ * authorized to perform the specified operation on the given acl name.
+ * @param aclName name of the key ACL
+ * @param ugi User's UserGroupInformation
+ * @param opType Operation Type
+ * @return true if user has access to the aclName and opType else false
+ */
+ public boolean hasAccessToKey(String aclName, UserGroupInformation ugi,
+ KeyOpType opType);
+
+ /**
+ *
+ * @param aclName ACL name
+ * @param opType Operation Type
+ * @return true if AclName exists else false
+ */
+ public boolean isACLPresent(String aclName, KeyOpType opType);
+
+ public void startACLReloader();
+
+ public void stopACLReloader();
+
+ public boolean hasAccess(KMSACLsType.Type aclType, UserGroupInformation ugi);
+
+ public void assertAccess(KMSACLsType.Type aclType, UserGroupInformation ugi,
+ KMSOp operation, String key) throws AccessControlException;
+ }
+
+ private final KeyProviderCryptoExtension provider;
+ private final KeyACLs acls;
+
+ /**
+ * The constructor takes a {@link KeyProviderCryptoExtension} and an
+ * implementation of <code>KeyACLs</code>. All calls are delegated to the
+ * provider keyProvider after authorization check (if required)
+ * @param keyProvider
+ * @param acls
+ */
+ public KeyAuthorizationKeyProvider(KeyProviderCryptoExtension keyProvider,
+ KeyACLs acls) {
+ super(keyProvider, null);
+ this.provider = keyProvider;
+ this.acls = acls;
+ }
+
+ // This method first checks if "key.acl.name" attribute is present as an
+ // attribute in the provider Options. If yes, use the aclName for any
+ // subsequent access checks, else use the keyName as the aclName and set it
+ // as the value of the "key.acl.name" in the key's metadata.
+ private void authorizeCreateKey(String keyName, Options options,
+ UserGroupInformation ugi) throws IOException{
+ Preconditions.checkNotNull(ugi, "UserGroupInformation cannot be null");
+ Map<String, String> attributes = options.getAttributes();
+ String aclName = attributes.get(KEY_ACL_NAME);
+ boolean success = false;
+ if (Strings.isNullOrEmpty(aclName)) {
+ if (acls.isACLPresent(keyName, KeyOpType.MANAGEMENT)) {
+ options.setAttributes(ImmutableMap.<String, String> builder()
+ .putAll(attributes).put(KEY_ACL_NAME, keyName).build());
+ success =
+ acls.hasAccessToKey(keyName, ugi, KeyOpType.MANAGEMENT)
+ || acls.hasAccessToKey(keyName, ugi, KeyOpType.ALL);
+ } else {
+ success = false;
+ }
+ } else {
+ success = acls.isACLPresent(aclName, KeyOpType.MANAGEMENT) &&
+ (acls.hasAccessToKey(aclName, ugi, KeyOpType.MANAGEMENT)
+ || acls.hasAccessToKey(aclName, ugi, KeyOpType.ALL));
+ }
+ if (!success)
+ throw new AuthorizationException(String.format("User [%s] is not"
+ + " authorized to create key !!", ugi.getShortUserName()));
+ }
+
+ private void checkAccess(String aclName, UserGroupInformation ugi,
+ KeyOpType opType) throws AuthorizationException {
+ Preconditions.checkNotNull(aclName, "Key ACL name cannot be null");
+ Preconditions.checkNotNull(ugi, "UserGroupInformation cannot be null");
+ if (acls.isACLPresent(aclName, KeyOpType.MANAGEMENT) &&
+ (acls.hasAccessToKey(aclName, ugi, opType)
+ || acls.hasAccessToKey(aclName, ugi, KeyOpType.ALL))) {
+ return;
+ } else {
+ throw new AuthorizationException(String.format("User [%s] is not"
+ + " authorized to perform [%s] on key with ACL name [%s]!!",
+ ugi.getShortUserName(), opType, aclName));
+ }
+ }
+
+ @Override
+ public KeyVersion createKey(String name, Options options)
+ throws NoSuchAlgorithmException, IOException {
+ authorizeCreateKey(name, options, getUser());
+ return provider.createKey(name, options);
+ }
+
+ @Override
+ public KeyVersion createKey(String name, byte[] material, Options options)
+ throws IOException {
+ authorizeCreateKey(name, options, getUser());
+ return provider.createKey(name, material, options);
+ }
+
+ @Override
+ public KeyVersion rollNewVersion(String name)
+ throws NoSuchAlgorithmException, IOException {
+ doAccessCheck(name, KeyOpType.MANAGEMENT);
+ return provider.rollNewVersion(name);
+ }
+
+ @Override
+ public void deleteKey(String name) throws IOException {
+ doAccessCheck(name, KeyOpType.MANAGEMENT);
+ provider.deleteKey(name);
+ }
+
+ @Override
+ public KeyVersion rollNewVersion(String name, byte[] material)
+ throws IOException {
+ doAccessCheck(name, KeyOpType.MANAGEMENT);
+ return provider.rollNewVersion(name, material);
+ }
+
+ @Override
+ public void warmUpEncryptedKeys(String... names) throws IOException {
+ for (String name : names) {
+ doAccessCheck(name, KeyOpType.GENERATE_EEK);
+ }
+ provider.warmUpEncryptedKeys(names);
+ }
+
+ @Override
+ public EncryptedKeyVersion generateEncryptedKey(String encryptionKeyName)
+ throws IOException, GeneralSecurityException {
+ doAccessCheck(encryptionKeyName, KeyOpType.GENERATE_EEK);
+ return provider.generateEncryptedKey(encryptionKeyName);
+ }
+
+ private void verifyKeyVersionBelongsToKey(EncryptedKeyVersion ekv)
+ throws IOException {
+ String kn = ekv.getEncryptionKeyName();
+ String kvn = ekv.getEncryptionKeyVersionName();
+ KeyVersion kv = provider.getKeyVersion(kvn);
+ if (!kv.getName().equals(kn)) {
+ throw new IllegalArgumentException(String.format(
+ "KeyVersion '%s' does not belong to the key '%s'", kvn, kn));
+ }
+ }
+
+ @Override
+ public KeyVersion decryptEncryptedKey(EncryptedKeyVersion encryptedKeyVersion)
+ throws IOException, GeneralSecurityException {
+ verifyKeyVersionBelongsToKey(encryptedKeyVersion);
+ doAccessCheck(
+ encryptedKeyVersion.getEncryptionKeyName(), KeyOpType.DECRYPT_EEK);
+ return provider.decryptEncryptedKey(encryptedKeyVersion);
+ }
+
+ @Override
+ public KeyVersion getKeyVersion(String versionName) throws IOException {
+ KeyVersion keyVersion = provider.getKeyVersion(versionName);
+ if (keyVersion != null) {
+ doAccessCheck(keyVersion.getName(), KeyOpType.READ);
+ }
+ return keyVersion;
+ }
+
+ @Override
+ public List<String> getKeys() throws IOException {
+ return provider.getKeys();
+ }
+
+ @Override
+ public List<KeyVersion> getKeyVersions(String name) throws IOException {
+ doAccessCheck(name, KeyOpType.READ);
+ return provider.getKeyVersions(name);
+ }
+
+ @Override
+ public Metadata getMetadata(String name) throws IOException {
+ doAccessCheck(name, KeyOpType.READ);
+ return provider.getMetadata(name);
+ }
+
+ @Override
+ public Metadata[] getKeysMetadata(String... names) throws IOException {
+ for (String name : names) {
+ doAccessCheck(name, KeyOpType.READ);
+ }
+ return provider.getKeysMetadata(names);
+ }
+
+ @Override
+ public KeyVersion getCurrentKey(String name) throws IOException {
+ doAccessCheck(name, KeyOpType.READ);
+ return provider.getCurrentKey(name);
+ }
+
+ @Override
+ public void flush() throws IOException {
+ provider.flush();
+ }
+
+ @Override
+ public boolean isTransient() {
+ return provider.isTransient();
+ }
+
+ private void doAccessCheck(String keyName, KeyOpType opType) throws
+ IOException {
+ Metadata metadata = provider.getMetadata(keyName);
+ if (metadata != null) {
+ String aclName = metadata.getAttributes().get(KEY_ACL_NAME);
+ checkAccess((aclName == null) ? keyName : aclName, getUser(), opType);
+ }
+ }
+
+ private UserGroupInformation getUser() throws IOException {
+ return UserGroupInformation.getCurrentUser();
+ }
+
+ @Override
+ protected KeyProvider getKeyProvider() {
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return provider.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/ranger/entity/XXDBBase.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/ranger/entity/XXDBBase.java b/kms/src/main/java/org/apache/ranger/entity/XXDBBase.java
new file mode 100644
index 0000000..08ab1f7
--- /dev/null
+++ b/kms/src/main/java/org/apache/ranger/entity/XXDBBase.java
@@ -0,0 +1,216 @@
+package org.apache.ranger.entity;
+
+/**
+ * Base JPA class with id, versionNumber and other common attributes
+ *
+ */
+
+import java.util.Calendar;
+import java.util.Date;
+import java.util.GregorianCalendar;
+import java.util.TimeZone;
+
+import javax.persistence.Column;
+import javax.persistence.MappedSuperclass;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@MappedSuperclass
+@XmlRootElement
+public abstract class XXDBBase extends Object implements java.io.Serializable {
+ private static final long serialVersionUID = 1L;
+ public static final int CLASS_TYPE_NONE = 0;
+ private static final TimeZone gmtTimeZone = TimeZone.getTimeZone("GMT+0");
+
+ /**
+ * Date/Time creation of this user.
+ * <ul>
+ * </ul>
+ *
+ */
+ @Temporal(TemporalType.TIMESTAMP)
+ @Column(name="CREATE_TIME" )
+ protected Date createTime = getUTCDate();
+
+ /**
+ * Date value.
+ * <ul>
+ * </ul>
+ *
+ */
+ @Temporal(TemporalType.TIMESTAMP)
+ @Column(name="UPDATE_TIME" )
+ protected Date updateTime = getUTCDate();
+
+ /**
+ * Added by
+ * <ul>
+ * </ul>
+ *
+ */
+ @Column(name="ADDED_BY_ID" )
+ protected Long addedByUserId;
+
+
+ /**
+ * Last updated by
+ * <ul>
+ * </ul>
+ *
+ */
+ @Column(name="UPD_BY_ID" )
+ protected Long updatedByUserId;
+
+
+ /**
+ * Default constructor. This will set all the attributes to default value.
+ */
+ public XXDBBase ( ) {
+ }
+
+ public int getMyClassType( ) {
+ return CLASS_TYPE_NONE;
+ }
+
+ public String getMyDisplayValue() {
+ return null;
+ }
+
+ /**
+ * This method sets the value to the member attribute <b>id</b>.
+ * You cannot set null to the attribute.
+ * @param id Value to set member attribute <b>id</b>
+ */
+ public abstract void setId( Long id ) ;
+
+ /**
+ * Returns the value for the member attribute <b>id</b>
+ * @return Long - value of member attribute <b>id</b>.
+ */
+ public abstract Long getId( );
+
+ /**
+ * This method sets the value to the member attribute <b>createTime</b>.
+ * You cannot set null to the attribute.
+ * @param createTime Value to set member attribute <b>createTime</b>
+ */
+ public void setCreateTime( Date createTime ) {
+ this.createTime = createTime;
+ }
+
+ /**
+ * Returns the value for the member attribute <b>createTime</b>
+ * @return Date - value of member attribute <b>createTime</b>.
+ */
+ public Date getCreateTime( ) {
+ return this.createTime;
+ }
+
+ /**
+ * This method sets the value to the member attribute <b>updateTime</b>.
+ * You cannot set null to the attribute.
+ * @param updateTime Value to set member attribute <b>updateTime</b>
+ */
+ public void setUpdateTime( Date updateTime ) {
+ this.updateTime = updateTime;
+ }
+
+ /**
+ * Returns the value for the member attribute <b>updateTime</b>
+ * @return Date - value of member attribute <b>updateTime</b>.
+ */
+ public Date getUpdateTime( ) {
+ return this.updateTime;
+ }
+
+ /**
+ * This method sets the value to the member attribute <b>addedByUserId</b>.
+ * You cannot set null to the attribute.
+ * @param addedByUserId Value to set member attribute <b>addedByUserId</b>
+ */
+ public void setAddedByUserId( Long addedByUserId ) {
+ this.addedByUserId = addedByUserId;
+ }
+
+ /**
+ * Returns the value for the member attribute <b>addedByUserId</b>
+ * @return Long - value of member attribute <b>addedByUserId</b>.
+ */
+ public Long getAddedByUserId( ) {
+ return this.addedByUserId;
+ }
+
+
+ /**
+ * This method sets the value to the member attribute <b>updatedByUserId</b>.
+ * You cannot set null to the attribute.
+ * @param updatedByUserId Value to set member attribute <b>updatedByUserId</b>
+ */
+ public void setUpdatedByUserId( Long updatedByUserId ) {
+ this.updatedByUserId = updatedByUserId;
+ }
+
+ /**
+ * Returns the value for the member attribute <b>updatedByUserId</b>
+ * @return Long - value of member attribute <b>updatedByUserId</b>.
+ */
+ public Long getUpdatedByUserId( ) {
+ return this.updatedByUserId;
+ }
+
+
+ /**
+ * This return the bean content in string format
+ * @return formatedStr
+ */
+ @Override
+ public String toString( ) {
+ String str = "XXDBBase={";
+ //`str += "id={" + id + "} ";
+ str += "createTime={" + createTime + "} ";
+ str += "updateTime={" + updateTime + "} ";
+ str += "addedByUserId={" + addedByUserId + "} ";
+ str += "updatedByUserId={" + updatedByUserId + "} ";
+ str += "}";
+ return str;
+ }
+
+ /**
+ * Checks for all attributes except referenced db objects
+ * @return true if all attributes match
+ */
+ @Override
+ public boolean equals( Object obj) {
+ XXDBBase other = (XXDBBase) obj;
+ if ((this.createTime == null && other.createTime != null) || (this.createTime != null && !this.createTime.equals(other.createTime))) {
+ return false;
+ }
+ if ((this.updateTime == null && other.updateTime != null) || (this.updateTime != null && !this.updateTime.equals(other.updateTime))) {
+ return false;
+ }
+ if ((this.addedByUserId == null && other.addedByUserId != null) || (this.addedByUserId != null && !this.addedByUserId.equals(other.addedByUserId))) {
+ return false;
+ }
+ if ((this.updatedByUserId == null && other.updatedByUserId != null) || (this.updatedByUserId != null && !this.updatedByUserId.equals(other.updatedByUserId))) {
+ return false;
+ }
+ return true;
+ }
+ public static String getEnumName(String fieldName ) {
+ return null;
+ }
+
+ private static Date getUTCDate(){
+ try{
+ Calendar local=Calendar.getInstance();
+ int offset = local.getTimeZone().getOffset(local.getTimeInMillis());
+ GregorianCalendar utc = new GregorianCalendar(gmtTimeZone);
+ utc.setTimeInMillis(local.getTimeInMillis());
+ utc.add(Calendar.MILLISECOND, -offset);
+ return utc.getTime();
+ }catch(Exception ex){
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/ranger/entity/XXRangerKeyStore.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/ranger/entity/XXRangerKeyStore.java b/kms/src/main/java/org/apache/ranger/entity/XXRangerKeyStore.java
new file mode 100644
index 0000000..c9eb77e
--- /dev/null
+++ b/kms/src/main/java/org/apache/ranger/entity/XXRangerKeyStore.java
@@ -0,0 +1,121 @@
+package org.apache.ranger.entity;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Lob;
+import javax.persistence.SequenceGenerator;
+import javax.persistence.Table;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@Entity
+@Table(name="ranger_keystore")
+@XmlRootElement
+public class XXRangerKeyStore extends XXDBBase implements java.io.Serializable {
+ private static final long serialVersionUID = 1L;
+
+ @Id
+ @SequenceGenerator(name="kmskeys",sequenceName="kmskeys",allocationSize=1)
+ @GeneratedValue(strategy=GenerationType.AUTO,generator="kmskeys")
+ @Column(name="ID")
+ protected Long id;
+ @Override
+ public void setId(Long id) {
+ this.id=id;
+ }
+
+ @Override
+ public Long getId() {
+ return id;
+ }
+
+ @Column(name="kms_alias" , length=255 )
+ protected String alias;
+
+ public String getAlias() {
+ return alias;
+ }
+
+ public void setAlias(String alias) {
+ this.alias = alias;
+ }
+
+ @Column(name="kms_createdDate" , length=255 )
+ protected Long createdDate;
+
+ public Long getCreatedDate() {
+ return createdDate;
+ }
+
+ public void setCreatedDate(Long createdDate) {
+ this.createdDate = createdDate;
+ }
+
+ @Lob
+ @Column(name="kms_encoded")
+ protected String encoded;
+
+ public String getEncoded() {
+ return encoded;
+ }
+
+ public void setEncoded(String encoded) {
+ this.encoded = encoded;
+ }
+
+ @Column(name="kms_cipher" , length=255)
+ protected String cipher;
+
+ public String getCipher() {
+ return cipher;
+ }
+
+ public void setCipher(String cipher) {
+ this.cipher = cipher;
+ }
+
+ @Column(name="kms_bitLength")
+ protected int bitLength;
+
+ public int getBitLength() {
+ return bitLength;
+ }
+
+ public void setBitLength(int bitLength) {
+ this.bitLength = bitLength;
+ }
+
+ @Column(name="kms_description")
+ protected String description;
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ @Column(name="kms_version")
+ protected int version;
+
+ public int getVersion() {
+ return version;
+ }
+
+ public void setVersion(int version) {
+ this.version = version;
+ }
+
+ @Column(name="kms_attributes")
+ protected String attributes;
+ public String getAttributes() {
+ return attributes;
+ }
+
+ public void setAttributes(String attributes) {
+ this.attributes = attributes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/ranger/entity/XXRangerMasterKey.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/ranger/entity/XXRangerMasterKey.java b/kms/src/main/java/org/apache/ranger/entity/XXRangerMasterKey.java
new file mode 100644
index 0000000..6175286
--- /dev/null
+++ b/kms/src/main/java/org/apache/ranger/entity/XXRangerMasterKey.java
@@ -0,0 +1,67 @@
+package org.apache.ranger.entity;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Lob;
+import javax.persistence.SequenceGenerator;
+import javax.persistence.Table;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@Entity
+@Table(name="ranger_masterkey")
+@XmlRootElement
+public class XXRangerMasterKey extends XXDBBase implements java.io.Serializable {
+ private static final long serialVersionUID = 1L;
+
+ @Id
+ @SequenceGenerator(name="rangermasterkey",sequenceName="rangermasterkey",allocationSize=1)
+ @GeneratedValue(strategy=GenerationType.AUTO,generator="rangermasterkey")
+ @Column(name="ID")
+ protected Long id;
+ @Override
+ public void setId(Long id) {
+ this.id=id;
+ }
+
+ @Override
+ public Long getId() {
+ return id;
+ }
+
+ @Column(name="cipher")
+ protected String cipher;
+
+ public String getCipher() {
+ return cipher;
+ }
+
+ public void setCipher(String cipher) {
+ this.cipher = cipher;
+ }
+
+ @Column(name="bitlength")
+ protected int bitLength;
+
+ public int getBitLength() {
+ return bitLength;
+ }
+
+ public void setBitLength(int bitLength) {
+ this.bitLength = bitLength;
+ }
+
+ @Lob
+ @Column(name="masterkey")
+ protected String masterKey;
+
+ public String getMasterKey() {
+ return masterKey;
+ }
+
+ public void setMasterKey(String masterKey) {
+ this.masterKey = masterKey;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/ranger/kms/biz/RangerKMSStartUp.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/ranger/kms/biz/RangerKMSStartUp.java b/kms/src/main/java/org/apache/ranger/kms/biz/RangerKMSStartUp.java
new file mode 100644
index 0000000..af9bfa1
--- /dev/null
+++ b/kms/src/main/java/org/apache/ranger/kms/biz/RangerKMSStartUp.java
@@ -0,0 +1,35 @@
+package org.apache.ranger.kms.biz;
+
+import javax.annotation.PostConstruct;
+import javax.servlet.http.HttpServlet;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.RangerKeyStoreProvider;
+import org.apache.hadoop.crypto.key.RangerMasterKey;
+import org.springframework.stereotype.Component;
+
+@SuppressWarnings("serial")
+@Component
+public class RangerKMSStartUp extends HttpServlet
+{
+ public static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password";
+ private static Logger LOG = LoggerFactory.getLogger(RangerKMSStartUp.class);
+
+ @PostConstruct
+ public void initRangerMasterKey() {
+ LOG.info("Ranger KMSStartUp");
+ RangerMasterKey rangerMasterKey = new RangerMasterKey();
+ try {
+ Configuration conf = RangerKeyStoreProvider.getDBKSConf();
+ String password = conf.get(ENCRYPTION_KEY);
+ boolean check = rangerMasterKey.generateMasterKey(password);
+ if(check){
+ LOG.info("MasterKey Generated..");
+ }
+ } catch (Throwable e) {
+ e.printStackTrace();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/ranger/kms/dao/BaseDao.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/ranger/kms/dao/BaseDao.java b/kms/src/main/java/org/apache/ranger/kms/dao/BaseDao.java
new file mode 100644
index 0000000..f835bcc
--- /dev/null
+++ b/kms/src/main/java/org/apache/ranger/kms/dao/BaseDao.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ package org.apache.ranger.kms.dao;
+
+
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.persistence.EntityManager;
+import javax.persistence.EntityTransaction;
+import javax.persistence.NoResultException;
+import javax.persistence.TypedQuery;
+
+import org.apache.log4j.Logger;
+
+public abstract class BaseDao<T> {
+ static final Logger logger = Logger.getLogger(BaseDao.class);
+
+ protected DaoManager daoManager;
+
+ protected Class<T> tClass;
+
+ public BaseDao(DaoManagerBase daoManager) {
+ this.init(daoManager);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void init(DaoManagerBase daoManager) {
+ this.daoManager = (DaoManager) daoManager;
+
+ ParameterizedType genericSuperclass = (ParameterizedType) getClass()
+ .getGenericSuperclass();
+
+ Type type = genericSuperclass.getActualTypeArguments()[0];
+
+ if (type instanceof ParameterizedType) {
+ this.tClass = (Class<T>) ((ParameterizedType) type).getRawType();
+ } else {
+ this.tClass = (Class<T>) type;
+ }
+ }
+
+ public EntityManager getEntityManager() {
+ return daoManager.getEntityManager();
+ }
+
+ public boolean beginTransaction() {
+ boolean ret = false;
+
+ EntityManager em = getEntityManager();
+
+ if(em != null) {
+ EntityTransaction et = em.getTransaction();
+
+ // check the transaction is not already active
+ if(et != null && !et.isActive()) {
+ et.begin();
+ ret = true;
+ }
+ }
+
+ return ret;
+ }
+
+ public void commitTransaction() {
+ EntityManager em = getEntityManager();
+
+ if(em != null) {
+ em.flush();
+
+ EntityTransaction et = em.getTransaction();
+
+ if(et != null) {
+ et.commit();
+ }
+ }
+ }
+
+ public void rollbackTransaction() {
+ EntityManager em = getEntityManager();
+
+ if(em != null) {
+ EntityTransaction et = em.getTransaction();
+
+ if(et != null) {
+ et.rollback();
+ }
+ }
+ }
+
+ public T create(T obj) {
+ T ret = null;
+
+ boolean trxBegan = beginTransaction();
+
+ getEntityManager().persist(obj);
+
+ if(trxBegan) {
+ commitTransaction();
+ }
+
+ ret = obj;
+
+ return ret;
+ }
+
+ public T update(T obj) {
+ boolean trxBegan = beginTransaction();
+
+ getEntityManager().merge(obj);
+
+ if(trxBegan) {
+ commitTransaction();
+ }
+
+ return obj;
+ }
+
+ public boolean remove(Long id) {
+ return remove(getById(id));
+ }
+
+ public boolean remove(T obj) {
+ if (obj == null) {
+ return true;
+ }
+
+ boolean ret = false;
+
+ boolean trxBegan = beginTransaction();
+
+ getEntityManager().remove(obj);
+
+ if(trxBegan) {
+ commitTransaction();
+ }
+
+ ret = true;
+
+ return ret;
+ }
+
+ public T getById(Long id) {
+ if (id == null) {
+ return null;
+ }
+ T ret = null;
+ try {
+ ret = getEntityManager().find(tClass, id);
+ } catch (NoResultException e) {
+ return null;
+ }
+ return ret;
+ }
+
+ public List<T> getAll() {
+ List<T> ret = null;
+
+ TypedQuery<T> qry = getEntityManager().createQuery(
+ "SELECT t FROM " + tClass.getSimpleName() + " t", tClass);
+
+ ret = qry.getResultList();
+
+ return ret;
+ }
+
+ public Long getAllCount() {
+ Long ret = null;
+
+ TypedQuery<Long> qry = getEntityManager().createQuery(
+ "SELECT count(t) FROM " + tClass.getSimpleName() + " t",
+ Long.class);
+
+ ret = qry.getSingleResult();
+
+ return ret;
+ }
+
+ public T getUniqueResult(TypedQuery<T> qry) {
+ T ret = null;
+
+ try {
+ ret = qry.getSingleResult();
+ } catch (NoResultException e) {
+ // ignore
+ }
+ return ret;
+ }
+
+ public List<T> executeQuery(TypedQuery<T> qry) {
+ List<T> ret = null;
+
+ ret = qry.getResultList();
+
+ return ret;
+ }
+
+ public List<T> findByNamedQuery(String namedQuery, String paramName,
+ Object refId) {
+ List<T> ret = new ArrayList<T>();
+
+ if (namedQuery == null) {
+ return ret;
+ }
+ try {
+ TypedQuery<T> qry = getEntityManager().createNamedQuery(namedQuery, tClass);
+ qry.setParameter(paramName, refId);
+ ret = qry.getResultList();
+ } catch (NoResultException e) {
+ // ignore
+ }
+ return ret;
+ }
+
+ public T findByAlias(String namedQuery, String alias) {
+ try {
+ return getEntityManager()
+ .createNamedQuery(namedQuery, tClass)
+ .setParameter("alias", alias)
+ .getSingleResult();
+ } catch (NoResultException e) {
+ }
+ return null;
+ }
+
+ public int deleteByAlias(String namedQuery, String alias) {
+ boolean trxBegan = beginTransaction();
+ try {
+ int i = getEntityManager()
+ .createNamedQuery(namedQuery, tClass)
+ .setParameter("alias", alias).executeUpdate();
+ if(trxBegan) {
+ commitTransaction();
+ }
+ return i;
+ } catch (NoResultException e) {
+ e.printStackTrace();
+ rollbackTransaction();
+ }
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/ranger/kms/dao/DaoManager.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/ranger/kms/dao/DaoManager.java b/kms/src/main/java/org/apache/ranger/kms/dao/DaoManager.java
new file mode 100644
index 0000000..be67204
--- /dev/null
+++ b/kms/src/main/java/org/apache/ranger/kms/dao/DaoManager.java
@@ -0,0 +1,35 @@
+package org.apache.ranger.kms.dao;
+
+import javax.persistence.EntityManager;
+import javax.persistence.EntityManagerFactory;
+import javax.persistence.PersistenceContext;
+
+public class DaoManager extends DaoManagerBase {
+
+ @PersistenceContext
+ private EntityManagerFactory emf;
+
+ static ThreadLocal<EntityManager> sEntityManager;
+
+ public void setEntityManagerFactory(EntityManagerFactory emf) {
+ this.emf = emf;
+ sEntityManager = new ThreadLocal<EntityManager>();
+ }
+
+ @Override
+ public EntityManager getEntityManager() {
+ EntityManager em = null;
+
+ if(sEntityManager != null) {
+ em = sEntityManager.get();
+
+ if(em == null && this.emf != null) {
+ em = this.emf.createEntityManager();
+
+ sEntityManager.set(em);
+ }
+ }
+
+ return em;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/ranger/kms/dao/DaoManagerBase.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/ranger/kms/dao/DaoManagerBase.java b/kms/src/main/java/org/apache/ranger/kms/dao/DaoManagerBase.java
new file mode 100644
index 0000000..5cd84b0
--- /dev/null
+++ b/kms/src/main/java/org/apache/ranger/kms/dao/DaoManagerBase.java
@@ -0,0 +1,31 @@
+package org.apache.ranger.kms.dao;
+
+import javax.persistence.EntityManager;
+import org.apache.log4j.Logger;
+
+public abstract class DaoManagerBase {
+ final static Logger logger = Logger.getLogger(DaoManagerBase.class);
+
+ abstract public EntityManager getEntityManager();
+
+ private RangerMasterKeyDao rangerMasterKeyDao = null;
+ private RangerKMSDao rangerKmsDao = null;
+
+ public DaoManagerBase() {
+ }
+
+ public RangerMasterKeyDao getRangerMasterKeyDao() {
+ if(rangerMasterKeyDao == null) {
+ rangerMasterKeyDao = new RangerMasterKeyDao(this);
+ }
+
+ return rangerMasterKeyDao;
+ }
+
+ public RangerKMSDao getRangerKMSDao(){
+ if(rangerKmsDao == null){
+ rangerKmsDao = new RangerKMSDao(this);
+ }
+ return rangerKmsDao;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/ranger/kms/dao/RangerKMSDao.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/ranger/kms/dao/RangerKMSDao.java b/kms/src/main/java/org/apache/ranger/kms/dao/RangerKMSDao.java
new file mode 100644
index 0000000..d5c94fe
--- /dev/null
+++ b/kms/src/main/java/org/apache/ranger/kms/dao/RangerKMSDao.java
@@ -0,0 +1,18 @@
+package org.apache.ranger.kms.dao;
+
+import org.apache.ranger.entity.XXRangerKeyStore;
+
+public class RangerKMSDao extends BaseDao<XXRangerKeyStore> {
+
+ public RangerKMSDao(DaoManagerBase daoManager) {
+ super(daoManager);
+ }
+
+ public XXRangerKeyStore findByAlias(String alias){
+ return super.findByAlias("XXRangerKeyStore.findByAlias", alias);
+ }
+
+ public int deleteByAlias(String alias){
+ return super.deleteByAlias("XXRangerKeyStore.deleteByAlias", alias);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/ranger/kms/dao/RangerMasterKeyDao.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/ranger/kms/dao/RangerMasterKeyDao.java b/kms/src/main/java/org/apache/ranger/kms/dao/RangerMasterKeyDao.java
new file mode 100644
index 0000000..4170433
--- /dev/null
+++ b/kms/src/main/java/org/apache/ranger/kms/dao/RangerMasterKeyDao.java
@@ -0,0 +1,10 @@
+package org.apache.ranger.kms.dao;
+
+import org.apache.ranger.entity.XXRangerMasterKey;
+
+public class RangerMasterKeyDao extends BaseDao<XXRangerMasterKey> {
+
+ public RangerMasterKeyDao(DaoManagerBase daoManager) {
+ super(daoManager);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/resources/META-INF/kms_jpa_named_queries.xml
----------------------------------------------------------------------
diff --git a/kms/src/main/resources/META-INF/kms_jpa_named_queries.xml b/kms/src/main/resources/META-INF/kms_jpa_named_queries.xml
new file mode 100644
index 0000000..8fd3128
--- /dev/null
+++ b/kms/src/main/resources/META-INF/kms_jpa_named_queries.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<entity-mappings version="1.0"
+ xmlns="http://java.sun.com/xml/ns/persistence/orm" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://java.sun.com/xml/ns/persistence/orm http://java.sun.com/xml/ns/persistence/orm_1_0.xsd ">
+ <named-query name="XXRangerKeyStore.findByAlias">
+ <query>SELECT Obj FROM XXRangerKeyStore obj
+ WHERE obj.alias=:alias
+ </query>
+ </named-query>
+
+ <named-query name="XXRangerKeyStore.deleteByAlias">
+ <query>DELETE FROM XXRangerKeyStore obj
+ WHERE obj.alias=:alias
+ </query>
+ </named-query>
+</entity-mappings>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/resources/META-INF/persistence.xml
----------------------------------------------------------------------
diff --git a/kms/src/main/resources/META-INF/persistence.xml b/kms/src/main/resources/META-INF/persistence.xml
new file mode 100644
index 0000000..31c0bc4
--- /dev/null
+++ b/kms/src/main/resources/META-INF/persistence.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<persistence version="2.0" xmlns="http://java.sun.com/xml/ns/persistence" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://java.sun.com/xml/ns/persistence http://java.sun.com/xml/ns/persistence/persistence_2_0.xsd">
+ <persistence-unit name="persistence_ranger_server">
+ <mapping-file>META-INF/kms_jpa_named_queries.xml</mapping-file>
+ <class>org.apache.ranger.entity.XXRangerMasterKey</class>
+ <class>org.apache.ranger.entity.XXRangerKeyStore</class>
+
+ <properties>
+ <property name="eclipselink.logging.level" value="SEVERE"/>
+ </properties>
+ </persistence-unit>
+</persistence>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory
----------------------------------------------------------------------
diff --git a/kms/src/main/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory b/kms/src/main/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory
new file mode 100644
index 0000000..bb9f0bb
--- /dev/null
+++ b/kms/src/main/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.apache.hadoop.crypto.key.RangerKeyStoreProvider$Factory
+org.apache.hadoop.crypto.key.JavaKeyStoreProvider$Factory
+org.apache.hadoop.crypto.key.UserProvider$Factory
+org.apache.hadoop.crypto.key.kms.KMSClientProvider$Factory
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/resources/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/kms/src/main/resources/WEB-INF/web.xml b/kms/src/main/resources/WEB-INF/web.xml
new file mode 100644
index 0000000..71c580c
--- /dev/null
+++ b/kms/src/main/resources/WEB-INF/web.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<web-app version="2.4" xmlns="http://java.sun.com/xml/ns/j2ee">
+
+ <display-name>ranger-kms</display-name>
+
+ <listener>
+ <listener-class>org.apache.hadoop.crypto.key.kms.server.KMSWebApp</listener-class>
+ </listener>
+
+ <servlet>
+ <servlet-name>webservices-driver</servlet-name>
+ <servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class>
+ <init-param>
+ <param-name>com.sun.jersey.config.property.packages</param-name>
+ <param-value>org.apache.hadoop.crypto.key.kms.server</param-value>
+ </init-param>
+ <load-on-startup>1</load-on-startup>
+ </servlet>
+
+<!-- <servlet>
+ <servlet-name>RangerKMSStartUp</servlet-name>
+ <servlet-class>org.apache.ranger.kms.biz.RangerKMSStartUp</servlet-class>
+ <load-on-startup>2</load-on-startup>
+ </servlet> -->
+
+ <servlet>
+ <servlet-name>jmx-servlet</servlet-name>
+ <servlet-class>org.apache.hadoop.crypto.key.kms.server.KMSJMXServlet</servlet-class>
+ </servlet>
+
+ <servlet-mapping>
+ <servlet-name>webservices-driver</servlet-name>
+ <url-pattern>/*</url-pattern>
+ </servlet-mapping>
+
+ <servlet-mapping>
+ <servlet-name>jmx-servlet</servlet-name>
+ <url-pattern>/jmx</url-pattern>
+ </servlet-mapping>
+
+ <filter>
+ <filter-name>authFilter</filter-name>
+ <filter-class>org.apache.hadoop.crypto.key.kms.server.KMSAuthenticationFilter</filter-class>
+ </filter>
+
+ <filter>
+ <filter-name>MDCFilter</filter-name>
+ <filter-class>org.apache.hadoop.crypto.key.kms.server.KMSMDCFilter</filter-class>
+ </filter>
+
+ <filter-mapping>
+ <filter-name>authFilter</filter-name>
+ <url-pattern>/*</url-pattern>
+ </filter-mapping>
+
+ <filter-mapping>
+ <filter-name>MDCFilter</filter-name>
+ <url-pattern>/*</url-pattern>
+ </filter-mapping>
+
+</web-app>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/resources/log4j-kmsaudit.properties
----------------------------------------------------------------------
diff --git a/kms/src/main/resources/log4j-kmsaudit.properties b/kms/src/main/resources/log4j-kmsaudit.properties
new file mode 100644
index 0000000..cca6941
--- /dev/null
+++ b/kms/src/main/resources/log4j-kmsaudit.properties
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# LOG Appender
+log4j.appender.kms-audit=org.apache.log4j.ConsoleAppender
+log4j.appender.kms-audit.Target=System.err
+log4j.appender.kms-audit.layout=org.apache.log4j.PatternLayout
+log4j.appender.kms-audit.layout.ConversionPattern=%m
+
+log4j.rootLogger=INFO, kms-audit
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/kms/src/main/resources/log4j.properties b/kms/src/main/resources/log4j.properties
new file mode 100644
index 0000000..5cd037a
--- /dev/null
+++ b/kms/src/main/resources/log4j.properties
@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# STDOUT Appender
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n
+
+log4j.rootLogger=WARN, stdout
+log4j.logger.org.apache.hadoop.conf=ERROR
+log4j.logger.org.apache.hadoop.crytpo.key.kms.server=ALL
+log4j.logger.com.sun.jersey.server.wadl.generators.WadlGeneratorJAXBGrammarGenerator=OFF
+log4j.logger.org.apache.hadoop.security=OFF
+log4j.logger.org.apache.directory.server.core=OFF
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/resources/mini-kms-acls-default.xml
----------------------------------------------------------------------
diff --git a/kms/src/main/resources/mini-kms-acls-default.xml b/kms/src/main/resources/mini-kms-acls-default.xml
new file mode 100644
index 0000000..24a46b8
--- /dev/null
+++ b/kms/src/main/resources/mini-kms-acls-default.xml
@@ -0,0 +1,135 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+ <!-- This file is hot-reloaded when it changes -->
+
+ <!-- KMS ACLs -->
+
+ <property>
+ <name>hadoop.kms.acl.CREATE</name>
+ <value>*</value>
+ <description>
+ ACL for create-key operations.
+ If the user does is not in the GET ACL, the key material is not returned
+ as part of the response.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.DELETE</name>
+ <value>*</value>
+ <description>
+ ACL for delete-key operations.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.ROLLOVER</name>
+ <value>*</value>
+ <description>
+ ACL for rollover-key operations.
+ If the user does is not in the GET ACL, the key material is not returned
+ as part of the response.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.GET</name>
+ <value>*</value>
+ <description>
+ ACL for get-key-version and get-current-key operations.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.GET_KEYS</name>
+ <value>*</value>
+ <description>
+ ACL for get-keys operation.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.GET_METADATA</name>
+ <value>*</value>
+ <description>
+ ACL for get-key-metadata an get-keys-metadata operations.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.SET_KEY_MATERIAL</name>
+ <value>*</value>
+ <description>
+ Complimentary ACL for CREATE and ROLLOVER operation to allow the client
+ to provide the key material when creating or rolling a key.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.GENERATE_EEK</name>
+ <value>*</value>
+ <description>
+ ACL for generateEncryptedKey CryptoExtension operations
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.DECRYPT_EEK</name>
+ <value>*</value>
+ <description>
+ ACL for decrypt EncryptedKey CryptoExtension operations
+ </description>
+ </property>
+
+ <property>
+ <name>default.key.acl.MANAGEMENT</name>
+ <value>*</value>
+ <description>
+ default ACL for MANAGEMENT operations for all key acls that are not
+ explicitly defined.
+ </description>
+ </property>
+
+ <property>
+ <name>default.key.acl.GENERATE_EEK</name>
+ <value>*</value>
+ <description>
+ default ACL for GENERATE_EEK operations for all key acls that are not
+ explicitly defined.
+ </description>
+ </property>
+
+ <property>
+ <name>default.key.acl.DECRYPT_EEK</name>
+ <value>*</value>
+ <description>
+ default ACL for DECRYPT_EEK operations for all key acls that are not
+ explicitly defined.
+ </description>
+ </property>
+
+ <property>
+ <name>default.key.acl.READ</name>
+ <value>*</value>
+ <description>
+ default ACL for READ operations for all key acls that are not
+ explicitly defined.
+ </description>
+ </property>
+
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/kms/src/main/webapp/WEB-INF/web.xml b/kms/src/main/webapp/WEB-INF/web.xml
new file mode 100644
index 0000000..7d87e90
--- /dev/null
+++ b/kms/src/main/webapp/WEB-INF/web.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<web-app version="2.4" xmlns="http://java.sun.com/xml/ns/j2ee">
+
+ <display-name>ranger-kms</display-name>
+
+ <listener>
+ <listener-class>org.apache.hadoop.crypto.key.kms.server.KMSWebApp</listener-class>
+ </listener>
+
+ <servlet>
+ <servlet-name>webservices-driver</servlet-name>
+ <servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class>
+ <init-param>
+ <param-name>com.sun.jersey.config.property.packages</param-name>
+ <param-value>org.apache.hadoop.crypto.key.kms.server</param-value>
+ </init-param>
+ <load-on-startup>1</load-on-startup>
+ </servlet>
+
+ <!-- <servlet>
+ <servlet-name>RangerKMSStartUp</servlet-name>
+ <servlet-class>org.apache.ranger.kms.biz.RangerKMSStartUp</servlet-class>
+ <load-on-startup>2</load-on-startup>
+ </servlet> -->
+
+ <servlet>
+ <servlet-name>jmx-servlet</servlet-name>
+ <servlet-class>org.apache.hadoop.crypto.key.kms.server.KMSJMXServlet</servlet-class>
+ </servlet>
+
+ <servlet-mapping>
+ <servlet-name>webservices-driver</servlet-name>
+ <url-pattern>/*</url-pattern>
+ </servlet-mapping>
+
+ <servlet-mapping>
+ <servlet-name>jmx-servlet</servlet-name>
+ <url-pattern>/jmx</url-pattern>
+ </servlet-mapping>
+
+ <filter>
+ <filter-name>authFilter</filter-name>
+ <filter-class>org.apache.hadoop.crypto.key.kms.server.KMSAuthenticationFilter</filter-class>
+ </filter>
+
+ <filter>
+ <filter-name>MDCFilter</filter-name>
+ <filter-class>org.apache.hadoop.crypto.key.kms.server.KMSMDCFilter</filter-class>
+ </filter>
+
+ <filter-mapping>
+ <filter-name>authFilter</filter-name>
+ <url-pattern>/*</url-pattern>
+ </filter-mapping>
+
+ <filter-mapping>
+ <filter-name>MDCFilter</filter-name>
+ <url-pattern>/*</url-pattern>
+ </filter-mapping>
+
+</web-app>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
----------------------------------------------------------------------
diff --git a/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java b/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
new file mode 100644
index 0000000..4f802cc
--- /dev/null
+++ b/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
@@ -0,0 +1,238 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.ssl.SslSocketConnectorSecure;
+import org.mortbay.jetty.Connector;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.security.SslSocketConnector;
+import org.mortbay.jetty.webapp.WebAppContext;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Writer;
+import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.ServerSocket;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.UUID;
+
+public class MiniKMS {
+
+ private static Server createJettyServer(String keyStore, String password, int inPort) {
+ try {
+ boolean ssl = keyStore != null;
+ InetAddress localhost = InetAddress.getByName("localhost");
+ String host = "localhost";
+ ServerSocket ss = new ServerSocket((inPort < 0) ? 0 : inPort, 50, localhost);
+ int port = ss.getLocalPort();
+ ss.close();
+ Server server = new Server(0);
+ if (!ssl) {
+ server.getConnectors()[0].setHost(host);
+ server.getConnectors()[0].setPort(port);
+ } else {
+ SslSocketConnector c = new SslSocketConnectorSecure();
+ c.setHost(host);
+ c.setPort(port);
+ c.setNeedClientAuth(false);
+ c.setKeystore(keyStore);
+ c.setKeystoreType("jks");
+ c.setKeyPassword(password);
+ server.setConnectors(new Connector[]{c});
+ }
+ return server;
+ } catch (Exception ex) {
+ throw new RuntimeException("Could not start embedded servlet container, "
+ + ex.getMessage(), ex);
+ }
+ }
+
+ private static URL getJettyURL(Server server) {
+ boolean ssl = server.getConnectors()[0].getClass()
+ == SslSocketConnectorSecure.class;
+ try {
+ String scheme = (ssl) ? "https" : "http";
+ return new URL(scheme + "://" +
+ server.getConnectors()[0].getHost() + ":" +
+ server.getConnectors()[0].getPort());
+ } catch (MalformedURLException ex) {
+ throw new RuntimeException("It should never happen, " + ex.getMessage(),
+ ex);
+ }
+ }
+
+ public static class Builder {
+ private File kmsConfDir;
+ private String log4jConfFile;
+ private File keyStoreFile;
+ private String keyStorePassword;
+ private int inPort = -1;
+
+ public Builder() {
+ kmsConfDir = new File("target/test-classes").getAbsoluteFile();
+ log4jConfFile = "kms-log4j.properties";
+ }
+
+ public Builder setKmsConfDir(File confDir) {
+ Preconditions.checkNotNull(confDir, "KMS conf dir is NULL");
+ Preconditions.checkArgument(confDir.exists(),
+ "KMS conf dir does not exist");
+ kmsConfDir = confDir;
+ return this;
+ }
+
+ public Builder setLog4jConfFile(String log4jConfFile) {
+ Preconditions.checkNotNull(log4jConfFile, "log4jconf file is NULL");
+ this.log4jConfFile = log4jConfFile;
+ return this;
+ }
+
+ public Builder setPort(int port) {
+ Preconditions.checkArgument(port > 0, "input port must be greater than 0");
+ this.inPort = port;
+ return this;
+ }
+
+ public Builder setSslConf(File keyStoreFile, String keyStorePassword) {
+ Preconditions.checkNotNull(keyStoreFile, "keystore file is NULL");
+ Preconditions.checkNotNull(keyStorePassword, "keystore password is NULL");
+ Preconditions.checkArgument(keyStoreFile.exists(),
+ "keystore file does not exist");
+ this.keyStoreFile = keyStoreFile;
+ this.keyStorePassword = keyStorePassword;
+ return this;
+ }
+
+ public MiniKMS build() {
+ Preconditions.checkArgument(kmsConfDir.exists(),
+ "KMS conf dir does not exist");
+ return new MiniKMS(kmsConfDir.getAbsolutePath(), log4jConfFile,
+ (keyStoreFile != null) ? keyStoreFile.getAbsolutePath() : null,
+ keyStorePassword, inPort);
+ }
+ }
+
+ private String kmsConfDir;
+ private String log4jConfFile;
+ private String keyStore;
+ private String keyStorePassword;
+ private Server jetty;
+ private int inPort;
+ private URL kmsURL;
+
+ public MiniKMS(String kmsConfDir, String log4ConfFile, String keyStore,
+ String password, int inPort) {
+ this.kmsConfDir = kmsConfDir;
+ this.log4jConfFile = log4ConfFile;
+ this.keyStore = keyStore;
+ this.keyStorePassword = password;
+ this.inPort = inPort;
+ }
+
+ public void start() throws Exception {
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+ System.setProperty(KMSConfiguration.KMS_CONFIG_DIR, kmsConfDir);
+ File aclsFile = new File(kmsConfDir, "kms-acls.xml");
+ if (!aclsFile.exists()) {
+ InputStream is = cl.getResourceAsStream("mini-kms-acls-default.xml");
+ OutputStream os = new FileOutputStream(aclsFile);
+ IOUtils.copy(is, os);
+ is.close();
+ os.close();
+ }
+ File coreFile = new File(kmsConfDir, "core-site.xml");
+ if (!coreFile.exists()) {
+ Configuration core = new Configuration();
+ Writer writer = new FileWriter(coreFile);
+ core.writeXml(writer);
+ writer.close();
+ }
+ File kmsFile = new File(kmsConfDir, "kms-site.xml");
+ if (!kmsFile.exists()) {
+ Configuration kms = new Configuration(false);
+ kms.set(KMSConfiguration.KEY_PROVIDER_URI,
+ "jceks://file@" + new Path(kmsConfDir, "kms.keystore").toUri());
+ kms.set("hadoop.kms.authentication.type", "simple");
+ Writer writer = new FileWriter(kmsFile);
+ kms.writeXml(writer);
+ writer.close();
+ }
+ System.setProperty("log4j.configuration", log4jConfFile);
+ jetty = createJettyServer(keyStore, keyStorePassword, inPort);
+
+ // we need to do a special handling for MiniKMS to work when in a dir and
+ // when in a JAR in the classpath thanks to Jetty way of handling of webapps
+ // when they are in the a DIR, WAR or JAR.
+ URL webXmlUrl = cl.getResource("kms-webapp/WEB-INF/web.xml");
+ if (webXmlUrl == null) {
+ throw new RuntimeException(
+ "Could not find kms-webapp/ dir in test classpath");
+ }
+ boolean webXmlInJar = webXmlUrl.getPath().contains(".jar!/");
+ String webappPath;
+ if (webXmlInJar) {
+ File webInf = new File("target/" + UUID.randomUUID().toString() +
+ "/kms-webapp/WEB-INF");
+ webInf.mkdirs();
+ new File(webInf, "web.xml").delete();
+ InputStream is = cl.getResourceAsStream("kms-webapp/WEB-INF/web.xml");
+ OutputStream os = new FileOutputStream(new File(webInf, "web.xml"));
+ IOUtils.copy(is, os);
+ is.close();
+ os.close();
+ webappPath = webInf.getParentFile().getAbsolutePath();
+ } else {
+ webappPath = cl.getResource("kms-webapp").getPath();
+ }
+ WebAppContext context = new WebAppContext(webappPath, "/kms");
+ if (webXmlInJar) {
+ context.setClassLoader(cl);
+ }
+ jetty.addHandler(context);
+ jetty.start();
+ kmsURL = new URL(getJettyURL(jetty), "kms");
+ }
+
+ public URL getKMSUrl() {
+ return kmsURL;
+ }
+
+ public void stop() {
+ if (jetty != null && jetty.isRunning()) {
+ try {
+ jetty.stop();
+ jetty = null;
+ } catch (Exception ex) {
+ throw new RuntimeException("Could not stop MiniKMS embedded Jetty, " +
+ ex.getMessage(), ex);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSACLs.java
----------------------------------------------------------------------
diff --git a/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSACLs.java b/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSACLs.java
new file mode 100644
index 0000000..12945d7
--- /dev/null
+++ b/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSACLs.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.kms.server.KMSACLsType.Type;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestKMSACLs {
+
+ @Test
+ public void testDefaults() {
+ KMSACLs acls = new KMSACLs(new Configuration(false));
+ for (Type type : Type.values()) {
+ Assert.assertTrue(acls.hasAccess(type,
+ UserGroupInformation.createRemoteUser("foo")));
+ }
+ }
+
+ @Test
+ public void testCustom() {
+ Configuration conf = new Configuration(false);
+ for (Type type : Type.values()) {
+ conf.set(type.getAclConfigKey(), type.toString() + " ");
+ }
+ KMSACLs acls = new KMSACLs(conf);
+ for (Type type : Type.values()) {
+ Assert.assertTrue(acls.hasAccess(type,
+ UserGroupInformation.createRemoteUser(type.toString())));
+ Assert.assertFalse(acls.hasAccess(type,
+ UserGroupInformation.createRemoteUser("foo")));
+ }
+ }
+
+}
[3/4] incubator-ranger git commit: (RANGER-247)Development of Ranger
Key Storage Provider
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java
new file mode 100644
index 0000000..274b5f8
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java
@@ -0,0 +1,341 @@
+package org.apache.hadoop.crypto.key;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URL;
+import java.security.Key;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.crypto.spec.SecretKeySpec;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderFactory;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.apache.hadoop.fs.Path;
+import org.apache.ranger.kms.dao.DaoManager;
+
+public class RangerKeyStoreProvider extends KeyProvider{
+
+ public static final String SCHEME_NAME = "dbks";
+ public static final String KMS_CONFIG_DIR = "kms.config.dir";
+ public static final String DBKS_SITE_XML = "dbks-site.xml";
+ public static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password";
+ private static final String KEY_METADATA = "KeyMetadata";
+ private final RangerKeyStore dbStore;
+ private char[] masterKey;
+ private boolean changed = false;
+ private final Map<String, Metadata> cache = new HashMap<String, Metadata>();
+ private DaoManager daoManager;
+
+ public RangerKeyStoreProvider(Configuration conf) throws Throwable {
+ super(conf);
+ conf = getDBKSConf();
+ RangerKMSDB rangerKMSDB = new RangerKMSDB(conf);
+ daoManager = rangerKMSDB.getDaoManager();
+ RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager);
+ dbStore = new RangerKeyStore(daoManager);
+ String password = conf.get(ENCRYPTION_KEY);
+ rangerMasterKey.generateMasterKey(password);
+ //code to retrieve rangerMasterKey password
+ masterKey = rangerMasterKey.getMasterKey(password).toCharArray();
+ if(masterKey == null){
+ // Master Key does not exists
+ throw new IOException("Ranger MasterKey does not exists");
+ }
+ try {
+ loadKeys(masterKey);
+ } catch (NoSuchAlgorithmException e) {
+ e.printStackTrace();
+ throw new IOException("Can't load Keys");
+ }catch(CertificateException e){
+ e.printStackTrace();
+ throw new IOException("Can't load Keys");
+ }
+ }
+
+ public static Configuration getDBKSConf() {
+ return getConfiguration(true, DBKS_SITE_XML);
+ }
+
+ static Configuration getConfiguration(boolean loadHadoopDefaults,
+ String ... resources) {
+ Configuration conf = new Configuration(loadHadoopDefaults);
+ String confDir = System.getProperty(KMS_CONFIG_DIR);
+ if (confDir != null) {
+ try {
+ Path confPath = new Path(confDir);
+ if (!confPath.isUriPathAbsolute()) {
+ throw new RuntimeException("System property '" + KMS_CONFIG_DIR +
+ "' must be an absolute path: " + confDir);
+ }
+ for (String resource : resources) {
+ conf.addResource(new URL("file://" + new Path(confDir, resource).toUri()));
+ }
+ } catch (MalformedURLException ex) {
+ ex.printStackTrace();
+ throw new RuntimeException(ex);
+ }
+ } else {
+ for (String resource : resources) {
+ conf.addResource(resource);
+ }
+ }
+ return conf;
+ }
+
+ private void loadKeys(char[] masterKey) throws NoSuchAlgorithmException, CertificateException, IOException {
+ dbStore.engineLoad(null, masterKey);
+ }
+
+ @Override
+ public KeyVersion createKey(String name, byte[] material, Options options)
+ throws IOException {
+ if (dbStore.engineContainsAlias(name) || cache.containsKey(name)) {
+ throw new IOException("Key " + name + " already exists");
+ }
+ Metadata meta = new Metadata(options.getCipher(), options.getBitLength(),
+ options.getDescription(), options.getAttributes(), new Date(), 1);
+ if (options.getBitLength() != 8 * material.length) {
+ throw new IOException("Wrong key length. Required " +
+ options.getBitLength() + ", but got " + (8 * material.length));
+ }
+ cache.put(name, meta);
+ String versionName = buildVersionName(name, 0);
+ return innerSetKeyVersion(name, versionName, material, meta.getCipher(), meta.getBitLength(), meta.getDescription(), meta.getVersions(), meta.getAttributes());
+ }
+
+ KeyVersion innerSetKeyVersion(String name, String versionName, byte[] material, String cipher, int bitLength, String description, int version, Map<String, String> attributes) throws IOException {
+ try {
+ ObjectMapper om = new ObjectMapper();
+ String attribute = om.writeValueAsString(attributes);
+ dbStore.engineSetKeyEntry(versionName, new SecretKeySpec(material, cipher), masterKey, cipher, bitLength, description, version, attribute);
+ } catch (KeyStoreException e) {
+ e.printStackTrace();
+ throw new IOException("Can't store key " + versionName,e);
+ }
+ changed = true;
+ return new KeyVersion(name, versionName, material);
+ }
+
+ @Override
+ public void deleteKey(String name) throws IOException {
+ Metadata meta = getMetadata(name);
+ if (meta == null) {
+ throw new IOException("Key " + name + " does not exist");
+ }
+ for(int v=0; v < meta.getVersions(); ++v) {
+ String versionName = buildVersionName(name, v);
+ try {
+ if (dbStore.engineContainsAlias(versionName)) {
+ dbStore.engineDeleteEntry(versionName);
+ }
+ } catch (KeyStoreException e) {
+ throw new IOException("Problem removing " + versionName, e);
+ }
+ }
+ try {
+ if (dbStore.engineContainsAlias(name)) {
+ dbStore.engineDeleteEntry(name);
+ }
+ } catch (KeyStoreException e) {
+ e.printStackTrace();
+ throw new IOException("Problem removing " + name + " from " + this, e);
+ }
+ cache.remove(name);
+ changed = true;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ try {
+ if (!changed) {
+ return;
+ }
+ // put all of the updates into the db
+ for(Map.Entry<String, Metadata> entry: cache.entrySet()) {
+ try {
+ Metadata metadata = entry.getValue();
+ ObjectMapper om = new ObjectMapper();
+ String attributes = om.writeValueAsString(metadata.getAttributes());
+ dbStore.engineSetKeyEntry(entry.getKey(), new KeyMetadata(metadata), masterKey, metadata.getAlgorithm(), metadata.getBitLength(), metadata.getDescription(), metadata.getVersions(), attributes);
+ } catch (KeyStoreException e) {
+ e.printStackTrace();
+ throw new IOException("Can't set metadata key " + entry.getKey(),e );
+ }
+ }
+ try {
+ dbStore.engineStore(null, masterKey);
+ } catch (NoSuchAlgorithmException e) {
+ e.printStackTrace();
+ throw new IOException("No such algorithm storing key", e);
+ } catch (CertificateException e) {
+ e.printStackTrace();
+ throw new IOException("Certificate exception storing key", e);
+ }
+ changed = false;
+ }catch (IOException ioe) {
+ ioe.printStackTrace();
+ throw ioe;
+ }
+ }
+
+ @Override
+ public KeyVersion getKeyVersion(String versionName) throws IOException {
+ SecretKeySpec key = null;
+ try {
+ if (!dbStore.engineContainsAlias(versionName)) {
+ return null;
+ }
+ key = (SecretKeySpec) dbStore.engineGetKey(versionName, masterKey);
+ } catch (NoSuchAlgorithmException e) {
+ e.printStackTrace();
+ throw new IOException("Can't get algorithm for key " + key, e);
+ } catch (UnrecoverableKeyException e) {
+ e.printStackTrace();
+ throw new IOException("Can't recover key " + key, e);
+ }
+ return new KeyVersion(getBaseName(versionName), versionName, key.getEncoded());
+ }
+
+ @Override
+ public List<KeyVersion> getKeyVersions(String name) throws IOException {
+ List<KeyVersion> list = new ArrayList<KeyVersion>();
+ Metadata km = getMetadata(name);
+ if (km != null) {
+ int latestVersion = km.getVersions();
+ KeyVersion v = null;
+ String versionName = null;
+ for (int i = 0; i < latestVersion; i++) {
+ versionName = buildVersionName(name, i);
+ v = getKeyVersion(versionName);
+ if (v != null) {
+ list.add(v);
+ }
+ }
+ }
+ return list;
+ }
+
+ @Override
+ public List<String> getKeys() throws IOException {
+ ArrayList<String> list = new ArrayList<String>();
+ String alias = null;
+ Enumeration<String> e = dbStore.engineAliases();
+ while (e.hasMoreElements()) {
+ alias = e.nextElement();
+ // only include the metadata key names in the list of names
+ if (!alias.contains("@")) {
+ list.add(alias);
+ }
+ }
+ return list;
+ }
+
+ @Override
+ public Metadata getMetadata(String name) throws IOException {
+ if (cache.containsKey(name)) {
+ return cache.get(name);
+ }
+ try {
+ if (!dbStore.engineContainsAlias(name)) {
+ return null;
+ }
+ Metadata meta = ((KeyMetadata) dbStore.engineGetKey(name, masterKey)).metadata;
+ cache.put(name, meta);
+ return meta;
+ } catch (NoSuchAlgorithmException e) {
+ e.printStackTrace();
+ throw new IOException("Can't get algorithm for " + name, e);
+ } catch (UnrecoverableKeyException e) {
+ e.printStackTrace();
+ throw new IOException("Can't recover key for " + name, e);
+ }
+ }
+
+ @Override
+ public KeyVersion rollNewVersion(String name, byte[] material)throws IOException {
+ Metadata meta = getMetadata(name);
+ if (meta == null) {
+ throw new IOException("Key " + name + " not found");
+ }
+ if (meta.getBitLength() != 8 * material.length) {
+ throw new IOException("Wrong key length. Required " + meta.getBitLength() + ", but got " + (8 * material.length));
+ }
+ int nextVersion = meta.addVersion();
+ String versionName = buildVersionName(name, nextVersion);
+ return innerSetKeyVersion(name, versionName, material, meta.getCipher(), meta.getBitLength(), meta.getDescription(), meta.getVersions(), meta.getAttributes());
+ }
+
+ /**
+ * The factory to create JksProviders, which is used by the ServiceLoader.
+ */
+ public static class Factory extends KeyProviderFactory {
+ @Override
+ public KeyProvider createProvider(URI providerName,
+ Configuration conf) throws IOException {
+ try {
+ if (SCHEME_NAME.equals(providerName.getScheme())) {
+ return new RangerKeyStoreProvider(conf);
+ }
+ } catch (Throwable e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+ }
+
+ /**
+ * An adapter between a KeyStore Key and our Metadata. This is used to store
+ * the metadata in a KeyStore even though isn't really a key.
+ */
+ public static class KeyMetadata implements Key, Serializable {
+ private Metadata metadata;
+ private final static long serialVersionUID = 8405872419967874451L;
+
+ private KeyMetadata(Metadata meta) {
+ this.metadata = meta;
+ }
+
+ @Override
+ public String getAlgorithm() {
+ return metadata.getCipher();
+ }
+
+ @Override
+ public String getFormat() {
+ return KEY_METADATA;
+ }
+
+ @Override
+ public byte[] getEncoded() {
+ return new byte[0];
+ }
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ byte[] serialized = metadata.serialize();
+ out.writeInt(serialized.length);
+ out.write(serialized);
+ }
+
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ byte[] buf = new byte[in.readInt()];
+ in.readFully(buf);
+ metadata = new Metadata(buf);
+ }
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java
new file mode 100644
index 0000000..6102bfc
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java
@@ -0,0 +1,210 @@
+package org.apache.hadoop.crypto.key;
+
+import java.security.Key;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import javax.crypto.Cipher;
+import javax.crypto.KeyGenerator;
+import javax.crypto.SecretKey;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.PBEKeySpec;
+import javax.crypto.spec.PBEParameterSpec;
+import javax.crypto.spec.SecretKeySpec;
+import org.apache.log4j.Logger;
+import org.apache.ranger.kms.dao.DaoManager;
+import org.apache.ranger.kms.dao.RangerMasterKeyDao;
+import org.apache.ranger.entity.XXRangerMasterKey;
+
+import com.sun.org.apache.xml.internal.security.exceptions.Base64DecodingException;
+import com.sun.org.apache.xml.internal.security.utils.Base64;
+
+public class RangerMasterKey {
+
+ static final Logger logger = Logger.getLogger(RangerMasterKey.class);
+
+ private static final String MK_CIPHER = "AES";
+ private static final int MK_KeySize = 256;
+ private static final int SALT_SIZE = 8;
+ private static final String PBE_ALGO = "PBEWithMD5AndTripleDES";
+ private static final String MD_ALGO = "MD5";
+
+ public static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password";
+
+ private DaoManager daoManager;
+
+ public RangerMasterKey() {
+ }
+
+ public RangerMasterKey(DaoManager daoManager) {
+ this.daoManager = daoManager;
+ }
+
+ /**
+ * To get Master Key
+ * @param password password to be used for decryption
+ * @return Decrypted Master Key
+ * @throws Throwable
+ */
+ public String getMasterKey(String password) throws Throwable{
+ logger.info("Getting Master Key");
+ byte masterKeyByte[] = getEncryptedMK();
+ if(masterKeyByte != null && masterKeyByte.length > 0){
+ String masterKey = decryptMasterKey(masterKeyByte, password);
+ return masterKey;
+ }else{
+ throw new Exception("No Master Key Found");
+ }
+ }
+
+ /**
+ * Generate the master key encrypt's it and save it in database
+ * @param password password to be used for encryption
+ * @return true if successfully created the master key
+ * false if master key generation was unsuccessful or already master key exists
+ * @throws Throwable
+ */
+ public boolean generateMasterKey(String password) throws Throwable{
+ logger.info("Generating Master Key");
+ String encryptedMasterKey = encryptMasterKey(password);
+ String savedKey = saveEncryptedMK(encryptedMasterKey, daoManager);
+ if(savedKey != null && !savedKey.trim().equals("")){
+ logger.debug("Master Key Created with id = "+savedKey);
+ return true;
+ }
+ return false;
+ }
+
+ private String decryptMasterKey(byte masterKey[], String password) throws Throwable {
+ logger.debug("Decrypting Master Key");
+ PBEKeySpec pbeKeyspec = getPBEParameterSpec(password) ;
+ byte[] masterKeyFromDBDecrypted = decryptKey(masterKey, pbeKeyspec) ;
+ SecretKey masterKeyFromDB = getMasterKeyFromBytes(masterKeyFromDBDecrypted) ;
+ return Base64.encode(masterKeyFromDB.getEncoded());
+ }
+
+ private byte[] getEncryptedMK() throws Base64DecodingException {
+ logger.debug("Retrieving Encrypted Master Key from database");
+ try{
+ if(daoManager != null){
+ RangerMasterKeyDao rangerKMSDao = new RangerMasterKeyDao(daoManager);
+ List<XXRangerMasterKey> lstRangerMasterKey = rangerKMSDao.getAll();
+ if(lstRangerMasterKey.size() < 1){
+ throw new Exception("No Master Key exists");
+ }else if(lstRangerMasterKey.size() > 1){
+ throw new Exception("More than one Master Key exists");
+ }else {
+ XXRangerMasterKey rangerMasterKey = rangerKMSDao.getById(lstRangerMasterKey.get(0).getId());
+ String masterKeyStr = rangerMasterKey.getMasterKey();
+ byte[] masterKeyFromDBEncrypted = Base64.decode(masterKeyStr) ;
+ return masterKeyFromDBEncrypted;
+ }
+ }
+ }catch(Exception e){
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ private String saveEncryptedMK(String encryptedMasterKey, DaoManager daoManager) {
+ logger.debug("Saving Encrypted Master Key to database");
+ XXRangerMasterKey xxRangerMasterKey = new XXRangerMasterKey();
+ xxRangerMasterKey.setCipher(MK_CIPHER);
+ xxRangerMasterKey.setBitLength(MK_KeySize);
+ xxRangerMasterKey.setMasterKey(encryptedMasterKey);
+ try{
+ if(daoManager != null){
+ RangerMasterKeyDao rangerKMSDao = new RangerMasterKeyDao(daoManager);
+ Long l = rangerKMSDao.getAllCount();
+ if(l < 1){
+ XXRangerMasterKey rangerMasterKey = rangerKMSDao.create(xxRangerMasterKey);
+ return rangerMasterKey.getId().toString();
+ }
+ }
+ }catch(Exception e){
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ private String encryptMasterKey(String password) throws Throwable {
+ logger.debug("Encrypting Master Key");
+ Key secretKey = generateMasterKey();
+ PBEKeySpec pbeKeySpec = getPBEParameterSpec(password);
+ byte[] masterKeyToDB = encryptKey(secretKey.getEncoded(), pbeKeySpec);
+ String masterKey = Base64.encode(masterKeyToDB) ;
+ return masterKey;
+ }
+
+ private Key generateMasterKey() throws NoSuchAlgorithmException{
+ KeyGenerator kg = KeyGenerator.getInstance(MK_CIPHER);
+ kg.init(MK_KeySize);
+ return kg.generateKey();
+ }
+
+ private PBEKeySpec getPBEParameterSpec(String password) throws Throwable {
+ MessageDigest md = MessageDigest.getInstance(MD_ALGO) ;
+ byte[] saltGen = md.digest(password.getBytes()) ;
+ byte[] salt = new byte[SALT_SIZE] ;
+ System.arraycopy(saltGen, 0, salt, 0, SALT_SIZE);
+ int iteration = password.toCharArray().length + 1 ;
+ PBEKeySpec spec = new PBEKeySpec(password.toCharArray(), salt, iteration) ;
+ return spec ;
+ }
+ private byte[] encryptKey(byte[] data, PBEKeySpec keyspec) throws Throwable {
+ SecretKey key = getPasswordKey(keyspec) ;
+ PBEParameterSpec paramSpec = new PBEParameterSpec(keyspec.getSalt(), keyspec.getIterationCount()) ;
+ Cipher c = Cipher.getInstance(key.getAlgorithm()) ;
+ c.init(Cipher.ENCRYPT_MODE, key,paramSpec);
+ byte[] encrypted = c.doFinal(data) ;
+
+ return encrypted ;
+ }
+ private SecretKey getPasswordKey(PBEKeySpec keyspec) throws Throwable {
+ SecretKeyFactory factory = SecretKeyFactory.getInstance(PBE_ALGO) ;
+ SecretKey PbKey = factory.generateSecret(keyspec) ;
+ return PbKey ;
+ }
+ private byte[] decryptKey(byte[] encrypted, PBEKeySpec keyspec) throws Throwable {
+ SecretKey key = getPasswordKey(keyspec) ;
+ PBEParameterSpec paramSpec = new PBEParameterSpec(keyspec.getSalt(), keyspec.getIterationCount()) ;
+ Cipher c = Cipher.getInstance(key.getAlgorithm()) ;
+ c.init(Cipher.DECRYPT_MODE, key, paramSpec);
+ byte[] data = c.doFinal(encrypted) ;
+ return data ;
+ }
+ private SecretKey getMasterKeyFromBytes(byte[] keyData) throws Throwable {
+ SecretKeySpec sks = new SecretKeySpec(keyData, MK_CIPHER) ;
+ return sks ;
+ }
+
+ public Map<String, String> getPropertiesWithPrefix(Properties props, String prefix) {
+ Map<String, String> prefixedProperties = new HashMap<String, String>();
+
+ if(props != null && prefix != null) {
+ for(String key : props.stringPropertyNames()) {
+ if(key == null) {
+ continue;
+ }
+
+ String val = props.getProperty(key);
+
+ if(key.startsWith(prefix)) {
+ key = key.substring(prefix.length());
+
+ if(key == null) {
+ continue;
+ }
+
+ prefixedProperties.put(key, val);
+ }
+ }
+ }
+
+ return prefixedProperties;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
new file mode 100644
index 0000000..64af2b6
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.crypto.key.kms.server;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.NoSuchAlgorithmException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.crypto.key.kms.ValueQueue;
+import org.apache.hadoop.crypto.key.kms.ValueQueue.SyncGenerationPolicy;
+
+/**
+ * A {@link KeyProviderCryptoExtension} that pre-generates and caches encrypted
+ * keys.
+ */
+@InterfaceAudience.Private
+public class EagerKeyGeneratorKeyProviderCryptoExtension
+ extends KeyProviderCryptoExtension {
+
+ private static final String KEY_CACHE_PREFIX =
+ "hadoop.security.kms.encrypted.key.cache.";
+
+ public static final String KMS_KEY_CACHE_SIZE =
+ KEY_CACHE_PREFIX + "size";
+ public static final int KMS_KEY_CACHE_SIZE_DEFAULT = 100;
+
+ public static final String KMS_KEY_CACHE_LOW_WATERMARK =
+ KEY_CACHE_PREFIX + "low.watermark";
+ public static final float KMS_KEY_CACHE_LOW_WATERMARK_DEFAULT = 0.30f;
+
+ public static final String KMS_KEY_CACHE_EXPIRY_MS =
+ KEY_CACHE_PREFIX + "expiry";
+ public static final int KMS_KEY_CACHE_EXPIRY_DEFAULT = 43200000;
+
+ public static final String KMS_KEY_CACHE_NUM_REFILL_THREADS =
+ KEY_CACHE_PREFIX + "num.fill.threads";
+ public static final int KMS_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT = 2;
+
+
+ private static class CryptoExtension
+ implements KeyProviderCryptoExtension.CryptoExtension {
+
+ private class EncryptedQueueRefiller implements
+ ValueQueue.QueueRefiller<EncryptedKeyVersion> {
+
+ @Override
+ public void fillQueueForKey(String keyName,
+ Queue<EncryptedKeyVersion> keyQueue, int numKeys) throws IOException {
+ List<EncryptedKeyVersion> retEdeks =
+ new LinkedList<EncryptedKeyVersion>();
+ for (int i = 0; i < numKeys; i++) {
+ try {
+ retEdeks.add(keyProviderCryptoExtension.generateEncryptedKey(
+ keyName));
+ } catch (GeneralSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+ keyQueue.addAll(retEdeks);
+ }
+ }
+
+ private KeyProviderCryptoExtension keyProviderCryptoExtension;
+ private final ValueQueue<EncryptedKeyVersion> encKeyVersionQueue;
+
+ public CryptoExtension(Configuration conf,
+ KeyProviderCryptoExtension keyProviderCryptoExtension) {
+ this.keyProviderCryptoExtension = keyProviderCryptoExtension;
+ encKeyVersionQueue =
+ new ValueQueue<KeyProviderCryptoExtension.EncryptedKeyVersion>(
+ conf.getInt(KMS_KEY_CACHE_SIZE,
+ KMS_KEY_CACHE_SIZE_DEFAULT),
+ conf.getFloat(KMS_KEY_CACHE_LOW_WATERMARK,
+ KMS_KEY_CACHE_LOW_WATERMARK_DEFAULT),
+ conf.getInt(KMS_KEY_CACHE_EXPIRY_MS,
+ KMS_KEY_CACHE_EXPIRY_DEFAULT),
+ conf.getInt(KMS_KEY_CACHE_NUM_REFILL_THREADS,
+ KMS_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT),
+ SyncGenerationPolicy.LOW_WATERMARK, new EncryptedQueueRefiller()
+ );
+ }
+
+ @Override
+ public void warmUpEncryptedKeys(String... keyNames) throws
+ IOException {
+ try {
+ encKeyVersionQueue.initializeQueuesForKeys(keyNames);
+ } catch (ExecutionException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public void drain(String keyName) {
+ encKeyVersionQueue.drain(keyName);
+ }
+
+ @Override
+ public EncryptedKeyVersion generateEncryptedKey(String encryptionKeyName)
+ throws IOException, GeneralSecurityException {
+ try {
+ return encKeyVersionQueue.getNext(encryptionKeyName);
+ } catch (ExecutionException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public KeyVersion
+ decryptEncryptedKey(EncryptedKeyVersion encryptedKeyVersion)
+ throws IOException, GeneralSecurityException {
+ return keyProviderCryptoExtension.decryptEncryptedKey(
+ encryptedKeyVersion);
+ }
+ }
+
+ /**
+ * This class is a proxy for a <code>KeyProviderCryptoExtension</code> that
+ * decorates the underlying <code>CryptoExtension</code> with one that eagerly
+ * caches pre-generated Encrypted Keys using a <code>ValueQueue</code>
+ *
+ * @param conf Configuration object to load parameters from
+ * @param keyProviderCryptoExtension <code>KeyProviderCryptoExtension</code>
+ * to delegate calls to.
+ */
+ public EagerKeyGeneratorKeyProviderCryptoExtension(Configuration conf,
+ KeyProviderCryptoExtension keyProviderCryptoExtension) {
+ super(keyProviderCryptoExtension,
+ new CryptoExtension(conf, keyProviderCryptoExtension));
+ }
+
+ @Override
+ public KeyVersion rollNewVersion(String name)
+ throws NoSuchAlgorithmException, IOException {
+ KeyVersion keyVersion = super.rollNewVersion(name);
+ getExtension().drain(name);
+ return keyVersion;
+ }
+
+ @Override
+ public KeyVersion rollNewVersion(String name, byte[] material)
+ throws IOException {
+ KeyVersion keyVersion = super.rollNewVersion(name, material);
+ getExtension().drain(name);
+ return keyVersion;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
new file mode 100644
index 0000000..9b5b5a1
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
@@ -0,0 +1,482 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
+import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
+import org.apache.hadoop.crypto.key.kms.server.KMSACLsType.Type;
+import org.apache.hadoop.security.token.delegation.web.HttpUserGroupInformation;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Class providing the REST bindings, via Jersey, for the KMS.
+ */
+@Path(KMSRESTConstants.SERVICE_VERSION)
+@InterfaceAudience.Private
+public class KMS {
+
+ public static enum KMSOp {
+ CREATE_KEY, DELETE_KEY, ROLL_NEW_VERSION,
+ GET_KEYS, GET_KEYS_METADATA,
+ GET_KEY_VERSIONS, GET_METADATA, GET_KEY_VERSION, GET_CURRENT_KEY,
+ GENERATE_EEK, DECRYPT_EEK
+ }
+
+ private KeyProviderCryptoExtension provider;
+ private KMSAudit kmsAudit;
+
+ public KMS() throws Exception {
+ provider = KMSWebApp.getKeyProvider();
+ kmsAudit= KMSWebApp.getKMSAudit();
+ }
+
+ private void assertAccess(Type aclType, UserGroupInformation ugi,
+ KMSOp operation) throws AccessControlException {
+ KMSWebApp.getACLs().assertAccess(aclType, ugi, operation, null);
+ }
+
+ private void assertAccess(Type aclType, UserGroupInformation ugi,
+ KMSOp operation, String key) throws AccessControlException {
+ KMSWebApp.getACLs().assertAccess(aclType, ugi, operation, key);
+ }
+
+ private static KeyProvider.KeyVersion removeKeyMaterial(
+ KeyProvider.KeyVersion keyVersion) {
+ return new KMSClientProvider.KMSKeyVersion(keyVersion.getName(),
+ keyVersion.getVersionName(), null);
+ }
+
+ private static URI getKeyURI(String name) throws URISyntaxException {
+ return new URI(KMSRESTConstants.SERVICE_VERSION + "/" +
+ KMSRESTConstants.KEY_RESOURCE + "/" + name);
+ }
+
+ @POST
+ @Path(KMSRESTConstants.KEYS_RESOURCE)
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ @SuppressWarnings("unchecked")
+ public Response createKey(Map jsonKey) throws Exception {
+ KMSWebApp.getAdminCallsMeter().mark();
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ final String name = (String) jsonKey.get(KMSRESTConstants.NAME_FIELD);
+ KMSClientProvider.checkNotEmpty(name, KMSRESTConstants.NAME_FIELD);
+ assertAccess(Type.CREATE, user, KMSOp.CREATE_KEY, name);
+ String cipher = (String) jsonKey.get(KMSRESTConstants.CIPHER_FIELD);
+ final String material = (String) jsonKey.get(KMSRESTConstants.MATERIAL_FIELD);
+ int length = (jsonKey.containsKey(KMSRESTConstants.LENGTH_FIELD))
+ ? (Integer) jsonKey.get(KMSRESTConstants.LENGTH_FIELD) : 0;
+ String description = (String)
+ jsonKey.get(KMSRESTConstants.DESCRIPTION_FIELD);
+ Map<String, String> attributes = (Map<String, String>)
+ jsonKey.get(KMSRESTConstants.ATTRIBUTES_FIELD);
+ if (material != null) {
+ assertAccess(Type.SET_KEY_MATERIAL, user,
+ KMSOp.CREATE_KEY, name);
+ }
+ final KeyProvider.Options options = new KeyProvider.Options(
+ KMSWebApp.getConfiguration());
+ if (cipher != null) {
+ options.setCipher(cipher);
+ }
+ if (length != 0) {
+ options.setBitLength(length);
+ }
+ options.setDescription(description);
+ options.setAttributes(attributes);
+
+ KeyProvider.KeyVersion keyVersion = user.doAs(
+ new PrivilegedExceptionAction<KeyVersion>() {
+ @Override
+ public KeyVersion run() throws Exception {
+ KeyProvider.KeyVersion keyVersion = (material != null)
+ ? provider.createKey(name, Base64.decodeBase64(material), options)
+ : provider.createKey(name, options);
+ provider.flush();
+ return keyVersion;
+ }
+ }
+ );
+
+ kmsAudit.ok(user, KMSOp.CREATE_KEY, name, "UserProvidedMaterial:" +
+ (material != null) + " Description:" + description);
+
+ if (!KMSWebApp.getACLs().hasAccess(Type.GET, user)) {
+ keyVersion = removeKeyMaterial(keyVersion);
+ }
+ Map json = KMSServerJSONUtils.toJSON(keyVersion);
+ String requestURL = KMSMDCFilter.getURL();
+ int idx = requestURL.lastIndexOf(KMSRESTConstants.KEYS_RESOURCE);
+ requestURL = requestURL.substring(0, idx);
+ String keyURL = requestURL + KMSRESTConstants.KEY_RESOURCE + "/" + name;
+ return Response.created(getKeyURI(name)).type(MediaType.APPLICATION_JSON).
+ header("Location", keyURL).entity(json).build();
+ }
+
+ @DELETE
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}")
+ public Response deleteKey(@PathParam("name") final String name)
+ throws Exception {
+ KMSWebApp.getAdminCallsMeter().mark();
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ assertAccess(Type.DELETE, user, KMSOp.DELETE_KEY, name);
+ KMSClientProvider.checkNotEmpty(name, "name");
+
+ user.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ provider.deleteKey(name);
+ provider.flush();
+ return null;
+ }
+ });
+
+ kmsAudit.ok(user, KMSOp.DELETE_KEY, name, "");
+
+ return Response.ok().build();
+ }
+
+ @POST
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}")
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response rolloverKey(@PathParam("name") final String name,
+ Map jsonMaterial) throws Exception {
+ KMSWebApp.getAdminCallsMeter().mark();
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ assertAccess(Type.ROLLOVER, user, KMSOp.ROLL_NEW_VERSION, name);
+ KMSClientProvider.checkNotEmpty(name, "name");
+ final String material = (String)
+ jsonMaterial.get(KMSRESTConstants.MATERIAL_FIELD);
+ if (material != null) {
+ assertAccess(Type.SET_KEY_MATERIAL, user,
+ KMSOp.ROLL_NEW_VERSION, name);
+ }
+
+ KeyProvider.KeyVersion keyVersion = user.doAs(
+ new PrivilegedExceptionAction<KeyVersion>() {
+ @Override
+ public KeyVersion run() throws Exception {
+ KeyVersion keyVersion = (material != null)
+ ? provider.rollNewVersion(name, Base64.decodeBase64(material))
+ : provider.rollNewVersion(name);
+ provider.flush();
+ return keyVersion;
+ }
+ }
+ );
+
+ kmsAudit.ok(user, KMSOp.ROLL_NEW_VERSION, name, "UserProvidedMaterial:" +
+ (material != null) + " NewVersion:" + keyVersion.getVersionName());
+
+ if (!KMSWebApp.getACLs().hasAccess(Type.GET, user)) {
+ keyVersion = removeKeyMaterial(keyVersion);
+ }
+ Map json = KMSServerJSONUtils.toJSON(keyVersion);
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEYS_METADATA_RESOURCE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getKeysMetadata(@QueryParam(KMSRESTConstants.KEY)
+ List<String> keyNamesList) throws Exception {
+ KMSWebApp.getAdminCallsMeter().mark();
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ final String[] keyNames = keyNamesList.toArray(
+ new String[keyNamesList.size()]);
+ assertAccess(Type.GET_METADATA, user, KMSOp.GET_KEYS_METADATA);
+
+ KeyProvider.Metadata[] keysMeta = user.doAs(
+ new PrivilegedExceptionAction<KeyProvider.Metadata[]>() {
+ @Override
+ public KeyProvider.Metadata[] run() throws Exception {
+ return provider.getKeysMetadata(keyNames);
+ }
+ }
+ );
+
+ Object json = KMSServerJSONUtils.toJSON(keyNames, keysMeta);
+ kmsAudit.ok(user, KMSOp.GET_KEYS_METADATA, "");
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEYS_NAMES_RESOURCE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getKeyNames() throws Exception {
+ KMSWebApp.getAdminCallsMeter().mark();
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ assertAccess(Type.GET_KEYS, user, KMSOp.GET_KEYS);
+
+ List<String> json = user.doAs(
+ new PrivilegedExceptionAction<List<String>>() {
+ @Override
+ public List<String> run() throws Exception {
+ return provider.getKeys();
+ }
+ }
+ );
+
+ kmsAudit.ok(user, KMSOp.GET_KEYS, "");
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}")
+ public Response getKey(@PathParam("name") String name)
+ throws Exception {
+ return getMetadata(name);
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" +
+ KMSRESTConstants.METADATA_SUB_RESOURCE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getMetadata(@PathParam("name") final String name)
+ throws Exception {
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ KMSClientProvider.checkNotEmpty(name, "name");
+ KMSWebApp.getAdminCallsMeter().mark();
+ assertAccess(Type.GET_METADATA, user, KMSOp.GET_METADATA, name);
+
+ KeyProvider.Metadata metadata = user.doAs(
+ new PrivilegedExceptionAction<KeyProvider.Metadata>() {
+ @Override
+ public KeyProvider.Metadata run() throws Exception {
+ return provider.getMetadata(name);
+ }
+ }
+ );
+
+ Object json = KMSServerJSONUtils.toJSON(name, metadata);
+ kmsAudit.ok(user, KMSOp.GET_METADATA, name, "");
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" +
+ KMSRESTConstants.CURRENT_VERSION_SUB_RESOURCE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getCurrentVersion(@PathParam("name") final String name)
+ throws Exception {
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ KMSClientProvider.checkNotEmpty(name, "name");
+ KMSWebApp.getKeyCallsMeter().mark();
+ assertAccess(Type.GET, user, KMSOp.GET_CURRENT_KEY, name);
+
+ KeyVersion keyVersion = user.doAs(
+ new PrivilegedExceptionAction<KeyVersion>() {
+ @Override
+ public KeyVersion run() throws Exception {
+ return provider.getCurrentKey(name);
+ }
+ }
+ );
+
+ Object json = KMSServerJSONUtils.toJSON(keyVersion);
+ kmsAudit.ok(user, KMSOp.GET_CURRENT_KEY, name, "");
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEY_VERSION_RESOURCE + "/{versionName:.*}")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getKeyVersion(
+ @PathParam("versionName") final String versionName) throws Exception {
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ KMSClientProvider.checkNotEmpty(versionName, "versionName");
+ KMSWebApp.getKeyCallsMeter().mark();
+ assertAccess(Type.GET, user, KMSOp.GET_KEY_VERSION);
+
+ KeyVersion keyVersion = user.doAs(
+ new PrivilegedExceptionAction<KeyVersion>() {
+ @Override
+ public KeyVersion run() throws Exception {
+ return provider.getKeyVersion(versionName);
+ }
+ }
+ );
+
+ if (keyVersion != null) {
+ kmsAudit.ok(user, KMSOp.GET_KEY_VERSION, keyVersion.getName(), "");
+ }
+ Object json = KMSServerJSONUtils.toJSON(keyVersion);
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @GET
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" +
+ KMSRESTConstants.EEK_SUB_RESOURCE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response generateEncryptedKeys(
+ @PathParam("name") final String name,
+ @QueryParam(KMSRESTConstants.EEK_OP) String edekOp,
+ @DefaultValue("1")
+ @QueryParam(KMSRESTConstants.EEK_NUM_KEYS) final int numKeys)
+ throws Exception {
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ KMSClientProvider.checkNotEmpty(name, "name");
+ KMSClientProvider.checkNotNull(edekOp, "eekOp");
+
+ Object retJSON;
+ if (edekOp.equals(KMSRESTConstants.EEK_GENERATE)) {
+ assertAccess(Type.GENERATE_EEK, user, KMSOp.GENERATE_EEK, name);
+
+ final List<EncryptedKeyVersion> retEdeks =
+ new LinkedList<EncryptedKeyVersion>();
+ try {
+
+ user.doAs(
+ new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ for (int i = 0; i < numKeys; i++) {
+ retEdeks.add(provider.generateEncryptedKey(name));
+ }
+ return null;
+ }
+ }
+ );
+
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ kmsAudit.ok(user, KMSOp.GENERATE_EEK, name, "");
+ retJSON = new ArrayList();
+ for (EncryptedKeyVersion edek : retEdeks) {
+ ((ArrayList)retJSON).add(KMSServerJSONUtils.toJSON(edek));
+ }
+ } else {
+ throw new IllegalArgumentException("Wrong " + KMSRESTConstants.EEK_OP +
+ " value, it must be " + KMSRESTConstants.EEK_GENERATE + " or " +
+ KMSRESTConstants.EEK_DECRYPT);
+ }
+ KMSWebApp.getGenerateEEKCallsMeter().mark();
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(retJSON)
+ .build();
+ }
+
+ @SuppressWarnings("rawtypes")
+ @POST
+ @Path(KMSRESTConstants.KEY_VERSION_RESOURCE + "/{versionName:.*}/" +
+ KMSRESTConstants.EEK_SUB_RESOURCE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response decryptEncryptedKey(
+ @PathParam("versionName") final String versionName,
+ @QueryParam(KMSRESTConstants.EEK_OP) String eekOp,
+ Map jsonPayload)
+ throws Exception {
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ KMSClientProvider.checkNotEmpty(versionName, "versionName");
+ KMSClientProvider.checkNotNull(eekOp, "eekOp");
+
+ final String keyName = (String) jsonPayload.get(
+ KMSRESTConstants.NAME_FIELD);
+ String ivStr = (String) jsonPayload.get(KMSRESTConstants.IV_FIELD);
+ String encMaterialStr =
+ (String) jsonPayload.get(KMSRESTConstants.MATERIAL_FIELD);
+ Object retJSON;
+ if (eekOp.equals(KMSRESTConstants.EEK_DECRYPT)) {
+ assertAccess(Type.DECRYPT_EEK, user, KMSOp.DECRYPT_EEK, keyName);
+ KMSClientProvider.checkNotNull(ivStr, KMSRESTConstants.IV_FIELD);
+ final byte[] iv = Base64.decodeBase64(ivStr);
+ KMSClientProvider.checkNotNull(encMaterialStr,
+ KMSRESTConstants.MATERIAL_FIELD);
+ final byte[] encMaterial = Base64.decodeBase64(encMaterialStr);
+
+ KeyProvider.KeyVersion retKeyVersion = user.doAs(
+ new PrivilegedExceptionAction<KeyVersion>() {
+ @Override
+ public KeyVersion run() throws Exception {
+ return provider.decryptEncryptedKey(
+ new KMSClientProvider.KMSEncryptedKeyVersion(keyName,
+ versionName, iv, KeyProviderCryptoExtension.EEK,
+ encMaterial)
+ );
+ }
+ }
+ );
+
+ retJSON = KMSServerJSONUtils.toJSON(retKeyVersion);
+ kmsAudit.ok(user, KMSOp.DECRYPT_EEK, keyName, "");
+ } else {
+ throw new IllegalArgumentException("Wrong " + KMSRESTConstants.EEK_OP +
+ " value, it must be " + KMSRESTConstants.EEK_GENERATE + " or " +
+ KMSRESTConstants.EEK_DECRYPT);
+ }
+ KMSWebApp.getDecryptEEKCallsMeter().mark();
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(retJSON)
+ .build();
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" +
+ KMSRESTConstants.VERSIONS_SUB_RESOURCE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getKeyVersions(@PathParam("name") final String name)
+ throws Exception {
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ KMSClientProvider.checkNotEmpty(name, "name");
+ KMSWebApp.getKeyCallsMeter().mark();
+ assertAccess(Type.GET, user, KMSOp.GET_KEY_VERSIONS, name);
+
+ List<KeyVersion> ret = user.doAs(
+ new PrivilegedExceptionAction<List<KeyVersion>>() {
+ @Override
+ public List<KeyVersion> run() throws Exception {
+ return provider.getKeyVersions(name);
+ }
+ }
+ );
+
+ Object json = KMSServerJSONUtils.toJSON(ret);
+ kmsAudit.ok(user, KMSOp.GET_KEY_VERSIONS, name, "");
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
new file mode 100644
index 0000000..f2298c0
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
@@ -0,0 +1,253 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.kms.server.KMS.KMSOp;
+import org.apache.hadoop.crypto.key.kms.server.KMSACLsType.Type;
+import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyACLs;
+import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyOpType;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+
+/**
+ * Provides access to the <code>AccessControlList</code>s used by KMS,
+ * hot-reloading them if the <code>kms-acls.xml</code> file where the ACLs
+ * are defined has been updated.
+ */
+@InterfaceAudience.Private
+public class KMSACLs implements Runnable, KeyACLs {
+ private static final Logger LOG = LoggerFactory.getLogger(KMSACLs.class);
+
+ private static final String UNAUTHORIZED_MSG_WITH_KEY =
+ "User:%s not allowed to do '%s' on '%s'";
+
+ private static final String UNAUTHORIZED_MSG_WITHOUT_KEY =
+ "User:%s not allowed to do '%s'";
+
+ public static final String ACL_DEFAULT = AccessControlList.WILDCARD_ACL_VALUE;
+
+ public static final int RELOADER_SLEEP_MILLIS = 1000;
+
+ private volatile Map<Type, AccessControlList> acls;
+ private volatile Map<Type, AccessControlList> blacklistedAcls;
+ private volatile Map<String, HashMap<KeyOpType, AccessControlList>> keyAcls;
+ private final Map<KeyOpType, AccessControlList> defaultKeyAcls =
+ new HashMap<KeyOpType, AccessControlList>();
+ private ScheduledExecutorService executorService;
+ private long lastReload;
+
+ KMSACLs(Configuration conf) {
+ if (conf == null) {
+ conf = loadACLs();
+ }
+ setKMSACLs(conf);
+ setKeyACLs(conf);
+ }
+
+ public KMSACLs() {
+ this(null);
+ }
+
+ private void setKMSACLs(Configuration conf) {
+ Map<Type, AccessControlList> tempAcls = new HashMap<Type, AccessControlList>();
+ Map<Type, AccessControlList> tempBlacklist = new HashMap<Type, AccessControlList>();
+ for (Type aclType : Type.values()) {
+ String aclStr = conf.get(aclType.getAclConfigKey(), ACL_DEFAULT);
+ tempAcls.put(aclType, new AccessControlList(aclStr));
+ String blacklistStr = conf.get(aclType.getBlacklistConfigKey());
+ if (blacklistStr != null) {
+ // Only add if blacklist is present
+ tempBlacklist.put(aclType, new AccessControlList(blacklistStr));
+ LOG.info("'{}' Blacklist '{}'", aclType, blacklistStr);
+ }
+ LOG.info("'{}' ACL '{}'", aclType, aclStr);
+ }
+ acls = tempAcls;
+ blacklistedAcls = tempBlacklist;
+ }
+
+ private void setKeyACLs(Configuration conf) {
+ Map<String, HashMap<KeyOpType, AccessControlList>> tempKeyAcls =
+ new HashMap<String, HashMap<KeyOpType,AccessControlList>>();
+ Map<String, String> allKeyACLS =
+ conf.getValByRegex(Pattern.quote(KMSConfiguration.KEY_ACL_PREFIX));
+ for (Map.Entry<String, String> keyAcl : allKeyACLS.entrySet()) {
+ String k = keyAcl.getKey();
+ // this should be of type "key.acl.<KEY_NAME>.<OP_TYPE>"
+ int keyNameStarts = KMSConfiguration.KEY_ACL_PREFIX.length();
+ int keyNameEnds = k.lastIndexOf(".");
+ if (keyNameStarts >= keyNameEnds) {
+ LOG.warn("Invalid key name '{}'", k);
+ } else {
+ String aclStr = keyAcl.getValue();
+ String keyName = k.substring(keyNameStarts, keyNameEnds);
+ String keyOp = k.substring(keyNameEnds + 1);
+ KeyOpType aclType = null;
+ try {
+ aclType = KeyOpType.valueOf(keyOp);
+ } catch (IllegalArgumentException e) {
+ LOG.warn("Invalid key Operation '{}'", keyOp);
+ }
+ if (aclType != null) {
+ // On the assumption this will be single threaded.. else we need to
+ // ConcurrentHashMap
+ HashMap<KeyOpType,AccessControlList> aclMap =
+ tempKeyAcls.get(keyName);
+ if (aclMap == null) {
+ aclMap = new HashMap<KeyOpType, AccessControlList>();
+ tempKeyAcls.put(keyName, aclMap);
+ }
+ aclMap.put(aclType, new AccessControlList(aclStr));
+ LOG.info("KEY_NAME '{}' KEY_OP '{}' ACL '{}'",
+ keyName, aclType, aclStr);
+ }
+ }
+ }
+
+ keyAcls = tempKeyAcls;
+ for (KeyOpType keyOp : KeyOpType.values()) {
+ if (!defaultKeyAcls.containsKey(keyOp)) {
+ String confKey = KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + keyOp;
+ String aclStr = conf.get(confKey);
+ if (aclStr != null) {
+ if (aclStr.equals("*")) {
+ LOG.info("Default Key ACL for KEY_OP '{}' is set to '*'", keyOp);
+ }
+ defaultKeyAcls.put(keyOp, new AccessControlList(aclStr));
+ }
+ }
+ }
+ }
+
+ @Override
+ public void run() {
+ try {
+ if (KMSConfiguration.isACLsFileNewer(lastReload)) {
+ setKMSACLs(loadACLs());
+ setKeyACLs(loadACLs());
+ }
+ } catch (Exception ex) {
+ LOG.warn(
+ String.format("Could not reload ACLs file: '%s'", ex.toString()), ex);
+ }
+ }
+
+ public synchronized void startReloader() {
+ if (executorService == null) {
+ executorService = Executors.newScheduledThreadPool(1);
+ executorService.scheduleAtFixedRate(this, RELOADER_SLEEP_MILLIS,
+ RELOADER_SLEEP_MILLIS, TimeUnit.MILLISECONDS);
+ }
+ }
+
+ public synchronized void stopReloader() {
+ if (executorService != null) {
+ executorService.shutdownNow();
+ executorService = null;
+ }
+ }
+
+ private Configuration loadACLs() {
+ LOG.debug("Loading ACLs file");
+ lastReload = System.currentTimeMillis();
+ Configuration conf = KMSConfiguration.getACLsConf();
+ // triggering the resource loading.
+ conf.get(Type.CREATE.getAclConfigKey());
+ return conf;
+ }
+
+ /**
+ * First Check if user is in ACL for the KMS operation, if yes, then
+ * return true if user is not present in any configured blacklist for
+ * the operation
+ * @param type KMS Operation
+ * @param ugi UserGroupInformation of user
+ * @return true is user has access
+ */
+ @Override
+ public boolean hasAccess(Type type, UserGroupInformation ugi) {
+ boolean access = acls.get(type).isUserAllowed(ugi);
+ if (access) {
+ AccessControlList blacklist = blacklistedAcls.get(type);
+ access = (blacklist == null) || !blacklist.isUserInList(ugi);
+ }
+ return access;
+ }
+
+ @Override
+ public void assertAccess(Type aclType,
+ UserGroupInformation ugi, KMSOp operation, String key)
+ throws AccessControlException {
+ if (!KMSWebApp.getACLs().hasAccess(aclType, ugi)) {
+ KMSWebApp.getUnauthorizedCallsMeter().mark();
+ KMSWebApp.getKMSAudit().unauthorized(ugi, operation, key);
+ throw new AuthorizationException(String.format(
+ (key != null) ? UNAUTHORIZED_MSG_WITH_KEY
+ : UNAUTHORIZED_MSG_WITHOUT_KEY,
+ ugi.getShortUserName(), operation, key));
+ }
+ }
+
+ @Override
+ public boolean hasAccessToKey(String keyName, UserGroupInformation ugi,
+ KeyOpType opType) {
+ Map<KeyOpType, AccessControlList> keyAcl = keyAcls.get(keyName);
+ if (keyAcl == null) {
+ // Get KeyAcl map of DEFAULT KEY.
+ keyAcl = defaultKeyAcls;
+ }
+ // If No key acl defined for this key, check to see if
+ // there are key defaults configured for this operation
+ AccessControlList acl = keyAcl.get(opType);
+ if (acl == null) {
+ // If no acl is specified for this operation,
+ // deny access
+ return false;
+ } else {
+ return acl.isUserAllowed(ugi);
+ }
+ }
+
+ @Override
+ public boolean isACLPresent(String keyName, KeyOpType opType) {
+ return (keyAcls.containsKey(keyName) || defaultKeyAcls.containsKey(opType));
+ }
+
+ @Override
+ public void startACLReloader() {
+ this.startReloader();
+ }
+
+ @Override
+ public void stopACLReloader() {
+ this.stopReloader();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLsType.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLsType.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLsType.java
new file mode 100644
index 0000000..0fd3091
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLsType.java
@@ -0,0 +1,17 @@
+package org.apache.hadoop.crypto.key.kms.server;
+
+public class KMSACLsType {
+
+ public enum Type {
+ CREATE, DELETE, ROLLOVER, GET, GET_KEYS, GET_METADATA,
+ SET_KEY_MATERIAL, GENERATE_EEK, DECRYPT_EEK;
+
+ public String getAclConfigKey() {
+ return KMSConfiguration.CONFIG_PREFIX + "acl." + this.toString();
+ }
+
+ public String getBlacklistConfigKey() {
+ return KMSConfiguration.CONFIG_PREFIX + "blacklist." + this.toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
new file mode 100644
index 0000000..7ff76e5
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
@@ -0,0 +1,230 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.hadoop.security.UserGroupInformation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Provides convenience methods for audit logging consistently the different
+ * types of events.
+ */
+public class KMSAudit {
+
+ private static class AuditEvent {
+ private final AtomicLong accessCount = new AtomicLong(-1);
+ private final String keyName;
+ private final String user;
+ private final KMS.KMSOp op;
+ private final String extraMsg;
+ private final long startTime = System.currentTimeMillis();
+
+ private AuditEvent(String keyName, String user, KMS.KMSOp op, String msg) {
+ this.keyName = keyName;
+ this.user = user;
+ this.op = op;
+ this.extraMsg = msg;
+ }
+
+ public String getExtraMsg() {
+ return extraMsg;
+ }
+
+ public AtomicLong getAccessCount() {
+ return accessCount;
+ }
+
+ public String getKeyName() {
+ return keyName;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public KMS.KMSOp getOp() {
+ return op;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+ }
+
+ public static enum OpStatus {
+ OK, UNAUTHORIZED, UNAUTHENTICATED, ERROR;
+ }
+
+ private static Set<KMS.KMSOp> AGGREGATE_OPS_WHITELIST = Sets.newHashSet(
+ KMS.KMSOp.GET_KEY_VERSION, KMS.KMSOp.GET_CURRENT_KEY,
+ KMS.KMSOp.DECRYPT_EEK, KMS.KMSOp.GENERATE_EEK
+ );
+
+ private Cache<String, AuditEvent> cache;
+
+ private ScheduledExecutorService executor;
+
+ public static final String KMS_LOGGER_NAME = "kms-audit";
+
+ private static Logger AUDIT_LOG = LoggerFactory.getLogger(KMS_LOGGER_NAME);
+
+ /**
+ * Create a new KMSAudit.
+ *
+ * @param windowMs Duplicate events within the aggregation window are quashed
+ * to reduce log traffic. A single message for aggregated
+ * events is printed at the end of the window, along with a
+ * count of the number of aggregated events.
+ */
+ KMSAudit(long windowMs) {
+ cache = CacheBuilder.newBuilder()
+ .expireAfterWrite(windowMs, TimeUnit.MILLISECONDS)
+ .removalListener(
+ new RemovalListener<String, AuditEvent>() {
+ @Override
+ public void onRemoval(
+ RemovalNotification<String, AuditEvent> entry) {
+ AuditEvent event = entry.getValue();
+ if (event.getAccessCount().get() > 0) {
+ KMSAudit.this.logEvent(event);
+ event.getAccessCount().set(0);
+ KMSAudit.this.cache.put(entry.getKey(), event);
+ }
+ }
+ }).build();
+ executor = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder()
+ .setDaemon(true).setNameFormat(KMS_LOGGER_NAME + "_thread").build());
+ executor.scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ cache.cleanUp();
+ }
+ }, windowMs / 10, windowMs / 10, TimeUnit.MILLISECONDS);
+ }
+
+ private void logEvent(AuditEvent event) {
+ AUDIT_LOG.info(
+ "OK[op={}, key={}, user={}, accessCount={}, interval={}ms] {}",
+ event.getOp(), event.getKeyName(), event.getUser(),
+ event.getAccessCount().get(),
+ (System.currentTimeMillis() - event.getStartTime()),
+ event.getExtraMsg());
+ }
+
+ private void op(OpStatus opStatus, final KMS.KMSOp op, final String user,
+ final String key, final String extraMsg) {
+ if (!Strings.isNullOrEmpty(user) && !Strings.isNullOrEmpty(key)
+ && (op != null)
+ && AGGREGATE_OPS_WHITELIST.contains(op)) {
+ String cacheKey = createCacheKey(user, key, op);
+ if (opStatus == OpStatus.UNAUTHORIZED) {
+ cache.invalidate(cacheKey);
+ AUDIT_LOG.info("UNAUTHORIZED[op={}, key={}, user={}] {}", op, key, user,
+ extraMsg);
+ } else {
+ try {
+ AuditEvent event = cache.get(cacheKey, new Callable<AuditEvent>() {
+ @Override
+ public AuditEvent call() throws Exception {
+ return new AuditEvent(key, user, op, extraMsg);
+ }
+ });
+ // Log first access (initialized as -1 so
+ // incrementAndGet() == 0 implies first access)
+ if (event.getAccessCount().incrementAndGet() == 0) {
+ event.getAccessCount().incrementAndGet();
+ logEvent(event);
+ }
+ } catch (ExecutionException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ } else {
+ List<String> kvs = new LinkedList<String>();
+ if (op != null) {
+ kvs.add("op=" + op);
+ }
+ if (!Strings.isNullOrEmpty(key)) {
+ kvs.add("key=" + key);
+ }
+ if (!Strings.isNullOrEmpty(user)) {
+ kvs.add("user=" + user);
+ }
+ if (kvs.size() == 0) {
+ AUDIT_LOG.info("{} {}", opStatus.toString(), extraMsg);
+ } else {
+ String join = Joiner.on(", ").join(kvs);
+ AUDIT_LOG.info("{}[{}] {}", opStatus.toString(), join, extraMsg);
+ }
+ }
+ }
+
+ public void ok(UserGroupInformation user, KMS.KMSOp op, String key,
+ String extraMsg) {
+ op(OpStatus.OK, op, user.getShortUserName(), key, extraMsg);
+ }
+
+ public void ok(UserGroupInformation user, KMS.KMSOp op, String extraMsg) {
+ op(OpStatus.OK, op, user.getShortUserName(), null, extraMsg);
+ }
+
+ public void unauthorized(UserGroupInformation user, KMS.KMSOp op, String key) {
+ op(OpStatus.UNAUTHORIZED, op, user.getShortUserName(), key, "");
+ }
+
+ public void error(UserGroupInformation user, String method, String url,
+ String extraMsg) {
+ op(OpStatus.ERROR, null, user.getShortUserName(), null, "Method:'" + method
+ + "' Exception:'" + extraMsg + "'");
+ }
+
+ public void unauthenticated(String remoteHost, String method,
+ String url, String extraMsg) {
+ op(OpStatus.UNAUTHENTICATED, null, null, null, "RemoteHost:"
+ + remoteHost + " Method:" + method
+ + " URL:" + url + " ErrorMsg:'" + extraMsg + "'");
+ }
+
+ private static String createCacheKey(String user, String key, KMS.KMSOp op) {
+ return user + "#" + key + "#" + op;
+ }
+
+ public void shutdown() {
+ executor.shutdownNow();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
new file mode 100644
index 0000000..79652f3
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
+import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler;
+import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler;
+import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticationHandler;
+
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.servlet.http.HttpServletResponseWrapper;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Authentication filter that takes the configuration from the KMS configuration
+ * file.
+ */
+@InterfaceAudience.Private
+public class KMSAuthenticationFilter
+ extends DelegationTokenAuthenticationFilter {
+
+ public static final String CONFIG_PREFIX = KMSConfiguration.CONFIG_PREFIX +
+ "authentication.";
+
+ @Override
+ protected Properties getConfiguration(String configPrefix,
+ FilterConfig filterConfig) {
+ Properties props = new Properties();
+ Configuration conf = KMSWebApp.getConfiguration();
+ for (Map.Entry<String, String> entry : conf) {
+ String name = entry.getKey();
+ if (name.startsWith(CONFIG_PREFIX)) {
+ String value = conf.get(name);
+ name = name.substring(CONFIG_PREFIX.length());
+ props.setProperty(name, value);
+ }
+ }
+ String authType = props.getProperty(AUTH_TYPE);
+ if (authType.equals(PseudoAuthenticationHandler.TYPE)) {
+ props.setProperty(AUTH_TYPE,
+ PseudoDelegationTokenAuthenticationHandler.class.getName());
+ } else if (authType.equals(KerberosAuthenticationHandler.TYPE)) {
+ props.setProperty(AUTH_TYPE,
+ KerberosDelegationTokenAuthenticationHandler.class.getName());
+ }
+ props.setProperty(DelegationTokenAuthenticationHandler.TOKEN_KIND,
+ KMSClientProvider.TOKEN_KIND);
+ return props;
+ }
+
+ protected Configuration getProxyuserConfiguration(FilterConfig filterConfig) {
+ Map<String, String> proxyuserConf = KMSWebApp.getConfiguration().
+ getValByRegex("hadoop\\.kms\\.proxyuser\\.");
+ Configuration conf = new Configuration(false);
+ for (Map.Entry<String, String> entry : proxyuserConf.entrySet()) {
+ conf.set(entry.getKey().substring("hadoop.kms.".length()),
+ entry.getValue());
+ }
+ return conf;
+ }
+
+ private static class KMSResponse extends HttpServletResponseWrapper {
+ public int statusCode;
+ public String msg;
+
+ public KMSResponse(ServletResponse response) {
+ super((HttpServletResponse)response);
+ }
+
+ @Override
+ public void setStatus(int sc) {
+ statusCode = sc;
+ super.setStatus(sc);
+ }
+
+ @Override
+ public void sendError(int sc, String msg) throws IOException {
+ statusCode = sc;
+ this.msg = msg;
+ super.sendError(sc, msg);
+ }
+
+ @Override
+ public void sendError(int sc) throws IOException {
+ statusCode = sc;
+ super.sendError(sc);
+ }
+
+ @Override
+ public void setStatus(int sc, String sm) {
+ statusCode = sc;
+ msg = sm;
+ super.setStatus(sc, sm);
+ }
+ }
+
+ @Override
+ public void doFilter(ServletRequest request, ServletResponse response,
+ FilterChain filterChain) throws IOException, ServletException {
+ KMSResponse kmsResponse = new KMSResponse(response);
+ super.doFilter(request, kmsResponse, filterChain);
+
+ if (kmsResponse.statusCode != HttpServletResponse.SC_OK &&
+ kmsResponse.statusCode != HttpServletResponse.SC_CREATED &&
+ kmsResponse.statusCode != HttpServletResponse.SC_UNAUTHORIZED) {
+ KMSWebApp.getInvalidCallsMeter().mark();
+ }
+
+ // HttpServletResponse.SC_UNAUTHORIZED is because the request does not
+ // belong to an authenticated user.
+ if (kmsResponse.statusCode == HttpServletResponse.SC_UNAUTHORIZED) {
+ KMSWebApp.getUnauthenticatedCallsMeter().mark();
+ String method = ((HttpServletRequest) request).getMethod();
+ StringBuffer requestURL = ((HttpServletRequest) request).getRequestURL();
+ String queryString = ((HttpServletRequest) request).getQueryString();
+ if (queryString != null) {
+ requestURL.append("?").append(queryString);
+ }
+
+ KMSWebApp.getKMSAudit().unauthenticated(
+ request.getRemoteHost(), method, requestURL.toString(),
+ kmsResponse.msg);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
new file mode 100644
index 0000000..d6c77ea
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import java.io.File;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+/**
+ * Utility class to load KMS configuration files.
+ */
+@InterfaceAudience.Private
+public class KMSConfiguration {
+
+ public static final String KMS_CONFIG_DIR = "kms.config.dir";
+ public static final String KMS_SITE_XML = "kms-site.xml";
+ public static final String KMS_ACLS_XML = "kms-acls.xml";
+
+ public static final String CONFIG_PREFIX = "hadoop.kms.";
+
+ public static final String KEY_ACL_PREFIX = "key.acl.";
+ public static final String DEFAULT_KEY_ACL_PREFIX = "default.key.acl.";
+
+ // Property to set the backing KeyProvider
+ public static final String KEY_PROVIDER_URI = CONFIG_PREFIX +
+ "key.provider.uri";
+
+ // Property to Enable/Disable Caching
+ public static final String KEY_CACHE_ENABLE = CONFIG_PREFIX +
+ "cache.enable";
+ // Timeout for the Key and Metadata Cache
+ public static final String KEY_CACHE_TIMEOUT_KEY = CONFIG_PREFIX +
+ "cache.timeout.ms";
+ // TImeout for the Current Key cache
+ public static final String CURR_KEY_CACHE_TIMEOUT_KEY = CONFIG_PREFIX +
+ "current.key.cache.timeout.ms";
+ // Delay for Audit logs that need aggregation
+ public static final String KMS_AUDIT_AGGREGATION_WINDOW = CONFIG_PREFIX +
+ "audit.aggregation.window.ms";
+
+ //for authorizer
+ public static final String KMS_SECURITY_AUTHORIZER = CONFIG_PREFIX + "security.authorization.manager";
+
+ public static final boolean KEY_CACHE_ENABLE_DEFAULT = true;
+ // 10 mins
+ public static final long KEY_CACHE_TIMEOUT_DEFAULT = 10 * 60 * 1000;
+ // 30 secs
+ public static final long CURR_KEY_CACHE_TIMEOUT_DEFAULT = 30 * 1000;
+ // 10 secs
+ public static final long KMS_AUDIT_AGGREGATION_WINDOW_DEFAULT = 10000;
+
+ // Property to Enable/Disable per Key authorization
+ public static final String KEY_AUTHORIZATION_ENABLE = CONFIG_PREFIX +
+ "key.authorization.enable";
+
+ public static final boolean KEY_AUTHORIZATION_ENABLE_DEFAULT = true;
+
+ static Configuration getConfiguration(boolean loadHadoopDefaults,
+ String ... resources) {
+ Configuration conf = new Configuration(loadHadoopDefaults);
+ String confDir = System.getProperty(KMS_CONFIG_DIR);
+ if (confDir != null) {
+ try {
+ Path confPath = new Path(confDir);
+ if (!confPath.isUriPathAbsolute()) {
+ throw new RuntimeException("System property '" + KMS_CONFIG_DIR +
+ "' must be an absolute path: " + confDir);
+ }
+ for (String resource : resources) {
+ conf.addResource(new URL("file://" + new Path(confDir, resource).toUri()));
+ }
+ } catch (MalformedURLException ex) {
+ throw new RuntimeException(ex);
+ }
+ } else {
+ for (String resource : resources) {
+ conf.addResource(resource);
+ }
+ }
+ return conf;
+ }
+
+ public static Configuration getKMSConf() {
+ return getConfiguration(true, "core-site.xml", KMS_SITE_XML);
+ }
+
+ public static Configuration getACLsConf() {
+ return getConfiguration(false, KMS_ACLS_XML);
+ }
+
+ public static boolean isACLsFileNewer(long time) {
+ boolean newer = false;
+ String confDir = System.getProperty(KMS_CONFIG_DIR);
+ if (confDir != null) {
+ Path confPath = new Path(confDir);
+ if (!confPath.isUriPathAbsolute()) {
+ throw new RuntimeException("System property '" + KMS_CONFIG_DIR +
+ "' must be an absolute path: " + confDir);
+ }
+ File f = new File(confDir, KMS_ACLS_XML);
+ // at least 100ms newer than time, we do this to ensure the file
+ // has been properly closed/flushed
+ newer = f.lastModified() - time > 100;
+ }
+ return newer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
new file mode 100644
index 0000000..5cb0885
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import com.sun.jersey.api.container.ContainerException;
+
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.util.HttpExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+import javax.ws.rs.ext.Provider;
+
+import java.io.IOException;
+
+/**
+ * Jersey provider that converts KMS exceptions into detailed HTTP errors.
+ */
+@Provider
+@InterfaceAudience.Private
+public class KMSExceptionsProvider implements ExceptionMapper<Exception> {
+ private static Logger LOG =
+ LoggerFactory.getLogger(KMSExceptionsProvider.class);
+
+ private static final String ENTER = System.getProperty("line.separator");
+
+ protected Response createResponse(Response.Status status, Throwable ex) {
+ return HttpExceptionUtils.createJerseyExceptionResponse(status, ex);
+ }
+
+ protected String getOneLineMessage(Throwable exception) {
+ String message = exception.getMessage();
+ if (message != null) {
+ int i = message.indexOf(ENTER);
+ if (i > -1) {
+ message = message.substring(0, i);
+ }
+ }
+ return message;
+ }
+
+ /**
+ * Maps different exceptions thrown by KMS to HTTP status codes.
+ */
+ @Override
+ public Response toResponse(Exception exception) {
+ Response.Status status;
+ boolean doAudit = true;
+ Throwable throwable = exception;
+ if (exception instanceof ContainerException) {
+ throwable = exception.getCause();
+ }
+ if (throwable instanceof SecurityException) {
+ status = Response.Status.FORBIDDEN;
+ } else if (throwable instanceof AuthenticationException) {
+ status = Response.Status.FORBIDDEN;
+ // we don't audit here because we did it already when checking access
+ doAudit = false;
+ } else if (throwable instanceof AuthorizationException) {
+ status = Response.Status.FORBIDDEN;
+ // we don't audit here because we did it already when checking access
+ doAudit = false;
+ } else if (throwable instanceof AccessControlException) {
+ status = Response.Status.FORBIDDEN;
+ } else if (exception instanceof IOException) {
+ status = Response.Status.INTERNAL_SERVER_ERROR;
+ } else if (exception instanceof UnsupportedOperationException) {
+ status = Response.Status.BAD_REQUEST;
+ } else if (exception instanceof IllegalArgumentException) {
+ status = Response.Status.BAD_REQUEST;
+ } else {
+ status = Response.Status.INTERNAL_SERVER_ERROR;
+ }
+ if (doAudit) {
+ KMSWebApp.getKMSAudit().error(KMSMDCFilter.getUgi(),
+ KMSMDCFilter.getMethod(),
+ KMSMDCFilter.getURL(), getOneLineMessage(exception));
+ }
+ return createResponse(status, throwable);
+ }
+
+ protected void log(Response.Status status, Throwable ex) {
+ UserGroupInformation ugi = KMSMDCFilter.getUgi();
+ String method = KMSMDCFilter.getMethod();
+ String url = KMSMDCFilter.getURL();
+ String msg = getOneLineMessage(ex);
+ LOG.warn("User:'{}' Method:{} URL:{} Response:{}-{}", ugi, method, url,
+ status, msg, ex);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java
new file mode 100644
index 0000000..6918015
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.jmx.JMXJsonServlet;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import java.io.IOException;
+
+@InterfaceAudience.Private
+public class KMSJMXServlet extends JMXJsonServlet {
+
+ @Override
+ protected boolean isInstrumentationAccessAllowed(HttpServletRequest request,
+ HttpServletResponse response) throws IOException {
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java
new file mode 100644
index 0000000..d3e0064
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.ext.MessageBodyReader;
+import javax.ws.rs.ext.Provider;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Type;
+import java.util.Map;
+
+@Provider
+@Consumes(MediaType.APPLICATION_JSON)
+@InterfaceAudience.Private
+public class KMSJSONReader implements MessageBodyReader<Map> {
+
+ @Override
+ public boolean isReadable(Class<?> type, Type genericType,
+ Annotation[] annotations, MediaType mediaType) {
+ return type.isAssignableFrom(Map.class);
+ }
+
+ @Override
+ public Map readFrom(Class<Map> type, Type genericType,
+ Annotation[] annotations, MediaType mediaType,
+ MultivaluedMap<String, String> httpHeaders, InputStream entityStream)
+ throws IOException, WebApplicationException {
+ ObjectMapper mapper = new ObjectMapper();
+ return mapper.readValue(entityStream, type);
+ }
+}
[4/4] incubator-ranger git commit: (RANGER-247)Development of Ranger
Key Storage Provider
Posted by ve...@apache.org.
(RANGER-247)Development of Ranger Key Storage Provider
Signed-off-by: Velmurugan Periasamy <ve...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/bb0bdced
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/bb0bdced
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/bb0bdced
Branch: refs/heads/master
Commit: bb0bdcedeb134f5aadeef17be11f4df7e8150b71
Parents: ce139e0
Author: Gautam Borad <gb...@gmail.com>
Authored: Tue Mar 24 12:12:19 2015 +0530
Committer: Velmurugan Periasamy <ve...@apache.org>
Committed: Tue Mar 24 13:26:30 2015 -0400
----------------------------------------------------------------------
kms/config/kms-webapp/dbks-site.xml | 120 ++++
kms/config/kms-webapp/kms-site.xml | 9 +-
kms/config/webserver/kms_webserver.properties | 20 +-
kms/dev-support/findbugsExcludeFile.xml | 41 ++
kms/pom.xml | 413 +++++++++++++-
kms/scripts/db/mysql/kms_core_db.sql | 30 +
kms/scripts/db_setup.py | 252 +++++++++
kms/scripts/ranger-kms | 2 +-
kms/scripts/ranger-kms-services.sh | 2 +-
kms/scripts/setup.sh | 172 ++++++
.../crypto/key/EncryptedPrivateKeyInfo.java | 111 ++++
.../apache/hadoop/crypto/key/KeyProtector.java | 541 +++++++++++++++++++
.../apache/hadoop/crypto/key/RangerKMSDB.java | 85 +++
.../hadoop/crypto/key/RangerKeyStore.java | 537 ++++++++++++++++++
.../crypto/key/RangerKeyStoreProvider.java | 341 ++++++++++++
.../hadoop/crypto/key/RangerMasterKey.java | 210 +++++++
...rKeyGeneratorKeyProviderCryptoExtension.java | 171 ++++++
.../hadoop/crypto/key/kms/server/KMS.java | 482 +++++++++++++++++
.../hadoop/crypto/key/kms/server/KMSACLs.java | 253 +++++++++
.../crypto/key/kms/server/KMSACLsType.java | 17 +
.../hadoop/crypto/key/kms/server/KMSAudit.java | 230 ++++++++
.../key/kms/server/KMSAuthenticationFilter.java | 154 ++++++
.../crypto/key/kms/server/KMSConfiguration.java | 126 +++++
.../key/kms/server/KMSExceptionsProvider.java | 113 ++++
.../crypto/key/kms/server/KMSJMXServlet.java | 36 ++
.../crypto/key/kms/server/KMSJSONReader.java | 54 ++
.../crypto/key/kms/server/KMSJSONWriter.java | 70 +++
.../crypto/key/kms/server/KMSMDCFilter.java | 93 ++++
.../key/kms/server/KMSServerJSONUtils.java | 102 ++++
.../hadoop/crypto/key/kms/server/KMSWebApp.java | 307 +++++++++++
.../kms/server/KeyAuthorizationKeyProvider.java | 299 ++++++++++
.../java/org/apache/ranger/entity/XXDBBase.java | 216 ++++++++
.../apache/ranger/entity/XXRangerKeyStore.java | 121 +++++
.../apache/ranger/entity/XXRangerMasterKey.java | 67 +++
.../apache/ranger/kms/biz/RangerKMSStartUp.java | 35 ++
.../java/org/apache/ranger/kms/dao/BaseDao.java | 261 +++++++++
.../org/apache/ranger/kms/dao/DaoManager.java | 35 ++
.../apache/ranger/kms/dao/DaoManagerBase.java | 31 ++
.../org/apache/ranger/kms/dao/RangerKMSDao.java | 18 +
.../ranger/kms/dao/RangerMasterKeyDao.java | 10 +
.../META-INF/kms_jpa_named_queries.xml | 32 ++
kms/src/main/resources/META-INF/persistence.xml | 28 +
....apache.hadoop.crypto.key.KeyProviderFactory | 18 +
kms/src/main/resources/WEB-INF/web.xml | 62 +++
.../main/resources/log4j-kmsaudit.properties | 25 +
kms/src/main/resources/log4j.properties | 31 ++
.../main/resources/mini-kms-acls-default.xml | 135 +++++
kms/src/main/webapp/WEB-INF/web.xml | 62 +++
.../hadoop/crypto/key/kms/server/MiniKMS.java | 238 ++++++++
.../crypto/key/kms/server/TestKMSACLs.java | 52 ++
.../crypto/key/kms/server/TestKMSAudit.java | 135 +++++
.../server/TestKeyAuthorizationKeyProvider.java | 271 ++++++++++
pom.xml | 22 +
src/main/assembly/kms.xml | 167 ++++--
54 files changed, 7418 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/config/kms-webapp/dbks-site.xml
----------------------------------------------------------------------
diff --git a/kms/config/kms-webapp/dbks-site.xml b/kms/config/kms-webapp/dbks-site.xml
new file mode 100644
index 0000000..a428c6f
--- /dev/null
+++ b/kms/config/kms-webapp/dbks-site.xml
@@ -0,0 +1,120 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+
+ <!-- Encryption key Password -->
+
+ <property>
+ <name>ranger.db.encrypt.key.password</name>
+ <value>Str0ngPassw0rd</value>
+ <description>
+ Password used for encrypting Master Key
+ </description>
+ </property>
+
+ <!-- db Details -->
+
+ <property>
+ <name>ranger.db.ks.database.flavor</name>
+ <value>MYSQL</value>
+ <description>
+ ULR for Database
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.db.ks.core.file</name>
+ <value>db/mysql/kms_core_db.sql</value>
+ <description>
+ ULR for Database
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.db.ks.javax.persistence.jdbc.url</name>
+ <value>jdbc:log4jdbc:mysql://localhost:3306/rangerkms</value>
+ <description>
+ ULR for Database
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.db.root.user</name>
+ <value>root</value>
+ <description>
+ Database root user name used for operation
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.db.root.password</name>
+ <value>root</value>
+ <description>
+ Database root user name used for operation
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.db.host</name>
+ <value>localhost</value>
+ <description>
+ Database root user name used for operation
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.db.ks.name</name>
+ <value>rangerkms</value>
+ <description>
+ Driver used for database
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.db.ks.javax.persistence.jdbc.user</name>
+ <value>kmsadmin</value>
+ <description>
+ Database user name used for operation
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.db.ks.javax.persistence.jdbc.password</name>
+ <value>kmsadmin</value>
+ <description>
+ Database user's password
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.db.ks.javax.persistence.jdbc.dialect</name>
+ <value>org.eclipse.persistence.platform.database.MySQLPlatform</value>
+ <description>
+ Dialect used for database
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.db.ks.javax.persistence.jdbc.driver</name>
+ <value>net.sf.log4jdbc.DriverSpy</value>
+ <description>
+ Driver used for database
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.db.ks.sql.connector.jar</name>
+ <value>/usr/share/java/mysql-connector-java.jar</value>
+ <description>
+ Driver used for database
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.db.ks.java.bin</name>
+ <value>java</value>
+ <description>
+ ULR for Database
+ </description>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/config/kms-webapp/kms-site.xml
----------------------------------------------------------------------
diff --git a/kms/config/kms-webapp/kms-site.xml b/kms/config/kms-webapp/kms-site.xml
index a810ca4..06e7ec4 100644
--- a/kms/config/kms-webapp/kms-site.xml
+++ b/kms/config/kms-webapp/kms-site.xml
@@ -18,7 +18,7 @@
<property>
<name>hadoop.kms.key.provider.uri</name>
- <value>jceks://file@/${user.home}/kms.keystore</value>
+ <value>dbks://http@localhost:9292/kms</value>
<description>
URI of the backing KeyProvider for the KMS.
</description>
@@ -31,7 +31,7 @@
If using the JavaKeyStoreProvider, the password for the keystore file.
</description>
</property>
-
+
<!-- KMS Cache -->
<property>
@@ -169,5 +169,10 @@
The Kerberos service principal used to connect to Zookeeper.
</description>
</property>
+
+ <property>
+ <name>hadoop.kms.security.authorization.manager</name>
+ <value></value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/config/webserver/kms_webserver.properties
----------------------------------------------------------------------
diff --git a/kms/config/webserver/kms_webserver.properties b/kms/config/webserver/kms_webserver.properties
index e96b9fc..1d227cf 100644
--- a/kms/config/webserver/kms_webserver.properties
+++ b/kms/config/webserver/kms_webserver.properties
@@ -44,4 +44,22 @@ accesslog.pattern=%h %l %u %t "%r" %s %b "%{Referer}i" "%{User-Agent}i"
# Web Application root folder
#
xa.webapp.contextName=/kms
-xa.webapp.dir=./webapp/root/hadoop-kms-2.6.0.war
+xa.webapp.dir=./webapp
+
+#
+# Values required for setup script
+#
+realScriptPath=`readlink -f $0`
+realScriptDir=`dirname $realScriptPath`
+RANGER_KMS_DIR=`(cd $realScriptDir/..; pwd)`
+RANGER_KMS_EWS_DIR=${RANGER_KMS_DIR}/ews
+
+app_home=${RANGER_KMS_EWS_DIR}/webapp
+SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar
+PYTHON_COMMAND_INVOKER="python"
+JAVA_VERSION_REQUIRED="1.7"
+#
+# Log file path
+#
+LOGFILE=${RANGER_KMS_DIR}/logfile
+LOGFILES="$LOGFILE"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/kms/dev-support/findbugsExcludeFile.xml b/kms/dev-support/findbugsExcludeFile.xml
new file mode 100644
index 0000000..bc92ed7
--- /dev/null
+++ b/kms/dev-support/findbugsExcludeFile.xml
@@ -0,0 +1,41 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FindBugsFilter>
+ <!--
+ Findbug is complaining about System.out being NULL
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.crypto.key.kms.server.KMSWebApp"/>
+ <Bug pattern="NP_ALWAYS_NULL"/>
+ </Match>
+ <!--
+ KMSWebApp is a webapp singleton managed by the servlet container via
+ ServletContextListener.
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.crypto.key.kms.server.KMSWebApp"/>
+ <Bug pattern="ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD"/>
+ </Match>
+ <!--
+ KMSWebApp does an exit to kill the servlet container if the initialization
+ fails.
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.crypto.key.kms.server.KMSWebApp"/>
+ <Bug pattern="DM_EXIT"/>
+ </Match>
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/pom.xml
----------------------------------------------------------------------
diff --git a/kms/pom.xml b/kms/pom.xml
index 4c0f73b..5ce8b11 100644
--- a/kms/pom.xml
+++ b/kms/pom.xml
@@ -26,14 +26,6 @@
<artifactId>ranger</artifactId>
<version>0.5.0</version>
</parent>
- <build>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- <filtering>true</filtering>
- </resource>
- </resources>
- </build>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -46,5 +38,410 @@
<artifactId>hadoop-common</artifactId>
<version>${hadoop-common.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.eclipse.persistence</groupId>
+ <artifactId>eclipselink</artifactId>
+ <version>${eclipse.jpa.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.persistence</groupId>
+ <artifactId>javax.persistence</artifactId>
+ <version>${javax.persistence.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-aop</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-asm</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-beans</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context-support</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-core</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-expression</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-jdbc</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-orm</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-tx</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-web</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.xml.security</groupId>
+ <artifactId>xml-security-impl</artifactId>
+ <version>1.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.googlecode.log4jdbc</groupId>
+ <artifactId>log4jdbc</artifactId>
+ <version>${googlecode.log4jdbc.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>${mysql-connector-java.version}</version>
+ </dependency>
+
+ <!-- Hadoop KMS dependency -->
+ <!-- <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minikdc</artifactId>
+ <version>${hadoop.minikdc.version}</version>
+ <scope>test</scope>
+ </dependency> -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${mockito.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <version>${hadoop-auth.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ <version>${jersey-server.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ <version>${jersey-server.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ <version>${servlet.api.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ <version>${mortbay.jetty.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <version>${log4j.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j-api.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>${slf4j-api.version}</version>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jul-to-slf4j</artifactId>
+ <version>${slf4j-api.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ <version>${mortbay.jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics.core.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
+ <version>${curator.test.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-bundle</artifactId>
+ <version>${sun-jersey-bundle.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>asm</groupId>
+ <artifactId>asm-all</artifactId>
+ <version>${asm.all.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpcomponents.httpclient.version}</version>
+ </dependency>
+
+ <!-- change for version variable -->
+ <dependency>
+ <groupId>javax.activation</groupId>
+ <artifactId>activation</artifactId>
+ <version>1.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.directory.server</groupId>
+ <artifactId>apacheds-i18n</artifactId>
+ <version>2.0.0-M15</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.directory.server</groupId>
+ <artifactId>apacheds-kerberos-codec</artifactId>
+ <version>2.0.0-M15</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.directory.api</groupId>
+ <artifactId>api-asn1-api</artifactId>
+ <version>1.0.0-M20</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.directory.api</groupId>
+ <artifactId>api-i18n</artifactId>
+ <version>1.0.0-M20</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.directory.api</groupId>
+ <artifactId>api-util</artifactId>
+ <version>1.0.0-M20</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <version>1.7.4</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ <version>1.7.0</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils-core</artifactId>
+ <version>1.8.0</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>1.2</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>1.4</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ <version>1.4.1</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ <version>1.6</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-digester</groupId>
+ <artifactId>commons-digester</artifactId>
+ <version>1.8</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.4</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>3.1.1</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-net</groupId>
+ <artifactId>commons-net</artifactId>
+ <version>3.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ <version>2.6.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.2.4</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ <version>${hadoop-auth.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.htrace</groupId>
+ <artifactId>htrace-core</artifactId>
+ <version>3.0.4</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>4.2.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-jaxrs</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-xc</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ <version>${jaxb-api.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.xml.bind</groupId>
+ <artifactId>jaxb-impl</artifactId>
+ <version>${jaxb-impl.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ <version>${jersey-server.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jettison</groupId>
+ <artifactId>jettison</artifactId>
+ <version>${jettison.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ <version>${jline.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ <version>${jsch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ <version>${jsr305.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ <version>${netty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.thoughtworks.paranamer</groupId>
+ <artifactId>paranamer</artifactId>
+ <version>${paranamer.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>${protobuf-java.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ <version>${snappy-java.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>xmlenc</groupId>
+ <artifactId>xmlenc</artifactId>
+ <version>${xmlenc.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.tukaani</groupId>
+ <artifactId>xz</artifactId>
+ <version>${xz.version}</version>
+ </dependency>
</dependencies>
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-war-plugin</artifactId>
+ <version>2.4</version>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </build>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/scripts/db/mysql/kms_core_db.sql
----------------------------------------------------------------------
diff --git a/kms/scripts/db/mysql/kms_core_db.sql b/kms/scripts/db/mysql/kms_core_db.sql
new file mode 100644
index 0000000..b95a75b
--- /dev/null
+++ b/kms/scripts/db/mysql/kms_core_db.sql
@@ -0,0 +1,30 @@
+DROP TABLE IF EXISTS `ranger_masterkey`;
+CREATE TABLE `ranger_masterkey` (
+`id` bigint( 20 ) NOT NULL AUTO_INCREMENT ,
+`create_time` datetime DEFAULT NULL ,
+`update_time` datetime DEFAULT NULL ,
+`added_by_id` bigint( 20 ) DEFAULT NULL ,
+`upd_by_id` bigint( 20 ) DEFAULT NULL ,
+`cipher` varchar( 255 ) DEFAULT NULL ,
+`bitlength` int DEFAULT NULL ,
+`masterkey` varchar(2048),
+PRIMARY KEY ( `id` )
+)ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+DROP TABLE IF EXISTS `ranger_keystore`;
+CREATE TABLE `ranger_keystore` (
+ `id` bigint(20) NOT NULL AUTO_INCREMENT,
+ `create_time` datetime DEFAULT NULL,
+ `update_time` datetime DEFAULT NULL,
+ `added_by_id` bigint(20) DEFAULT NULL,
+ `upd_by_id` bigint(20) DEFAULT NULL,
+ `kms_alias` varchar(255) NOT NULL,
+ `kms_createdDate` bigint(20) DEFAULT NULL,
+ `kms_cipher` varchar(255) DEFAULT NULL,
+ `kms_bitLength` bigint(20) DEFAULT NULL,
+ `kms_description` varchar(512) DEFAULT NULL,
+ `kms_version` bigint(20) DEFAULT NULL,
+ `kms_attributes` varchar(1024) DEFAULT NULL,
+ `kms_encoded`varchar(2048),
+ PRIMARY KEY (`id`)
+)ENGINE=InnoDB DEFAULT CHARSET=latin1;
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/scripts/db_setup.py
----------------------------------------------------------------------
diff --git a/kms/scripts/db_setup.py b/kms/scripts/db_setup.py
new file mode 100644
index 0000000..301bdcc
--- /dev/null
+++ b/kms/scripts/db_setup.py
@@ -0,0 +1,252 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+import os
+import re
+import sys
+import errno
+import shlex
+import logging
+import subprocess
+import fileinput
+from os.path import basename
+from datetime import date
+from xml.etree import ElementTree as ET
+globalDict = {}
+
+def check_output(query):
+ p = subprocess.Popen(query, stdout=subprocess.PIPE)
+ output = p.communicate ()[0]
+ return output
+
+def log(msg,type):
+ if type == 'info':
+ logging.info(" %s",msg)
+ if type == 'debug':
+ logging.debug(" %s",msg)
+ if type == 'warning':
+ logging.warning(" %s",msg)
+ if type == 'exception':
+ logging.exception(" %s",msg)
+
+def populate_global_dict():
+ global globalDict
+ os.chdir("..")
+ read_config_file = open(os.path.join(os.getcwd(),'ews/webapp/config/dbks-site.xml'))
+ dbks_site_properties = import_properties_from_xml(read_config_file, globalDict)
+
+class BaseDB(object):
+
+ def init_logfiles(self):
+ FORMAT = '%(asctime)-15s %(message)s'
+ logging.basicConfig(format=FORMAT, level=logging.DEBUG)
+
+ def create_rangerdb_user(self, root_user, db_user, db_password, db_root_password):
+ log("---------- Creating User ----------", "info")
+
+ def check_table(self, db_name, root_user, db_root_password, TABLE_NAME):
+ log("---------- Verifying table ----------", "info")
+ def import_file_to_db(self, root_user, db_name, db_user, db_password, db_root_password, file_name):
+ log("---------- Importing db schema ----------", "info")
+
+class MysqlConf(BaseDB):
+ # Constructor
+ def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN):
+ self.host = host
+ self.SQL_CONNECTOR_JAR = SQL_CONNECTOR_JAR
+ self.JAVA_BIN = JAVA_BIN
+ BaseDB.init_logfiles(self)
+
+ def get_jisql_cmd(self, user, password ,db_name):
+ #TODO: User array for forming command
+ jisql_cmd = "%s -cp %s:jisql/lib/* org.apache.util.sql.Jisql -driver mysqlconj -cstring jdbc:mysql://%s/%s -u %s -p %s -noheader -trim -c \;" %(self.JAVA_BIN,self.SQL_CONNECTOR_JAR,self.host,db_name,user,password)
+ return jisql_cmd
+
+
+ def create_rangerdb_user(self, root_user, db_user, db_password, db_root_password):
+ hosts_arr =["%", "localhost"]
+ for host in hosts_arr:
+ get_cmd = self.get_jisql_cmd(root_user, db_root_password ,'mysql')
+ query = get_cmd + " -query \"select user from mysql.user where user='%s' and host='%s';\"" %(db_user,host)
+ output = check_output(shlex.split(query))
+ if output.strip(db_user + " |"):
+ log( "\nMYSQL User: " + db_user + " already exists!", "debug")
+ else:
+ log("User does not exists", "info")
+ if db_password == "":
+ log ("Creating MySQL user: "+ db_user +" with DB password blank\n", "info")
+ query = get_cmd + " -query \"create user '%s'@'%s';\"" %(db_user, host)
+ ret = subprocess.check_call(shlex.split(query))
+ if ret == 0:
+ query = get_cmd + " -query \"select user from mysql.user where user='%s' and host='%s';\"" %(db_user,host)
+ output = check_output(shlex.split(query))
+ if output.strip(db_user + " |"):
+ log("Mysql user " + db_user +" created","info")
+ else:
+ log("Creating Mysql user " + db_user +" Failed","info")
+ sys.exit(1)
+ else:
+ log ("Creating MySQL user: "+ db_user +" with DB password\n", "info")
+ query = get_cmd + " -query \"create user '%s'@'%s' identified by '%s';\"" %(db_user, host, db_password)
+ ret = subprocess.check_call(shlex.split(query))
+ if ret == 0:
+ log("Mysql user " + db_user +" created","info")
+ else:
+ log("Creating Mysql user " + db_user +" Failed","info")
+ sys.exit(1)
+
+ def verify_db(self, root_user, db_root_password, db_name):
+ log("\nVerifying Database: " + db_name + "\n", "debug")
+ get_cmd = self.get_jisql_cmd(root_user, db_root_password, 'mysql')
+ query = get_cmd + " -query \"show databases like '%s';\"" %(db_name)
+ output = check_output(shlex.split(query))
+ if output.strip(db_name + " |"):
+ return True
+ else:
+ return False
+
+
+ def import_file_to_db(self, root_user, db_name, db_user, db_password, db_root_password, file_name):
+ log ("\nImporting db schema to Database: " + db_name,"debug");
+ if self.verify_db(root_user, db_root_password, db_name):
+ log("\nDatabase: "+db_name + " already exists. Ignoring import_db\n","info")
+ else:
+ log("\nDatabase does not exist. Creating database : " + db_name,"info")
+ get_cmd = self.get_jisql_cmd(root_user, db_root_password, 'mysql')
+ query = get_cmd + " -query \"create database %s;\"" %(db_name)
+ ret = subprocess.check_call(shlex.split(query))
+ if ret != 0:
+ log("\nDatabase creation failed!!\n","info")
+ sys.exit(1)
+ else:
+ if self.verify_db(root_user, db_root_password, db_name):
+ log("Creating database: " + db_name + " succeeded", "info")
+ self.import_db_file(db_name, root_user , db_user, db_password, db_root_password, file_name)
+ else:
+ log("\nDatabase creation failed!!\n","info")
+ sys.exit(1)
+
+
+ def grant_xa_db_user(self, root_user, db_name, db_user, db_password, db_root_password , is_revoke):
+ hosts_arr =["%", "localhost"]
+ if is_revoke:
+ for host in hosts_arr:
+ get_cmd = self.get_jisql_cmd(root_user, db_root_password, 'mysql')
+ query = get_cmd + " -query \"REVOKE ALL PRIVILEGES,GRANT OPTION FROM '%s'@'%s';\"" %(db_user, host)
+ ret = subprocess.check_call(shlex.split(query))
+ if ret == 0:
+ query = get_cmd + " -query \"FLUSH PRIVILEGES;\""
+ ret = subprocess.check_call(shlex.split(query))
+ if ret != 0:
+ sys.exit(1)
+ else:
+ sys.exit(1)
+
+ for host in hosts_arr:
+ log ("---------------GRANTING PRIVILEGES TO user '"+db_user+"'@'"+host+"' on db '"+db_name+"'-------------" , "info")
+ get_cmd = self.get_jisql_cmd(root_user, db_root_password, 'mysql')
+ query = get_cmd + " -query \"grant all privileges on %s.* to '%s'@'%s' with grant option;\"" %(db_name,db_user, host)
+ ret = subprocess.check_call(shlex.split(query))
+ if ret == 0:
+ log ("---------------FLUSH PRIVILEGES -------------" , "info")
+ query = get_cmd + " -query \"FLUSH PRIVILEGES;\""
+ ret = subprocess.check_call(shlex.split(query))
+ if ret == 0:
+ log("Privileges granted to '" + db_user + "' on '"+db_name+"'\n", "info")
+ else:
+ log("Granting privileges to '" +db_user+"' FAILED on '"+db_name+"'\n", "info")
+ sys.exit(1)
+ else:
+ log("\nGranting privileges to '" +db_user+"' FAILED on '"+db_name+"'\n", "info")
+ sys.exit(1)
+
+ def import_db_file(self, db_name, root_user, db_user, db_password, db_root_password, file_name):
+ name = basename(file_name)
+ if os.path.isfile(file_name):
+ log("Importing db schema to database : " + db_name + " from file: " + name,"info")
+ get_cmd = self.get_jisql_cmd(root_user, db_root_password, db_name)
+ query = get_cmd + " -input %s" %file_name
+ ret = subprocess.check_call(shlex.split(query))
+ if ret == 0:
+ log(name + " DB schema imported successfully\n","info")
+ else:
+ log(name + " DB Schema import failed!\n","info")
+ sys.exit(1)
+ else:
+ log("\nDB Schema file " + name+ " not found\n","exception")
+ sys.exit(1)
+
+ def check_table(self, db_name, root_user, db_root_password, TABLE_NAME):
+ if self.verify_db(root_user, db_root_password, db_name):
+ log("Verifying table " + TABLE_NAME +" in database " + db_name, "debug")
+
+ get_cmd = self.get_jisql_cmd(root_user, db_root_password, db_name)
+ query = get_cmd + " -query \"show tables like '%s';\"" %(TABLE_NAME)
+ output = check_output(shlex.split(query))
+ if output.strip(TABLE_NAME + " |"):
+ log("Table " + TABLE_NAME +" already exists in database " + db_name + "\n","info")
+ return True
+ else:
+ log("Table " + TABLE_NAME +" does not exist in database " + db_name + "\n","info")
+ return False
+ else:
+ log("Database " + db_name +" does not exist\n","info")
+ return False
+
+def import_properties_from_xml(xml_path, properties_from_xml=None):
+ xml = ET.parse(xml_path)
+ root = xml.getroot()
+ if properties_from_xml is None:
+ properties_from_xml = dict()
+ for child in root.findall('property'):
+ name = child.find("name").text.strip()
+ value = child.find("value").text.strip() if child.find("value").text is not None else ""
+ properties_from_xml[name] = value
+ return properties_from_xml
+
+
+def main():
+ populate_global_dict()
+ JAVA_BIN=globalDict['ranger.db.ks.java.bin']
+ XA_DB_FLAVOR=globalDict['ranger.db.ks.database.flavor']
+ XA_DB_FLAVOR.upper()
+
+ xa_db_host = globalDict['ranger.db.host']
+
+ mysql_core_file = globalDict['ranger.db.ks.core.file']
+
+ db_name = globalDict['ranger.db.ks.name']
+ db_user = globalDict['ranger.db.ks.javax.persistence.jdbc.user']
+ db_password = globalDict['ranger.db.ks.javax.persistence.jdbc.password']
+ xa_db_root_user = globalDict['ranger.db.root.user']
+ xa_db_root_password = globalDict['ranger.db.root.password']
+
+ if XA_DB_FLAVOR == "MYSQL":
+ MYSQL_CONNECTOR_JAR=globalDict['ranger.db.ks.sql.connector.jar']
+ xa_sqlObj = MysqlConf(xa_db_host,MYSQL_CONNECTOR_JAR,JAVA_BIN)
+ xa_db_core_file_script = os.path.join(os.getcwd(),'scripts')
+ xa_db_core_file = os.path.join(xa_db_core_file_script,mysql_core_file)
+ else:
+ log ("--------- NO SUCH FLAVOUR ---------", "info")
+ sys.exit(1)
+
+ # Methods Begin
+ log("\n--------- Creating kms user --------- \n","info")
+ xa_sqlObj.create_rangerdb_user(xa_db_root_user, db_user, db_password, xa_db_root_password)
+ log("\n--------- Importing DB Core Database ---------\n","info")
+ xa_sqlObj.import_file_to_db(xa_db_root_user, db_name, db_user, db_password, xa_db_root_password, xa_db_core_file)
+ xa_sqlObj.grant_xa_db_user(xa_db_root_user, db_name, db_user, db_password, xa_db_root_password, True)
+
+main()
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/scripts/ranger-kms
----------------------------------------------------------------------
diff --git a/kms/scripts/ranger-kms b/kms/scripts/ranger-kms
index a7cc20e..f049dd2 100755
--- a/kms/scripts/ranger-kms
+++ b/kms/scripts/ranger-kms
@@ -39,7 +39,7 @@ stop)
restart)
echo "Stopping Apache Ranger Kms."
${BIN_PATH}/${MOD_NAME} stop
- echo "Stopping Apache Ranger Kms."
+ echo "Starting Apache Ranger Kms."
${BIN_PATH}/${MOD_NAME} start
;;
*)
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/scripts/ranger-kms-services.sh
----------------------------------------------------------------------
diff --git a/kms/scripts/ranger-kms-services.sh b/kms/scripts/ranger-kms-services.sh
index 7c80cc8..8677c64 100755
--- a/kms/scripts/ranger-kms-services.sh
+++ b/kms/scripts/ranger-kms-services.sh
@@ -71,7 +71,7 @@ fi
KMS_CONF_DIR=${RANGER_KMS_EWS_DIR}/webapp/config/
-JAVA_OPTS="${JAVA_OPTS} -Dcatalina.base=${RANGER_KMS_EWS_DIR} -Dkms.config.dir=${KMS_CONF_DIR} -Dkms.log.dir=${TOMCAT_LOG_DIR} -cp ${RANGER_KMS_EWS_CONF_DIR}:${RANGER_KMS_EWS_LIB_DIR}/*:${JAVA_HOME}/lib/* "
+JAVA_OPTS="${JAVA_OPTS} -Dcatalina.base=${RANGER_KMS_EWS_DIR} -Dkms.config.dir=${KMS_CONF_DIR} -Dkms.log.dir=${TOMCAT_LOG_DIR} -cp ${RANGER_KMS_EWS_CONF_DIR}:${RANGER_KMS_EWS_LIB_DIR}/*:${RANGER_KMS_EWS_DIR}/webapp/lib/*:${JAVA_HOME}/lib/* "
if [ "${action^^}" == "START" ]; then
echo "+ java -D${PROC_NAME} ${JAVA_OPTS} ${START_CLASS_NAME} ${KMS_CONFIG_FILENAME} "
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/scripts/setup.sh
----------------------------------------------------------------------
diff --git a/kms/scripts/setup.sh b/kms/scripts/setup.sh
new file mode 100644
index 0000000..fa88bba
--- /dev/null
+++ b/kms/scripts/setup.sh
@@ -0,0 +1,172 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -------------------------------------------------------------------------------------
+#
+# Ranger KMS Setup Script
+#
+# This script will install ranger kms webapplication under tomcat and also, initialize the database with ranger kms users/tables.
+
+realScriptPath=`readlink -f $0`
+realScriptDir=`dirname $realScriptPath`
+RANGER_KMS_DIR=`(cd $realScriptDir/..; pwd)`
+RANGER_KMS_EWS_DIR=${RANGER_KMS_DIR}/ews
+RANGER_KMS_EWS_CONF_DIR="${RANGER_KMS_EWS_DIR}/conf"
+RANGER_KMS_EWS_LIB_DIR="${RANGER_KMS_EWS_DIR}/lib"
+
+PROPFILE=${RANGER_KMS_EWS_CONF_DIR}/kms_webserver.properties
+propertyValue=''
+
+. $PROPFILE 1>/dev/null 2>&1
+if [ ! $? = "0" ];then
+ log "$PROPFILE file not found....!!";
+ exit 1;
+fi
+
+usage() {
+ [ "$*" ] && echo "$0: $*"
+ sed -n '/^##/,/^$/s/^## \{0,1\}//p' "$0"
+ exit 2
+} 2>/dev/null
+
+log() {
+ local prefix="[$(date +%Y/%m/%d\ %H:%M:%S)]: "
+ echo "${prefix} $@" >> $LOGFILE
+ echo "${prefix} $@"
+}
+
+check_ret_status(){
+ if [ $1 -ne 0 ]; then
+ log "[E] $2";
+ exit 1;
+ fi
+}
+
+is_command () {
+ log "[I] check if command $1 exists"
+ type "$1" >/dev/null
+}
+
+get_distro(){
+ log "[I] Checking distribution name.."
+ ver=$(cat /etc/*{issues,release,version} 2> /dev/null)
+ if [[ $(echo $ver | grep DISTRIB_ID) ]]; then
+ DIST_NAME=$(lsb_release -si)
+ else
+ DIST_NAME=$(echo $ver | cut -d ' ' -f 1 | sort -u | head -1)
+ fi
+ export $DIST_NAME
+ log "[I] Found distribution : $DIST_NAME"
+
+}
+
+init_logfiles () {
+ for f in $LOGFILES; do
+ touch $f
+ done
+}
+
+init_variables(){
+ curDt=`date '+%Y%m%d%H%M%S'`
+
+ INSTALL_DIR=${RANGER_KMS_DIR}
+
+ DB_FLAVOR=`echo $DB_FLAVOR | tr '[:lower:]' '[:upper:]'`
+ if [ "${DB_FLAVOR}" == "" ]
+ then
+ DB_FLAVOR="MYSQL"
+ fi
+ log "[I] DB_FLAVOR=${DB_FLAVOR}"
+}
+
+check_python_command() {
+ if is_command ${PYTHON_COMMAND_INVOKER} ; then
+ log "[I] '${PYTHON_COMMAND_INVOKER}' command found"
+ else
+ log "[E] '${PYTHON_COMMAND_INVOKER}' command not found"
+ exit 1;
+ fi
+}
+
+check_java_version() {
+ #Check for JAVA_HOME
+ if [ "${JAVA_HOME}" == "" ]
+ then
+ log "[E] JAVA_HOME environment property not defined, aborting installation."
+ exit 1
+ fi
+
+ export JAVA_BIN=${JAVA_HOME}/bin/java
+
+ if is_command ${JAVA_BIN} ; then
+ log "[I] '${JAVA_BIN}' command found"
+ else
+ log "[E] '${JAVA_BIN}' command not found"
+ exit 1;
+ fi
+
+ $JAVA_BIN -version 2>&1 | grep -q $JAVA_VERSION_REQUIRED
+ if [ $? != 0 ] ; then
+ log "[E] Java 1.7 is required"
+ exit 1;
+ fi
+}
+
+sanity_check_files() {
+
+ if test -d $app_home; then
+ log "[I] $app_home folder found"
+ else
+ log "[E] $app_home does not exists" ; exit 1;
+ fi
+ if [ "${DB_FLAVOR}" == "MYSQL" ]
+ then
+ if test -f $mysql_core_file; then
+ log "[I] $mysql_core_file file found"
+ else
+ log "[E] $mysql_core_file does not exists" ; exit 1;
+ fi
+ fi
+}
+
+copy_db_connector(){
+ log "[I] Copying ${DB_FLAVOR} Connector to $app_home/lib ";
+ cp -f $SQL_CONNECTOR_JAR $app_home/lib
+ check_ret_status $? "Copying ${DB_FLAVOR} Connector to $app_home/lib failed"
+ log "[I] Copying ${DB_FLAVOR} Connector to $app_home/lib DONE";
+}
+
+setup_kms(){
+ #copying ranger kms provider
+ cd ${RANGER_KMS_EWS_DIR}/webapp
+ log "[I] Adding ranger kms provider as services in hadoop-common jar"
+ jar -uf lib/hadoop-common*.jar META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory
+}
+
+
+init_logfiles
+log " --------- Running ranger kms Web Application Install Script --------- "
+log "[I] uname=`uname`"
+log "[I] hostname=`hostname`"
+init_variables
+get_distro
+check_java_version
+sanity_check_files
+copy_db_connector
+check_python_command
+$PYTHON_COMMAND_INVOKER db_setup.py
+setup_kms
+
+echo "Installation of ranger kms is completed."
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/EncryptedPrivateKeyInfo.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/EncryptedPrivateKeyInfo.java b/kms/src/main/java/org/apache/hadoop/crypto/key/EncryptedPrivateKeyInfo.java
new file mode 100644
index 0000000..ec8f67d
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/EncryptedPrivateKeyInfo.java
@@ -0,0 +1,111 @@
+package org.apache.hadoop.crypto.key;
+
+import java.io.*;
+
+import sun.security.x509.AlgorithmId;
+import sun.security.util.*;
+
+/**
+ * This class implements the <code>EncryptedPrivateKeyInfo</code> type,
+ * which is defined in PKCS #8 as follows:
+ *
+ * <pre>
+ * EncryptedPrivateKeyInfo ::= SEQUENCE {
+ * encryptionAlgorithm AlgorithmIdentifier,
+ * encryptedData OCTET STRING }
+ * </pre>
+ *
+ * @author Jan Luehe
+ */
+final class EncryptedPrivateKeyInfo {
+
+ // the "encryptionAlgorithm" field
+ private AlgorithmId algid;
+
+ // the "encryptedData" field
+ private byte[] encryptedData;
+
+ // the ASN.1 encoded contents of this class
+ private byte[] encoded;
+
+ /**
+ * Constructs (i.e., parses) an <code>EncryptedPrivateKeyInfo</code> from
+ * its encoding.
+ */
+ @SuppressWarnings("restriction")
+ EncryptedPrivateKeyInfo(byte[] encoded) throws IOException {
+ DerValue val = new DerValue(encoded);
+
+ DerValue[] seq = new DerValue[2];
+
+ seq[0] = val.data.getDerValue();
+ seq[1] = val.data.getDerValue();
+
+ if (val.data.available() != 0) {
+ throw new IOException("overrun, bytes = " + val.data.available());
+ }
+
+ this.algid = AlgorithmId.parse(seq[0]);
+ if (seq[0].data.available() != 0) {
+ throw new IOException("encryptionAlgorithm field overrun");
+ }
+
+ this.encryptedData = seq[1].getOctetString();
+ if (seq[1].data.available() != 0)
+ throw new IOException("encryptedData field overrun");
+
+ this.encoded = (byte[])encoded.clone();
+ }
+
+ /**
+ * Constructs an <code>EncryptedPrivateKeyInfo</code> from the
+ * encryption algorithm and the encrypted data.
+ */
+ @SuppressWarnings("restriction")
+ EncryptedPrivateKeyInfo(AlgorithmId algid, byte[] encryptedData) {
+ this.algid = algid;
+ this.encryptedData = (byte[])encryptedData.clone();
+ this.encoded = null; // lazy generation of encoding
+ }
+
+ /**
+ * Returns the encryption algorithm.
+ */
+ @SuppressWarnings("restriction")
+ AlgorithmId getAlgorithm() {
+ return this.algid;
+ }
+
+ /**
+ * Returns the encrypted data.
+ */
+ byte[] getEncryptedData() {
+ return (byte[])this.encryptedData.clone();
+ }
+
+ /**
+ * Returns the ASN.1 encoding of this class.
+ */
+ @SuppressWarnings("restriction")
+ byte[] getEncoded()
+ throws IOException
+ {
+ if (this.encoded != null) return (byte[])this.encoded.clone();
+
+
+ DerOutputStream out = new DerOutputStream();
+ DerOutputStream tmp = new DerOutputStream();
+
+ // encode encryption algorithm
+ algid.encode(tmp);
+
+ // encode encrypted data
+ tmp.putOctetString(encryptedData);
+
+ // wrap everything into a SEQUENCE
+ out.write(DerValue.tag_Sequence, tmp);
+ this.encoded = out.toByteArray();
+
+ return (byte[])this.encoded.clone();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/KeyProtector.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/KeyProtector.java b/kms/src/main/java/org/apache/hadoop/crypto/key/KeyProtector.java
new file mode 100644
index 0000000..5c43301
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/KeyProtector.java
@@ -0,0 +1,541 @@
+package org.apache.hadoop.crypto.key;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.math.BigInteger;
+import java.security.KeyRep;
+import java.security.Security;
+import java.security.Key;
+import java.security.PrivateKey;
+import java.security.Provider;
+import java.security.KeyFactory;
+import java.security.MessageDigest;
+import java.security.GeneralSecurityException;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.SecureRandom;
+import java.security.UnrecoverableKeyException;
+import java.security.AlgorithmParameters;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+
+import javax.crypto.Cipher;
+import javax.crypto.CipherSpi;
+import javax.crypto.SecretKey;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.SealedObject;
+import javax.crypto.spec.*;
+
+import com.sun.crypto.provider.PBEWithMD5AndTripleDESCipher;
+
+import sun.security.x509.AlgorithmId;
+import sun.security.util.DerValue;
+import sun.security.util.ObjectIdentifier;
+
+/**
+ * This class implements a protection mechanism for private keys. In JCE, we
+ * use a stronger protection mechanism than in the JDK, because we can use
+ * the <code>Cipher</code> class.
+ * Private keys are protected using the JCE mechanism, and are recovered using
+ * either the JDK or JCE mechanism, depending on how the key has been
+ * protected. This allows us to parse Sun's keystore implementation that ships
+ * with JDK 1.2.
+ *
+ * @author Jan Luehe
+ *
+ *
+ * @see JceKeyStore
+ */
+
+final class KeyProtector {
+
+ // defined by SunSoft (SKI project)
+ private static final String PBE_WITH_MD5_AND_DES3_CBC_OID
+ = "1.3.6.1.4.1.42.2.19.1";
+
+ // JavaSoft proprietary key-protection algorithm (used to protect private
+ // keys in the keystore implementation that comes with JDK 1.2)
+ private static final String KEY_PROTECTOR_OID = "1.3.6.1.4.1.42.2.17.1.1";
+
+ private static final int SALT_LEN = 20; // the salt length
+ private static final int DIGEST_LEN = 20;
+
+ // the password used for protecting/recovering keys passed through this
+ // key protector
+ private char[] password;
+
+ private static final Provider PROV = Security.getProvider("SunJCE");
+
+ KeyProtector(char[] password) {
+ if (password == null) {
+ throw new IllegalArgumentException("password can't be null");
+ }
+ this.password = password;
+ }
+
+ /**
+ * Protects the given cleartext private key, using the password provided at
+ * construction time.
+ */
+ byte[] protect(PrivateKey key)
+ throws Exception
+ {
+ // create a random salt (8 bytes)
+ byte[] salt = new byte[8];
+ new SecureRandom().nextBytes(salt);
+
+ // create PBE parameters from salt and iteration count
+ PBEParameterSpec pbeSpec = new PBEParameterSpec(salt, 20);
+
+ // create PBE key from password
+ PBEKeySpec pbeKeySpec = new PBEKeySpec(this.password);
+ SecretKey sKey = new PBEKey(pbeKeySpec, "PBEWithMD5AndTripleDES");
+ pbeKeySpec.clearPassword();
+
+ // encrypt private key
+ Cipher cipher = Cipher.getInstance("PBEWithMD5AndTripleDES");
+ cipher.init(Cipher.ENCRYPT_MODE, sKey, pbeSpec, null);
+ byte[] plain = (byte[])key.getEncoded();
+ byte[] encrKey = cipher.doFinal(plain, 0, plain.length);
+
+ // wrap encrypted private key in EncryptedPrivateKeyInfo
+ // (as defined in PKCS#8)
+ AlgorithmParameters pbeParams = AlgorithmParameters.getInstance("PBE", PROV);
+ pbeParams.init(pbeSpec);
+
+ AlgorithmId encrAlg = new AlgorithmId(new ObjectIdentifier(PBE_WITH_MD5_AND_DES3_CBC_OID), pbeParams);
+ return new EncryptedPrivateKeyInfo(encrAlg,encrKey).getEncoded();
+ }
+
+ /*
+ * Recovers the cleartext version of the given key (in protected format),
+ * using the password provided at construction time.
+ */
+ Key recover(EncryptedPrivateKeyInfo encrInfo)
+ throws UnrecoverableKeyException, NoSuchAlgorithmException
+ {
+ byte[] plain;
+
+ try {
+ String encrAlg = encrInfo.getAlgorithm().getOID().toString();
+ if (!encrAlg.equals(PBE_WITH_MD5_AND_DES3_CBC_OID)
+ && !encrAlg.equals(KEY_PROTECTOR_OID)) {
+ throw new UnrecoverableKeyException("Unsupported encryption "
+ + "algorithm");
+ }
+
+ if (encrAlg.equals(KEY_PROTECTOR_OID)) {
+ // JDK 1.2 style recovery
+ plain = recover(encrInfo.getEncryptedData());
+ } else {
+ byte[] encodedParams =
+ encrInfo.getAlgorithm().getEncodedParams();
+
+ // parse the PBE parameters into the corresponding spec
+ AlgorithmParameters pbeParams =
+ AlgorithmParameters.getInstance("PBE");
+ pbeParams.init(encodedParams);
+ PBEParameterSpec pbeSpec = (PBEParameterSpec)
+ pbeParams.getParameterSpec(PBEParameterSpec.class);
+
+ // create PBE key from password
+ PBEKeySpec pbeKeySpec = new PBEKeySpec(this.password);
+ SecretKey sKey =
+ new PBEKey(pbeKeySpec, "PBEWithMD5AndTripleDES");
+ pbeKeySpec.clearPassword();
+
+ // decrypt private key
+ Cipher cipher = Cipher.getInstance("PBEWithMD5AndTripleDES");
+ cipher.init(Cipher.DECRYPT_MODE, sKey, pbeSpec, null);
+ plain= cipher.doFinal(encrInfo.getEncryptedData(), 0, encrInfo.getEncryptedData().length);
+ }
+
+ // determine the private-key algorithm, and parse private key
+ // using the appropriate key factory
+ String oidName = new AlgorithmId(new PrivateKeyInfo(plain).getAlgorithm().getOID()).getName();
+ KeyFactory kFac = KeyFactory.getInstance(oidName);
+ return kFac.generatePrivate(new PKCS8EncodedKeySpec(plain));
+
+ } catch (NoSuchAlgorithmException ex) {
+ // Note: this catch needed to be here because of the
+ // later catch of GeneralSecurityException
+ throw ex;
+ } catch (IOException ioe) {
+ throw new UnrecoverableKeyException(ioe.getMessage());
+ } catch (GeneralSecurityException gse) {
+ throw new UnrecoverableKeyException(gse.getMessage());
+ }
+ }
+
+ /*
+ * Recovers the cleartext version of the given key (in protected format),
+ * using the password provided at construction time. This method implements
+ * the recovery algorithm used by Sun's keystore implementation in
+ * JDK 1.2.
+ */
+ private byte[] recover(byte[] protectedKey)
+ throws UnrecoverableKeyException, NoSuchAlgorithmException
+ {
+ int i, j;
+ byte[] digest;
+ int numRounds;
+ int xorOffset; // offset in xorKey where next digest will be stored
+ int encrKeyLen; // the length of the encrpyted key
+
+ MessageDigest md = MessageDigest.getInstance("SHA");
+
+ // Get the salt associated with this key (the first SALT_LEN bytes of
+ // <code>protectedKey</code>)
+ byte[] salt = new byte[SALT_LEN];
+ System.arraycopy(protectedKey, 0, salt, 0, SALT_LEN);
+
+ // Determine the number of digest rounds
+ encrKeyLen = protectedKey.length - SALT_LEN - DIGEST_LEN;
+ numRounds = encrKeyLen / DIGEST_LEN;
+ if ((encrKeyLen % DIGEST_LEN) != 0)
+ numRounds++;
+
+ // Get the encrypted key portion and store it in "encrKey"
+ byte[] encrKey = new byte[encrKeyLen];
+ System.arraycopy(protectedKey, SALT_LEN, encrKey, 0, encrKeyLen);
+
+ // Set up the byte array which will be XORed with "encrKey"
+ byte[] xorKey = new byte[encrKey.length];
+
+ // Convert password to byte array, so that it can be digested
+ byte[] passwdBytes = new byte[password.length * 2];
+ for (i=0, j=0; i<password.length; i++) {
+ passwdBytes[j++] = (byte)(password[i] >> 8);
+ passwdBytes[j++] = (byte)password[i];
+ }
+
+ // Compute the digests, and store them in "xorKey"
+ for (i = 0, xorOffset = 0, digest = salt;
+ i < numRounds;
+ i++, xorOffset += DIGEST_LEN) {
+ md.update(passwdBytes);
+ md.update(digest);
+ digest = md.digest();
+ md.reset();
+ // Copy the digest into "xorKey"
+ if (i < numRounds - 1) {
+ System.arraycopy(digest, 0, xorKey, xorOffset,
+ digest.length);
+ } else {
+ System.arraycopy(digest, 0, xorKey, xorOffset,
+ xorKey.length - xorOffset);
+ }
+ }
+
+ // XOR "encrKey" with "xorKey", and store the result in "plainKey"
+ byte[] plainKey = new byte[encrKey.length];
+ for (i = 0; i < plainKey.length; i++) {
+ plainKey[i] = (byte)(encrKey[i] ^ xorKey[i]);
+ }
+
+ // Check the integrity of the recovered key by concatenating it with
+ // the password, digesting the concatenation, and comparing the
+ // result of the digest operation with the digest provided at the end
+ // of <code>protectedKey</code>. If the two digest values are
+ // different, throw an exception.
+ md.update(passwdBytes);
+ java.util.Arrays.fill(passwdBytes, (byte)0x00);
+ passwdBytes = null;
+ md.update(plainKey);
+ digest = md.digest();
+ md.reset();
+ for (i = 0; i < digest.length; i++) {
+ if (digest[i] != protectedKey[SALT_LEN + encrKeyLen + i]) {
+ throw new UnrecoverableKeyException("Cannot recover key");
+ }
+ }
+ return plainKey;
+ }
+
+ /**
+ * Seals the given cleartext key, using the password provided at
+ * construction time
+ */
+ SealedObject seal(Key key)
+ throws Exception
+ {
+ // create a random salt (8 bytes)
+ byte[] salt = new byte[8];
+ new SecureRandom().nextBytes(salt);
+
+ // create PBE parameters from salt and iteration count
+ PBEParameterSpec pbeSpec = new PBEParameterSpec(salt, 20);
+
+ // create PBE key from password
+ PBEKeySpec pbeKeySpec = new PBEKeySpec(this.password);
+ SecretKey sKey = new PBEKey(pbeKeySpec, "PBEWithMD5AndTripleDES");
+ pbeKeySpec.clearPassword();
+
+ // seal key
+ Cipher cipher;
+
+ PBEWithMD5AndTripleDESCipher cipherSpi;
+ cipherSpi = new PBEWithMD5AndTripleDESCipher();
+ cipher = new CipherForKeyProtector(cipherSpi, PROV,
+ "PBEWithMD5AndTripleDES");
+ cipher.init(Cipher.ENCRYPT_MODE, sKey, pbeSpec);
+ return new SealedObjectForKeyProtector(key, cipher);
+ }
+
+ /**
+ * Unseals the sealed key.
+ */
+ Key unseal(SealedObject so)
+ throws NoSuchAlgorithmException, UnrecoverableKeyException
+ {
+ try {
+ // create PBE key from password
+ PBEKeySpec pbeKeySpec = new PBEKeySpec(this.password);
+ SecretKey skey = new PBEKey(pbeKeySpec, "PBEWithMD5AndTripleDES");
+ pbeKeySpec.clearPassword();
+
+ SealedObjectForKeyProtector soForKeyProtector = null;
+ if (!(so instanceof SealedObjectForKeyProtector)) {
+ soForKeyProtector = new SealedObjectForKeyProtector(so);
+ } else {
+ soForKeyProtector = (SealedObjectForKeyProtector)so;
+ }
+ AlgorithmParameters params = soForKeyProtector.getParameters();
+ if (params == null) {
+ throw new UnrecoverableKeyException("Cannot get " +
+ "algorithm parameters");
+ }
+ PBEWithMD5AndTripleDESCipher cipherSpi;
+ cipherSpi = new PBEWithMD5AndTripleDESCipher();
+ Cipher cipher = new CipherForKeyProtector(cipherSpi, PROV,
+ "PBEWithMD5AndTripleDES");
+ cipher.init(Cipher.DECRYPT_MODE, skey, params);
+ return (Key)soForKeyProtector.getObject(cipher);
+ } catch (NoSuchAlgorithmException ex) {
+ // Note: this catch needed to be here because of the
+ // later catch of GeneralSecurityException
+ throw ex;
+ } catch (IOException ioe) {
+ throw new UnrecoverableKeyException(ioe.getMessage());
+ } catch (ClassNotFoundException cnfe) {
+ throw new UnrecoverableKeyException(cnfe.getMessage());
+ } catch (GeneralSecurityException gse) {
+ throw new UnrecoverableKeyException(gse.getMessage());
+ }
+ }
+}
+
+
+final class CipherForKeyProtector extends javax.crypto.Cipher {
+ /**
+ * Creates a Cipher object.
+ *
+ * @param cipherSpi the delegate
+ * @param provider the provider
+ * @param transformation the transformation
+ */
+ protected CipherForKeyProtector(CipherSpi cipherSpi,
+ Provider provider,
+ String transformation) {
+ super(cipherSpi, provider, transformation);
+ }
+}
+
+final class SealedObjectForKeyProtector extends javax.crypto.SealedObject {
+
+ static final long serialVersionUID = -3650226485480866989L;
+
+ SealedObjectForKeyProtector(Serializable object, Cipher c)
+ throws IOException, IllegalBlockSizeException {
+ super(object, c);
+ }
+
+ SealedObjectForKeyProtector(SealedObject so) {
+ super(so);
+ }
+
+ AlgorithmParameters getParameters() {
+ AlgorithmParameters params = null;
+ if (super.encodedParams != null) {
+ try {
+ params = AlgorithmParameters.getInstance("PBE", "SunJCE");
+ params.init(super.encodedParams);
+ } catch (NoSuchProviderException nspe) {
+ // eat.
+ } catch (NoSuchAlgorithmException nsae) {
+ //eat.
+ } catch (IOException ioe) {
+ //eat.
+ }
+ }
+ return params;
+ }
+}
+
+final class PrivateKeyInfo {
+
+ // the version number defined by the PKCS #8 standard
+ private static final BigInteger VERSION = BigInteger.ZERO;
+
+ // the private-key algorithm
+ private AlgorithmId algid;
+
+ // the private-key value
+ private byte[] privkey;
+
+ /**
+ * Constructs a PKCS#8 PrivateKeyInfo from its ASN.1 encoding.
+ */
+ PrivateKeyInfo(byte[] encoded) throws IOException {
+ DerValue val = new DerValue(encoded);
+
+ if (val.tag != DerValue.tag_Sequence)
+ throw new IOException("private key parse error: not a sequence");
+
+ // version
+ BigInteger parsedVersion = val.data.getBigInteger();
+ if (!parsedVersion.equals(VERSION)) {
+ throw new IOException("version mismatch: (supported: " +
+ VERSION + ", parsed: " + parsedVersion);
+ }
+
+ // privateKeyAlgorithm
+ this.algid = AlgorithmId.parse(val.data.getDerValue());
+
+ // privateKey
+ this.privkey = val.data.getOctetString();
+
+ // OPTIONAL attributes not supported yet
+ }
+
+ /**
+ * Returns the private-key algorithm.
+ */
+ AlgorithmId getAlgorithm() {
+ return this.algid;
+ }
+}
+
+/**
+ * This class represents a PBE key.
+ *
+ * @author Jan Luehe
+ *
+ */
+final class PBEKey implements SecretKey {
+
+ static final long serialVersionUID = -2234768909660948176L;
+
+ private byte[] key;
+
+ private String type;
+
+ /**
+ * Creates a PBE key from a given PBE key specification.
+ *
+ * @param key the given PBE key specification
+ */
+ PBEKey(PBEKeySpec keySpec, String keytype) throws InvalidKeySpecException {
+ char[] passwd = keySpec.getPassword();
+ if (passwd == null) {
+ // Should allow an empty password.
+ passwd = new char[0];
+ }
+ for (int i=0; i<passwd.length; i++) {
+ if ((passwd[i] < '\u0020') || (passwd[i] > '\u007E')) {
+ throw new InvalidKeySpecException("Password is not ASCII");
+ }
+ }
+ this.key = new byte[passwd.length];
+ for (int i=0; i<passwd.length; i++)
+ this.key[i] = (byte) (passwd[i] & 0x7f);
+ java.util.Arrays.fill(passwd, ' ');
+ type = keytype;
+ }
+
+ public byte[] getEncoded() {
+ return (byte[])this.key.clone();
+ }
+
+ public String getAlgorithm() {
+ return type;
+ }
+
+ public String getFormat() {
+ return "RAW";
+ }
+
+ /**
+ * Calculates a hash code value for the object.
+ * Objects that are equal will also have the same hashcode.
+ */
+ public int hashCode() {
+ int retval = 0;
+ for (int i = 1; i < this.key.length; i++) {
+ retval += this.key[i] * i;
+ }
+ return(retval ^= getAlgorithm().toLowerCase().hashCode());
+ }
+
+ public boolean equals(Object obj) {
+ if (obj == this)
+ return true;
+
+ if (!(obj instanceof SecretKey))
+ return false;
+
+ SecretKey that = (SecretKey)obj;
+
+ if (!(that.getAlgorithm().equalsIgnoreCase(type)))
+ return false;
+
+ byte[] thatEncoded = that.getEncoded();
+ boolean ret = java.util.Arrays.equals(this.key, thatEncoded);
+ java.util.Arrays.fill(thatEncoded, (byte)0x00);
+ return ret;
+ }
+
+ /**
+ * readObject is called to restore the state of this key from
+ * a stream.
+ */
+ private void readObject(java.io.ObjectInputStream s)
+ throws java.io.IOException, ClassNotFoundException
+ {
+ s.defaultReadObject();
+ key = (byte[])key.clone();
+ }
+
+
+ /**
+ * Replace the PBE key to be serialized.
+ *
+ * @return the standard KeyRep object to be serialized
+ *
+ * @throws java.io.ObjectStreamException if a new object representing
+ * this PBE key could not be created
+ */
+ private Object writeReplace() throws java.io.ObjectStreamException {
+ return new KeyRep(KeyRep.Type.SECRET,
+ getAlgorithm(),
+ getFormat(),
+ getEncoded());
+ }
+
+ /**
+ * Ensures that the password bytes of this key are
+ * set to zero when there are no more references to it.
+ */
+ protected void finalize() throws Throwable {
+ try {
+ if (this.key != null) {
+ java.util.Arrays.fill(this.key, (byte)0x00);
+ this.key = null;
+ }
+ } finally {
+ super.finalize();
+ }
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSDB.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSDB.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSDB.java
new file mode 100644
index 0000000..649a321
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSDB.java
@@ -0,0 +1,85 @@
+package org.apache.hadoop.crypto.key;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.persistence.EntityManager;
+import javax.persistence.EntityManagerFactory;
+import javax.persistence.Persistence;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Logger;
+import org.apache.ranger.kms.dao.DaoManager;
+
+public class RangerKMSDB {
+
+ static final Logger logger = Logger.getLogger(RangerKMSDB.class);
+
+ private EntityManagerFactory entityManagerFactory;
+ private DaoManager daoManager;
+
+ private static Map<String, String> DB_PROPERTIES = null;
+
+ private static final String PROPERTY_PREFIX = "ranger.db.ks.";
+ private static final String DB_DIALECT = "javax.persistence.jdbc.dialect";
+ private static final String DB_DRIVER = "javax.persistence.jdbc.driver";
+ private static final String DB_URL = "javax.persistence.jdbc.url";
+ private static final String DB_USER = "javax.persistence.jdbc.user";
+ private static final String DB_PASSWORD = "javax.persistence.jdbc.password";
+
+ private final Configuration conf;
+
+ public RangerKMSDB(){
+ conf = new Configuration();
+ }
+
+ public RangerKMSDB(Configuration conf){
+ this.conf = conf;
+ initDBConnectivity();
+ }
+
+ public DaoManager getDaoManager(){
+ return daoManager;
+ }
+
+ private void initDBConnectivity(){
+ try {
+ DB_PROPERTIES = new HashMap<String, String>();
+ DB_PROPERTIES.put(DB_DIALECT, conf.get(PROPERTY_PREFIX+DB_DIALECT));
+ DB_PROPERTIES.put(DB_DRIVER, conf.get(PROPERTY_PREFIX+DB_DRIVER));
+ DB_PROPERTIES.put(DB_URL, conf.get(PROPERTY_PREFIX+DB_URL));
+ DB_PROPERTIES.put(DB_USER, conf.get(PROPERTY_PREFIX+DB_USER));
+ DB_PROPERTIES.put(DB_PASSWORD, conf.get(PROPERTY_PREFIX+DB_PASSWORD));
+
+ entityManagerFactory = Persistence.createEntityManagerFactory("persistence_ranger_server", DB_PROPERTIES);
+
+ daoManager = new DaoManager();
+ daoManager.setEntityManagerFactory(entityManagerFactory);
+
+ daoManager.getEntityManager(); // this forces the connection to be made to DB
+ logger.info("Connected to DB : "+isDbConnected());
+ } catch(Exception excp) {
+ excp.printStackTrace();
+ }
+ }
+
+ private boolean isDbConnected() {
+ EntityManager em = getEntityManager();
+
+ return em != null && em.isOpen();
+ }
+
+ private EntityManager getEntityManager() {
+ DaoManager daoMgr = daoManager;
+
+ if(daoMgr != null) {
+ try {
+ return daoMgr.getEntityManager();
+ } catch(Exception excp) {
+ excp.printStackTrace();
+ }
+ }
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java
new file mode 100644
index 0000000..9b1ff67
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java
@@ -0,0 +1,537 @@
+package org.apache.hadoop.crypto.key;
+
+import java.io.*;
+import java.security.*;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+import java.util.*;
+
+import javax.crypto.SealedObject;
+import javax.xml.bind.DatatypeConverter;
+
+import org.apache.log4j.Logger;
+import org.apache.ranger.entity.XXRangerKeyStore;
+import org.apache.ranger.kms.dao.DaoManager;
+import org.apache.ranger.kms.dao.RangerKMSDao;
+
+/**
+ * This class provides the Database store implementation.
+ *
+ *
+ * @see KeyProtector
+ *
+ */
+
+public class RangerKeyStore extends KeyStoreSpi {
+
+ static final Logger logger = Logger.getLogger(RangerKeyStore.class);
+
+ private DaoManager daoManager;
+
+ // keys
+ private static class KeyEntry {
+ Date date; // the creation date of this entry
+ };
+
+ // Secret key
+ private static final class SecretKeyEntry {
+ Date date; // the creation date of this entry
+ SealedObject sealedKey;
+ String cipher_field;
+ int bit_length;
+ String description;
+ String attributes;
+ int version;
+ }
+
+ /**
+ * keys are stored in a hashtable.
+ * Hash entries are keyed by alias names.
+ */
+ private final Hashtable<String, Object> entries;
+
+ RangerKeyStore() {
+ entries = new Hashtable<String, Object>();
+ }
+
+ RangerKeyStore(DaoManager daoManager) {
+ entries = new Hashtable<String, Object>();
+ this.daoManager = daoManager;
+ }
+
+ // convert an alias to internal form, overridden in subclasses:
+ String convertAlias(String alias){
+ return alias.toLowerCase();
+ }
+
+ /**
+ * Returns the key associated with the given alias, using the given
+ * password to recover it.
+ *
+ * @param alias the alias name
+ * @param password the password for recovering the key
+ *
+ * @return the requested key, or null if the given alias does not exist
+ * or does not identify a <i>key entry</i>.
+ *
+ * @exception NoSuchAlgorithmException if the algorithm for recovering the
+ * key cannot be found
+ * @exception UnrecoverableKeyException if the key cannot be recovered
+ * (e.g., the given password is wrong).
+ */
+ public Key engineGetKey(String alias, char[] password)
+ throws NoSuchAlgorithmException, UnrecoverableKeyException
+ {
+ Key key = null;
+
+ Object entry = entries.get(alias.toLowerCase());
+
+ if (!(entry instanceof SecretKeyEntry)) {
+ return null;
+ }
+
+ KeyProtector keyProtector = new KeyProtector(password);
+ key = keyProtector.unseal(((SecretKeyEntry)entry).sealedKey);
+ return key;
+ }
+
+ /**
+ * Returns the creation date of the entry identified by the given alias.
+ *
+ * @param alias the alias name
+ *
+ * @return the creation date of this entry, or null if the given alias does
+ * not exist
+ */
+ public Date engineGetCreationDate(String alias) {
+ Object entry = entries.get(convertAlias(alias));
+ if (entry != null) {
+ return new Date(((KeyEntry)entry).date.getTime());
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Assigns the given key to the given alias, protecting
+ * it with the given password as defined in PKCS8.
+ *
+ * <p>The given java.security.PrivateKey <code>key</code> must
+ * be accompanied by a certificate chain certifying the
+ * corresponding public key.
+ *
+ * <p>If the given alias already exists, the keystore information
+ * associated with it is overridden by the given key and certificate
+ * chain.
+ *
+ * @param alias the alias name
+ * @param key the key to be associated with the alias
+ * @param password the password to protect the key
+ * @param cipher the cipher used for the key
+ * @param bitLength bit length for the key
+ * @param description Description for the key
+ * @param version Key version
+ * @param attributes key attributes
+ *
+ * @exception KeyStoreException if the given key is not a private key,
+ * cannot be protected, or this operation fails for some other reason
+ */
+ public void engineSetKeyEntry(String alias, Key key, char[] password, String cipher, int bitLength, String description, int version, String attributes)
+ throws KeyStoreException
+ {
+ synchronized(entries) {
+ try {
+ KeyProtector keyProtector = new KeyProtector(password);
+
+ SecretKeyEntry entry = new SecretKeyEntry();
+ entry.date = new Date();
+ // seal and store the key
+ entry.sealedKey = keyProtector.seal(key);
+ entry.cipher_field = cipher;
+ entry.bit_length = bitLength;
+ entry.description = description;
+ entry.version = version;
+ entry.attributes = attributes;
+ entries.put(alias.toLowerCase(), entry);
+ } catch (Exception e) {
+ throw new KeyStoreException(e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * Deletes the entry identified by the given alias from this database.
+ *
+ * @param alias the alias name
+ *
+ * @exception KeyStoreException if the entry cannot be removed.
+ */
+ public void engineDeleteEntry(String alias)
+ throws KeyStoreException
+ {
+ synchronized(entries) {
+ dbOperationDelete(convertAlias(alias));
+ entries.remove(convertAlias(alias));
+ }
+ }
+
+ private void dbOperationDelete(String alias) {
+ try{
+ if(daoManager != null){
+ RangerKMSDao rangerKMSDao = new RangerKMSDao(daoManager);
+ rangerKMSDao.deleteByAlias(alias);
+ }
+ }catch(Exception e){
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * Lists all the alias names of this database.
+ *
+ * @return enumeration of the alias names
+ */
+ public Enumeration<String> engineAliases() {
+ return entries.keys();
+ }
+
+ /**
+ * Checks if the given alias exists in this database.
+ *
+ * @param alias the alias name
+ *
+ * @return true if the alias exists, false otherwise
+ */
+ public boolean engineContainsAlias(String alias) {
+ return entries.containsKey(convertAlias(alias));
+ }
+
+ /**
+ * Retrieves the number of entries in this database.
+ *
+ * @return the number of entries in this database
+ */
+ public int engineSize() {
+ return entries.size();
+ }
+
+ /**
+ * Stores this keystore to the provided ranger database, and protects its
+ * integrity with the given password.
+ *
+ * @param stream null.
+ * @param password the password to generate the keystore integrity check
+ *
+ * @exception IOException if there was an I/O problem with data
+ * @exception NoSuchAlgorithmException if the appropriate data integrity
+ * algorithm could not be found
+ * @exception CertificateException if any of the certificates included in
+ * the keystore data could not be stored
+ */
+ public void engineStore(OutputStream stream, char[] password)
+ throws IOException, NoSuchAlgorithmException, CertificateException
+ {
+ synchronized(entries) {
+ // password is mandatory when storing
+ if (password == null) {
+ throw new IllegalArgumentException("Ranger Master Key can't be null");
+ }
+
+ MessageDigest md = getPreKeyedHash(password);
+
+ byte digest[] = md.digest();
+ for (Enumeration<String> e = entries.keys(); e.hasMoreElements();) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(new DigestOutputStream(baos, md));
+
+ ObjectOutputStream oos = null;
+ try{
+
+ String alias = e.nextElement();
+ Object entry = entries.get(alias);
+
+ // write the sealed key
+ oos = new ObjectOutputStream(dos);
+ oos.writeObject(((SecretKeyEntry)entry).sealedKey);
+ /*
+ * Write the keyed hash which is used to detect tampering with
+ * the keystore (such as deleting or modifying key or
+ * certificate entries).
+ */
+ dos.write(digest);
+ dos.flush();
+ Long creationDate = ((SecretKeyEntry)entry).date.getTime();
+ SecretKeyEntry secretKey = (SecretKeyEntry)entry;
+ XXRangerKeyStore xxRangerKeyStore = mapObjectToEntity(alias,creationDate,baos.toByteArray(), secretKey.cipher_field, secretKey.bit_length, secretKey.description, secretKey.version, secretKey.attributes);
+ dbOperationStore(xxRangerKeyStore);
+ }finally {
+ if (oos != null) {
+ oos.close();
+ } else {
+ dos.close();
+ }
+ }
+ }
+ }
+ }
+
+ private XXRangerKeyStore mapObjectToEntity(String alias, Long creationDate,
+ byte[] byteArray, String cipher_field, int bit_length,
+ String description, int version, String attributes) {
+ XXRangerKeyStore xxRangerKeyStore = new XXRangerKeyStore();
+ xxRangerKeyStore.setAlias(alias);
+ xxRangerKeyStore.setCreatedDate(creationDate);
+ xxRangerKeyStore.setEncoded(DatatypeConverter.printBase64Binary(byteArray));
+ xxRangerKeyStore.setCipher(cipher_field);
+ xxRangerKeyStore.setBitLength(bit_length);
+ xxRangerKeyStore.setDescription(description);
+ xxRangerKeyStore.setVersion(version);
+ xxRangerKeyStore.setAttributes(attributes);
+ return xxRangerKeyStore;
+ }
+
+ private void dbOperationStore(XXRangerKeyStore rangerKeyStore) {
+ try{
+ if(daoManager != null){
+ RangerKMSDao rangerKMSDao = new RangerKMSDao(daoManager);
+ XXRangerKeyStore xxRangerKeyStore = rangerKMSDao.findByAlias(rangerKeyStore.getAlias());
+ boolean keyStoreExists = true;
+ if (xxRangerKeyStore == null) {
+ xxRangerKeyStore = new XXRangerKeyStore();
+ keyStoreExists = false;
+ }
+
+ xxRangerKeyStore = mapToEntityBean(rangerKeyStore, xxRangerKeyStore, 0);
+ if (keyStoreExists) {
+ xxRangerKeyStore = rangerKMSDao.update(xxRangerKeyStore);
+ } else {
+ xxRangerKeyStore = rangerKMSDao.create(xxRangerKeyStore);
+ }
+ }
+ }catch(Exception e){
+ e.printStackTrace();
+ }
+ }
+
+ private XXRangerKeyStore mapToEntityBean(XXRangerKeyStore rangerKMSKeyStore, XXRangerKeyStore xxRangerKeyStore,int i) {
+ xxRangerKeyStore.setAlias(rangerKMSKeyStore.getAlias());
+ xxRangerKeyStore.setCreatedDate(rangerKMSKeyStore.getCreatedDate());
+ xxRangerKeyStore.setEncoded(rangerKMSKeyStore.getEncoded());
+ xxRangerKeyStore.setCipher(rangerKMSKeyStore.getCipher());
+ xxRangerKeyStore.setBitLength(rangerKMSKeyStore.getBitLength());
+ xxRangerKeyStore.setDescription(rangerKMSKeyStore.getDescription());
+ xxRangerKeyStore.setVersion(rangerKMSKeyStore.getVersion());
+ xxRangerKeyStore.setAttributes(rangerKMSKeyStore.getAttributes());
+ return xxRangerKeyStore;
+ }
+
+ /**
+ * Loads the keystore from the given ranger database.
+ *
+ * <p>If a password is given, it is used to check the integrity of the
+ * keystore data. Otherwise, the integrity of the keystore is not checked.
+ *
+ * @param stream the input stream from which the keystore is loaded
+ * @param password the (optional) password used to check the integrity of
+ * the keystore.
+ *
+ * @exception IOException if there is an I/O or format problem with the
+ * keystore data
+ * @exception NoSuchAlgorithmException if the algorithm used to check
+ * the integrity of the keystore cannot be found
+ * @exception CertificateException if any of the certificates in the
+ * keystore could not be loaded
+ */
+ public void engineLoad(InputStream stream, char[] password)
+ throws IOException , NoSuchAlgorithmException, CertificateException
+ {
+ synchronized(entries) {
+ List<XXRangerKeyStore> rangerKeyDetails = dbOperationLoad();
+ DataInputStream dis;
+ MessageDigest md = null;
+
+ if(rangerKeyDetails == null || rangerKeyDetails.size() < 1){
+ return;
+ }
+
+ entries.clear();
+ if (password != null) {
+ md = getPreKeyedHash(password);
+ }
+ byte computed[];
+ computed = md.digest();
+ for(XXRangerKeyStore rangerKey : rangerKeyDetails){
+ String encoded = rangerKey.getEncoded();
+ byte[] data = DatatypeConverter.parseBase64Binary(encoded);
+
+ if(data != null && data.length > 0){
+ stream = new ByteArrayInputStream(data);
+ }else{
+ logger.error("No Key found for alias "+rangerKey.getAlias());
+ }
+
+ /*
+ * If a password has been provided, we check the keyed digest
+ * at the end. If this check fails, the store has been tampered
+ * with
+ */
+ if (password != null) {
+ int counter = 0;
+ for (int i = computed.length-1; i >= 0; i--) {
+ if (computed[i] != data[data.length-(1+counter)]) {
+ Throwable t = new UnrecoverableKeyException
+ ("Password verification failed");
+ throw (IOException)new IOException
+ ("Keystore was tampered with, or "
+ + "password was incorrect").initCause(t);
+ }else{
+ counter++;
+ }
+ }
+ }
+
+ if (password != null) {
+ dis = new DataInputStream(new DigestInputStream(stream, md));
+ } else {
+ dis = new DataInputStream(stream);
+ }
+
+ ObjectInputStream ois = null;
+ try{
+ String alias;
+
+ SecretKeyEntry entry = new SecretKeyEntry();
+
+ //read the alias
+ alias = rangerKey.getAlias();
+
+ //read the (entry creation) date
+ entry.date = new Date(rangerKey.getCreatedDate());
+ entry.cipher_field = rangerKey.getCipher();
+ entry.bit_length = rangerKey.getBitLength();
+ entry.description = rangerKey.getDescription();
+ entry.version = rangerKey.getVersion();
+ entry.attributes = rangerKey.getAttributes();
+
+ //read the sealed key
+ try {
+ ois = new ObjectInputStream(dis);
+ entry.sealedKey = (SealedObject)ois.readObject();
+ } catch (ClassNotFoundException cnfe) {
+ throw new IOException(cnfe.getMessage());
+ }
+
+ //Add the entry to the list
+ entries.put(alias, entry);
+ }finally {
+ if (ois != null) {
+ ois.close();
+ } else {
+ dis.close();
+ }
+ }
+ }
+ }
+ }
+
+ private List<XXRangerKeyStore> dbOperationLoad() throws IOException {
+ try{
+ if(daoManager != null){
+ RangerKMSDao rangerKMSDao = new RangerKMSDao(daoManager);
+ return rangerKMSDao.getAll();
+ }
+ }catch(Exception e){
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ /**
+ * To guard against tampering with the keystore, we append a keyed
+ * hash with a bit of whitener.
+ */
+ private MessageDigest getPreKeyedHash(char[] password)
+ throws NoSuchAlgorithmException, UnsupportedEncodingException
+ {
+ int i, j;
+
+ MessageDigest md = MessageDigest.getInstance("SHA");
+ byte[] passwdBytes = new byte[password.length * 2];
+ for (i=0, j=0; i<password.length; i++) {
+ passwdBytes[j++] = (byte)(password[i] >> 8);
+ passwdBytes[j++] = (byte)password[i];
+ }
+ md.update(passwdBytes);
+ for (i=0; i<passwdBytes.length; i++)
+ passwdBytes[i] = 0;
+ md.update("Mighty Aphrodite".getBytes("UTF8"));
+ return md;
+ }
+
+ @Override
+ public void engineSetKeyEntry(String arg0, byte[] arg1, Certificate[] arg2)
+ throws KeyStoreException {
+
+ }
+
+ @Override
+ public Certificate engineGetCertificate(String alias) {
+ return null;
+ }
+
+ @Override
+ public String engineGetCertificateAlias(Certificate cert) {
+ return null;
+ }
+
+ @Override
+ public Certificate[] engineGetCertificateChain(String alias) {
+ return null;
+ }
+
+ @Override
+ public boolean engineIsCertificateEntry(String alias) {
+ return false;
+ }
+
+ @Override
+ public boolean engineIsKeyEntry(String alias) {
+ return false;
+ }
+
+ @Override
+ public void engineSetCertificateEntry(String alias, Certificate cert)
+ throws KeyStoreException {
+ }
+
+ @Override
+ public void engineSetKeyEntry(String alias, Key key, char[] password,
+ Certificate[] chain) throws KeyStoreException {
+ }
+
+ public Map<String, String> getPropertiesWithPrefix(Properties props, String prefix) {
+ Map<String, String> prefixedProperties = new HashMap<String, String>();
+
+ if(props != null && prefix != null) {
+ for(String key : props.stringPropertyNames()) {
+ if(key == null) {
+ continue;
+ }
+
+ String val = props.getProperty(key);
+
+ if(key.startsWith(prefix)) {
+ key = key.substring(prefix.length());
+
+ if(key == null) {
+ continue;
+ }
+
+ prefixedProperties.put(key, val);
+ }
+ }
+ }
+
+ return prefixedProperties;
+ }
+}
\ No newline at end of file