You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by tu...@apache.org on 2014/08/21 20:58:55 UTC
svn commit: r1619518 [1/3] - in
/hadoop/common/branches/branch-2/hadoop-common-project: ./ hadoop-common/
hadoop-common/dev-support/
hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/
hadoop-common/src/main/resources/META-INF/services/ hadoo...
Author: tucu
Date: Thu Aug 21 18:58:53 2014
New Revision: 1619518
URL: http://svn.apache.org/r1619518
Log:
HADOOP-10433. Key Management Server based on KeyProvider API. (tucu)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
hadoop-project/pom.xml
Added:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/dev-support/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/dev-support/findbugsExcludeFile.xml
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/pom.xml
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-log4j.properties
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSCacheKeyProvider.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSMDCFilter.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/libexec/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/sbin/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/tomcat/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/tomcat/ROOT/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/tomcat/ROOT/WEB-INF/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/tomcat/ROOT/WEB-INF/web.xml
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/tomcat/ROOT/index.html
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/tomcat/logging.properties
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/tomcat/server.xml
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/tomcat/ssl-server.xml
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/webapp/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/webapp/WEB-INF/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/webapp/WEB-INF/web.xml
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/site/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/site/apt/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/site/resources/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/site/resources/css/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/site/resources/css/site.css
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/site/site.xml
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/org/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/org/apache/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSACLs.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSCacheKeyProvider.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/resources/
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/resources/log4j.properties
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory
hadoop/common/branches/branch-2/hadoop-common-project/pom.xml
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619518&r1=1619517&r2=1619518&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Thu Aug 21 18:58:53 2014
@@ -6,6 +6,8 @@ Release 2.6.0 - UNRELEASED
NEW FEATURES
+ HADOOP-10433. Key Management Server based on KeyProvider API. (tucu)
+
IMPROVEMENTS
HADOOP-10808. Remove unused native code for munlock. (cnauroth)
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml?rev=1619518&r1=1619517&r2=1619518&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml Thu Aug 21 18:58:53 2014
@@ -379,4 +379,10 @@
<Bug code="NP" />
</Match>
+ <Match>
+ <Class name="org.apache.hadoop.crypto.key.kms.KMSClientProvider"/>
+ <Method name="validateResponse"/>
+ <Bug pattern="REC_CATCH_EXCEPTION"/>
+ </Match>
+
</FindBugsFilter>
Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java?rev=1619518&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java (added)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java Thu Aug 21 18:58:53 2014
@@ -0,0 +1,519 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
+import org.apache.hadoop.security.ssl.SSLFactory;
+import org.apache.http.client.utils.URIBuilder;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import javax.net.ssl.HttpsURLConnection;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.lang.reflect.Constructor;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.security.GeneralSecurityException;
+import java.security.NoSuchAlgorithmException;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * KMS client <code>KeyProvider</code> implementation.
+ */
+@InterfaceAudience.Private
+public class KMSClientProvider extends KeyProvider {
+
+ public static final String SCHEME_NAME = "kms";
+
+ private static final String UTF8 = "UTF-8";
+
+ private static final String CONTENT_TYPE = "Content-Type";
+ private static final String APPLICATION_JSON_MIME = "application/json";
+
+ private static final String HTTP_GET = "GET";
+ private static final String HTTP_POST = "POST";
+ private static final String HTTP_PUT = "PUT";
+ private static final String HTTP_DELETE = "DELETE";
+
+ private static KeyVersion parseJSONKeyVersion(Map valueMap) {
+ KeyVersion keyVersion = null;
+ if (!valueMap.isEmpty()) {
+ byte[] material = (valueMap.containsKey(KMSRESTConstants.MATERIAL_FIELD))
+ ? Base64.decodeBase64((String) valueMap.get(KMSRESTConstants.MATERIAL_FIELD))
+ : null;
+ keyVersion = new KMSKeyVersion((String)
+ valueMap.get(KMSRESTConstants.VERSION_NAME_FIELD), material);
+ }
+ return keyVersion;
+ }
+
+ private static Metadata parseJSONMetadata(Map valueMap) {
+ Metadata metadata = null;
+ if (!valueMap.isEmpty()) {
+ metadata = new KMSMetadata(
+ (String) valueMap.get(KMSRESTConstants.CIPHER_FIELD),
+ (Integer) valueMap.get(KMSRESTConstants.LENGTH_FIELD),
+ (String) valueMap.get(KMSRESTConstants.DESCRIPTION_FIELD),
+ new Date((Long) valueMap.get(KMSRESTConstants.CREATED_FIELD)),
+ (Integer) valueMap.get(KMSRESTConstants.VERSIONS_FIELD));
+ }
+ return metadata;
+ }
+
+ private static void writeJson(Map map, OutputStream os) throws IOException {
+ Writer writer = new OutputStreamWriter(os);
+ ObjectMapper jsonMapper = new ObjectMapper();
+ jsonMapper.writerWithDefaultPrettyPrinter().writeValue(writer, map);
+ }
+
+ /**
+ * The factory to create KMSClientProvider, which is used by the
+ * ServiceLoader.
+ */
+ public static class Factory extends KeyProviderFactory {
+
+ @Override
+ public KeyProvider createProvider(URI providerName, Configuration conf)
+ throws IOException {
+ if (SCHEME_NAME.equals(providerName.getScheme())) {
+ return new KMSClientProvider(providerName, conf);
+ }
+ return null;
+ }
+ }
+
+ public static <T> T checkNotNull(T o, String name)
+ throws IllegalArgumentException {
+ if (o == null) {
+ throw new IllegalArgumentException("Parameter '" + name +
+ "' cannot be null");
+ }
+ return o;
+ }
+
+
+ public static String checkNotEmpty(String s, String name)
+ throws IllegalArgumentException {
+ checkNotNull(s, name);
+ if (s.isEmpty()) {
+ throw new IllegalArgumentException("Parameter '" + name +
+ "' cannot be empty");
+ }
+ return s;
+ }
+
+ private String kmsUrl;
+ private SSLFactory sslFactory;
+
+ public KMSClientProvider(URI uri, Configuration conf) throws IOException {
+ Path path = unnestUri(uri);
+ URL url = path.toUri().toURL();
+ kmsUrl = createServiceURL(url);
+ if ("https".equalsIgnoreCase(url.getProtocol())) {
+ sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+ try {
+ sslFactory.init();
+ } catch (GeneralSecurityException ex) {
+ throw new IOException(ex);
+ }
+ }
+ }
+
+ private String createServiceURL(URL url) throws IOException {
+ String str = url.toExternalForm();
+ if (str.endsWith("/")) {
+ str = str.substring(0, str.length() - 1);
+ }
+ return new URL(str + KMSRESTConstants.SERVICE_VERSION + "/").
+ toExternalForm();
+ }
+
+ private URL createURL(String collection, String resource, String subResource,
+ Map<String, ?> parameters) throws IOException {
+ try {
+ StringBuilder sb = new StringBuilder();
+ sb.append(kmsUrl);
+ sb.append(collection);
+ if (resource != null) {
+ sb.append("/").append(URLEncoder.encode(resource, UTF8));
+ }
+ if (subResource != null) {
+ sb.append("/").append(subResource);
+ }
+ URIBuilder uriBuilder = new URIBuilder(sb.toString());
+ if (parameters != null) {
+ for (Map.Entry<String, ?> param : parameters.entrySet()) {
+ Object value = param.getValue();
+ if (value instanceof String) {
+ uriBuilder.addParameter(param.getKey(), (String) value);
+ } else {
+ for (String s : (String[]) value) {
+ uriBuilder.addParameter(param.getKey(), s);
+ }
+ }
+ }
+ }
+ return uriBuilder.build().toURL();
+ } catch (URISyntaxException ex) {
+ throw new IOException(ex);
+ }
+ }
+
+ private HttpURLConnection configureConnection(HttpURLConnection conn)
+ throws IOException {
+ if (sslFactory != null) {
+ HttpsURLConnection httpsConn = (HttpsURLConnection) conn;
+ try {
+ httpsConn.setSSLSocketFactory(sslFactory.createSSLSocketFactory());
+ } catch (GeneralSecurityException ex) {
+ throw new IOException(ex);
+ }
+ httpsConn.setHostnameVerifier(sslFactory.getHostnameVerifier());
+ }
+ return conn;
+ }
+
+ private HttpURLConnection createConnection(URL url, String method)
+ throws IOException {
+ HttpURLConnection conn;
+ try {
+ AuthenticatedURL authUrl = new AuthenticatedURL(new PseudoAuthenticator(),
+ sslFactory);
+ conn = authUrl.openConnection(url, new AuthenticatedURL.Token());
+ } catch (AuthenticationException ex) {
+ throw new IOException(ex);
+ }
+ conn.setUseCaches(false);
+ conn.setRequestMethod(method);
+ if (method.equals(HTTP_POST) || method.equals(HTTP_PUT)) {
+ conn.setDoOutput(true);
+ }
+ conn = configureConnection(conn);
+ return conn;
+ }
+
+ // trick, riding on generics to throw an undeclared exception
+
+ private static void throwEx(Throwable ex) {
+ KMSClientProvider.<RuntimeException>throwException(ex);
+ }
+
+ @SuppressWarnings("unchecked")
+ private static <E extends Throwable> void throwException(Throwable ex)
+ throws E {
+ throw (E) ex;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static void validateResponse(HttpURLConnection conn, int expected)
+ throws IOException {
+ int status = conn.getResponseCode();
+ if (status != expected) {
+ InputStream es = null;
+ try {
+ es = conn.getErrorStream();
+ ObjectMapper mapper = new ObjectMapper();
+ Map json = mapper.readValue(es, Map.class);
+ String exClass = (String) json.get(
+ KMSRESTConstants.ERROR_EXCEPTION_JSON);
+ String exMsg = (String)
+ json.get(KMSRESTConstants.ERROR_MESSAGE_JSON);
+ Exception toThrow;
+ try {
+ ClassLoader cl = KMSClientProvider.class.getClassLoader();
+ Class klass = cl.loadClass(exClass);
+ Constructor constr = klass.getConstructor(String.class);
+ toThrow = (Exception) constr.newInstance(exMsg);
+ } catch (Exception ex) {
+ toThrow = new IOException(MessageFormat.format(
+ "HTTP status [{0}], {1}", status, conn.getResponseMessage()));
+ }
+ throwEx(toThrow);
+ } finally {
+ if (es != null) {
+ es.close();
+ }
+ }
+ }
+ }
+
+ private static <T> T call(HttpURLConnection conn, Map jsonOutput,
+ int expectedResponse, Class<T> klass)
+ throws IOException {
+ T ret = null;
+ try {
+ if (jsonOutput != null) {
+ writeJson(jsonOutput, conn.getOutputStream());
+ }
+ } catch (IOException ex) {
+ conn.getInputStream().close();
+ throw ex;
+ }
+ validateResponse(conn, expectedResponse);
+ if (APPLICATION_JSON_MIME.equalsIgnoreCase(conn.getContentType())
+ && klass != null) {
+ ObjectMapper mapper = new ObjectMapper();
+ InputStream is = null;
+ try {
+ is = conn.getInputStream();
+ ret = mapper.readValue(is, klass);
+ } catch (IOException ex) {
+ if (is != null) {
+ is.close();
+ }
+ throw ex;
+ } finally {
+ if (is != null) {
+ is.close();
+ }
+ }
+ }
+ return ret;
+ }
+
+ public static class KMSKeyVersion extends KeyVersion {
+ public KMSKeyVersion(String versionName, byte[] material) {
+ super(versionName, material);
+ }
+ }
+
+ @Override
+ public KeyVersion getKeyVersion(String versionName) throws IOException {
+ checkNotEmpty(versionName, "versionName");
+ URL url = createURL(KMSRESTConstants.KEY_VERSION_RESOURCE,
+ versionName, null, null);
+ HttpURLConnection conn = createConnection(url, HTTP_GET);
+ Map response = call(conn, null, HttpURLConnection.HTTP_OK, Map.class);
+ return parseJSONKeyVersion(response);
+ }
+
+ @Override
+ public KeyVersion getCurrentKey(String name) throws IOException {
+ checkNotEmpty(name, "name");
+ URL url = createURL(KMSRESTConstants.KEY_RESOURCE, name,
+ KMSRESTConstants.CURRENT_VERSION_SUB_RESOURCE, null);
+ HttpURLConnection conn = createConnection(url, HTTP_GET);
+ Map response = call(conn, null, HttpURLConnection.HTTP_OK, Map.class);
+ return parseJSONKeyVersion(response);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public List<String> getKeys() throws IOException {
+ URL url = createURL(KMSRESTConstants.KEYS_NAMES_RESOURCE, null, null,
+ null);
+ HttpURLConnection conn = createConnection(url, HTTP_GET);
+ List response = call(conn, null, HttpURLConnection.HTTP_OK, List.class);
+ return (List<String>) response;
+ }
+
+ public static class KMSMetadata extends Metadata {
+ public KMSMetadata(String cipher, int bitLength, String description,
+ Date created, int versions) {
+ super(cipher, bitLength, description, created, versions);
+ }
+ }
+
+ // breaking keyNames into sets to keep resulting URL undler 2000 chars
+ private List<String[]> createKeySets(String[] keyNames) {
+ List<String[]> list = new ArrayList<String[]>();
+ List<String> batch = new ArrayList<String>();
+ int batchLen = 0;
+ for (String name : keyNames) {
+ int additionalLen = KMSRESTConstants.KEY_OP.length() + 1 + name.length();
+ batchLen += additionalLen;
+ // topping at 1500 to account for initial URL and encoded names
+ if (batchLen > 1500) {
+ list.add(batch.toArray(new String[batch.size()]));
+ batch = new ArrayList<String>();
+ batchLen = additionalLen;
+ }
+ batch.add(name);
+ }
+ if (!batch.isEmpty()) {
+ list.add(batch.toArray(new String[batch.size()]));
+ }
+ return list;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Metadata[] getKeysMetadata(String ... keyNames) throws IOException {
+ List<Metadata> keysMetadata = new ArrayList<Metadata>();
+ List<String[]> keySets = createKeySets(keyNames);
+ for (String[] keySet : keySets) {
+ if (keyNames.length > 0) {
+ Map<String, Object> queryStr = new HashMap<String, Object>();
+ queryStr.put(KMSRESTConstants.KEY_OP, keySet);
+ URL url = createURL(KMSRESTConstants.KEYS_METADATA_RESOURCE, null,
+ null, queryStr);
+ HttpURLConnection conn = createConnection(url, HTTP_GET);
+ List<Map> list = call(conn, null, HttpURLConnection.HTTP_OK, List.class);
+ for (Map map : list) {
+ keysMetadata.add(parseJSONMetadata(map));
+ }
+ }
+ }
+ return keysMetadata.toArray(new Metadata[keysMetadata.size()]);
+ }
+
+ private KeyVersion createKeyInternal(String name, byte[] material,
+ Options options)
+ throws NoSuchAlgorithmException, IOException {
+ checkNotEmpty(name, "name");
+ checkNotNull(options, "options");
+ Map<String, Object> jsonKey = new HashMap<String, Object>();
+ jsonKey.put(KMSRESTConstants.NAME_FIELD, name);
+ jsonKey.put(KMSRESTConstants.CIPHER_FIELD, options.getCipher());
+ jsonKey.put(KMSRESTConstants.LENGTH_FIELD, options.getBitLength());
+ if (material != null) {
+ jsonKey.put(KMSRESTConstants.MATERIAL_FIELD,
+ Base64.encodeBase64String(material));
+ }
+ if (options.getDescription() != null) {
+ jsonKey.put(KMSRESTConstants.DESCRIPTION_FIELD,
+ options.getDescription());
+ }
+ URL url = createURL(KMSRESTConstants.KEYS_RESOURCE, null, null, null);
+ HttpURLConnection conn = createConnection(url, HTTP_POST);
+ conn.setRequestProperty(CONTENT_TYPE, APPLICATION_JSON_MIME);
+ Map response = call(conn, jsonKey, HttpURLConnection.HTTP_CREATED,
+ Map.class);
+ return parseJSONKeyVersion(response);
+ }
+
+ @Override
+ public KeyVersion createKey(String name, Options options)
+ throws NoSuchAlgorithmException, IOException {
+ return createKeyInternal(name, null, options);
+ }
+
+ @Override
+ public KeyVersion createKey(String name, byte[] material, Options options)
+ throws IOException {
+ checkNotNull(material, "material");
+ try {
+ return createKeyInternal(name, material, options);
+ } catch (NoSuchAlgorithmException ex) {
+ throw new RuntimeException("It should not happen", ex);
+ }
+ }
+
+ private KeyVersion rollNewVersionInternal(String name, byte[] material)
+ throws NoSuchAlgorithmException, IOException {
+ checkNotEmpty(name, "name");
+ Map<String, String> jsonMaterial = new HashMap<String, String>();
+ if (material != null) {
+ jsonMaterial.put(KMSRESTConstants.MATERIAL_FIELD,
+ Base64.encodeBase64String(material));
+ }
+ URL url = createURL(KMSRESTConstants.KEY_RESOURCE, name, null, null);
+ HttpURLConnection conn = createConnection(url, HTTP_POST);
+ conn.setRequestProperty(CONTENT_TYPE, APPLICATION_JSON_MIME);
+ Map response = call(conn, jsonMaterial,
+ HttpURLConnection.HTTP_OK, Map.class);
+ return parseJSONKeyVersion(response);
+ }
+
+
+ @Override
+ public KeyVersion rollNewVersion(String name)
+ throws NoSuchAlgorithmException, IOException {
+ return rollNewVersionInternal(name, null);
+ }
+
+ @Override
+ public KeyVersion rollNewVersion(String name, byte[] material)
+ throws IOException {
+ checkNotNull(material, "material");
+ try {
+ return rollNewVersionInternal(name, material);
+ } catch (NoSuchAlgorithmException ex) {
+ throw new RuntimeException("It should not happen", ex);
+ }
+ }
+
+ @Override
+ public List<KeyVersion> getKeyVersions(String name) throws IOException {
+ checkNotEmpty(name, "name");
+ URL url = createURL(KMSRESTConstants.KEY_RESOURCE, name,
+ KMSRESTConstants.VERSIONS_SUB_RESOURCE, null);
+ HttpURLConnection conn = createConnection(url, HTTP_GET);
+ List response = call(conn, null, HttpURLConnection.HTTP_OK, List.class);
+ List<KeyVersion> versions = null;
+ if (!response.isEmpty()) {
+ versions = new ArrayList<KeyVersion>();
+ for (Object obj : response) {
+ versions.add(parseJSONKeyVersion((Map) obj));
+ }
+ }
+ return versions;
+ }
+
+ @Override
+ public Metadata getMetadata(String name) throws IOException {
+ checkNotEmpty(name, "name");
+ URL url = createURL(KMSRESTConstants.KEY_RESOURCE, name,
+ KMSRESTConstants.METADATA_SUB_RESOURCE, null);
+ HttpURLConnection conn = createConnection(url, HTTP_GET);
+ Map response = call(conn, null, HttpURLConnection.HTTP_OK, Map.class);
+ return parseJSONMetadata(response);
+ }
+
+ @Override
+ public void deleteKey(String name) throws IOException {
+ checkNotEmpty(name, "name");
+ URL url = createURL(KMSRESTConstants.KEY_RESOURCE, name, null, null);
+ HttpURLConnection conn = createConnection(url, HTTP_DELETE);
+ call(conn, null, HttpURLConnection.HTTP_OK, null);
+ }
+
+ @Override
+ public void flush() throws IOException {
+ // NOP
+ // the client does not keep any local state, thus flushing is not required
+ // because of the client.
+ // the server should not keep in memory state on behalf of clients either.
+ }
+
+ @VisibleForTesting
+ public static String buildVersionName(String name, int version) {
+ return KeyProvider.buildVersionName(name, version);
+ }
+
+}
Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java?rev=1619518&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java (added)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java Thu Aug 21 18:58:53 2014
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * KMS REST and JSON constants and utility methods for the KMSServer.
+ */
+@InterfaceAudience.Private
+public class KMSRESTConstants {
+
+ public static final String SERVICE_VERSION = "/v1";
+ public static final String KEY_RESOURCE = "key";
+ public static final String KEYS_RESOURCE = "keys";
+ public static final String KEYS_METADATA_RESOURCE = KEYS_RESOURCE +
+ "/metadata";
+ public static final String KEYS_NAMES_RESOURCE = KEYS_RESOURCE + "/names";
+ public static final String KEY_VERSION_RESOURCE = "keyversion";
+ public static final String METADATA_SUB_RESOURCE = "_metadata";
+ public static final String VERSIONS_SUB_RESOURCE = "_versions";
+ public static final String CURRENT_VERSION_SUB_RESOURCE = "_currentversion";
+
+ public static final String KEY_OP = "key";
+
+ public static final String NAME_FIELD = "name";
+ public static final String CIPHER_FIELD = "cipher";
+ public static final String LENGTH_FIELD = "length";
+ public static final String DESCRIPTION_FIELD = "description";
+ public static final String CREATED_FIELD = "created";
+ public static final String VERSIONS_FIELD = "versions";
+ public static final String MATERIAL_FIELD = "material";
+ public static final String VERSION_NAME_FIELD = "versionName";
+
+ public static final String ERROR_EXCEPTION_JSON = "exception";
+ public static final String ERROR_MESSAGE_JSON = "message";
+
+}
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory?rev=1619518&r1=1619517&r2=1619518&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory Thu Aug 21 18:58:53 2014
@@ -15,3 +15,4 @@
org.apache.hadoop.crypto.key.JavaKeyStoreProvider$Factory
org.apache.hadoop.crypto.key.UserProvider$Factory
+org.apache.hadoop.crypto.key.kms.KMSClientProvider$Factory
Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/dev-support/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/dev-support/findbugsExcludeFile.xml?rev=1619518&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/dev-support/findbugsExcludeFile.xml (added)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/dev-support/findbugsExcludeFile.xml Thu Aug 21 18:58:53 2014
@@ -0,0 +1,41 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FindBugsFilter>
+ <!--
+ Findbug is complaining about System.out being NULL
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.crypto.key.kms.server.KMSWebApp"/>
+ <Bug pattern="NP_ALWAYS_NULL"/>
+ </Match>
+ <!--
+ KMSWebApp is a webapp singleton managed by the servlet container via
+ ServletContextListener.
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.crypto.key.kms.server.KMSWebApp"/>
+ <Bug pattern="ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD"/>
+ </Match>
+ <!--
+ KMSWebApp does an exit to kill the servlet container if the initialization
+ fails.
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.crypto.key.kms.server.KMSWebApp"/>
+ <Bug pattern="DM_EXIT"/>
+ </Match>
+</FindBugsFilter>
Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/pom.xml?rev=1619518&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/pom.xml (added)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/pom.xml Thu Aug 21 18:58:53 2014
@@ -0,0 +1,408 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-project</artifactId>
+ <version>2.6.0-SNAPSHOT</version>
+ <relativePath>../../hadoop-project</relativePath>
+ </parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-kms</artifactId>
+ <version>2.6.0-SNAPSHOT</version>
+ <packaging>war</packaging>
+
+ <name>Apache Hadoop KMS</name>
+ <description>Apache Hadoop KMS</description>
+
+ <properties>
+ <tomcat.version>6.0.36</tomcat.version>
+ <kms.tomcat.dist.dir>
+ ${project.build.directory}/${project.artifactId}-${project.version}/share/hadoop/kms/tomcat
+ </kms.tomcat.dist.dir>
+ <tomcat.download.url>
+ http://archive.apache.org/dist/tomcat/tomcat-6/v${tomcat.version}/bin/apache-tomcat-${tomcat.version}.tar.gz
+ </tomcat.download.url>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minikdc</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.xml.stream</groupId>
+ <artifactId>stax-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-httpclient</groupId>
+ <artifactId>commons-httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-compiler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>jsp-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-api-2.1</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jdt</groupId>
+ <artifactId>core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-el</groupId>
+ <artifactId>commons-el</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jul-to-slf4j</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <forkMode>always</forkMode>
+ <threadCount>1</threadCount>
+ <forkedProcessTimeoutInSeconds>600</forkedProcessTimeoutInSeconds>
+ <properties>
+ <property>
+ <name>listener</name>
+ <value>org.apache.hadoop.test.TimedOutTestsListener</value>
+ </property>
+ </properties>
+ <excludes>
+ <exclude>**/${test.exclude}.java</exclude>
+ <exclude>${test.exclude.pattern}</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>create-web-xmls</id>
+ <phase>generate-test-resources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <mkdir dir="${project.build.directory}/test-classes/webapp"/>
+
+ <copy todir="${project.build.directory}/test-classes/webapp">
+ <fileset dir="${basedir}/src/main/webapp"/>
+ </copy>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-war-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>default-war</id>
+ <phase>package</phase>
+ <goals>
+ <goal>war</goal>
+ </goals>
+ <configuration>
+ <warName>kms</warName>
+ <webappDirectory>${project.build.directory}/kms
+ </webappDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml
+ </excludeFilterFile>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+ <profiles>
+ <profile>
+ <id>docs</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>docs</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>site</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ <profile>
+ <id>dist</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-assemblies</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>dist</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>${project.artifactId}-${project.version}
+ </finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <attach>false</attach>
+ <descriptorRefs>
+ <descriptorRef>hadoop-kms-dist</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- Downloading Tomcat TAR.GZ, using downloads/ dir to avoid downloading over an over -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>dist</id>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <target>
+ <mkdir dir="downloads"/>
+ <get
+ src="${tomcat.download.url}"
+ dest="downloads/apache-tomcat-${tomcat.version}.tar.gz"
+ verbose="true" skipexisting="true"/>
+ <delete dir="${project.build.directory}/tomcat.exp"/>
+ <mkdir dir="${project.build.directory}/tomcat.exp"/>
+
+ <!-- Using Unix script to preserve file permissions -->
+ <echo file="${project.build.directory}/tomcat-untar.sh">
+ cd "${project.build.directory}/tomcat.exp"
+ gzip -cd ../../downloads/apache-tomcat-${tomcat.version}.tar.gz | tar xf -
+ </echo>
+ <exec executable="sh" dir="${project.build.directory}"
+ failonerror="true">
+ <arg line="./tomcat-untar.sh"/>
+ </exec>
+
+ <move
+ file="${project.build.directory}/tomcat.exp/apache-tomcat-${tomcat.version}"
+ tofile="${kms.tomcat.dist.dir}"/>
+ <delete dir="${project.build.directory}/tomcat.exp"/>
+ <delete dir="${kms.tomcat.dist.dir}/webapps"/>
+ <mkdir dir="${kms.tomcat.dist.dir}/webapps"/>
+ <delete file="${kms.tomcat.dist.dir}/conf/server.xml"/>
+ <copy file="${basedir}/src/main/tomcat/server.xml"
+ toDir="${kms.tomcat.dist.dir}/conf"/>
+ <delete file="${kms.tomcat.dist.dir}/conf/ssl-server.xml"/>
+ <copy file="${basedir}/src/main/tomcat/ssl-server.xml"
+ toDir="${kms.tomcat.dist.dir}/conf"/>
+ <delete
+ file="${kms.tomcat.dist.dir}/conf/logging.properties"/>
+ <copy file="${basedir}/src/main/tomcat/logging.properties"
+ toDir="${kms.tomcat.dist.dir}/conf"/>
+ <copy toDir="${kms.tomcat.dist.dir}/webapps/ROOT">
+ <fileset dir="${basedir}/src/main/tomcat/ROOT"/>
+ </copy>
+ <copy toDir="${kms.tomcat.dist.dir}/webapps/kms">
+ <fileset dir="${project.build.directory}/kms"/>
+ </copy>
+ </target>
+ </configuration>
+ </execution>
+ <execution>
+ <id>tar</id>
+ <phase>package</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target if="tar">
+ <!-- Using Unix script to preserve symlinks -->
+ <echo file="${project.build.directory}/dist-maketar.sh">
+ cd "${project.build.directory}"
+ tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz
+ </echo>
+ <exec executable="sh" dir="${project.build.directory}"
+ failonerror="true">
+ <arg line="./dist-maketar.sh"/>
+ </exec>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml?rev=1619518&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml (added)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml Thu Aug 21 18:58:53 2014
@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+ <!-- This file is hot-reloaded when it changes -->
+
+ <!-- KMS ACLs -->
+
+ <property>
+ <name>hadoop.kms.acl.CREATE</name>
+ <value>*</value>
+ <description>
+ ACL for create-key operations.
+ If the user does is not in the GET ACL, the key material is not returned
+ as part of the response.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.DELETE</name>
+ <value>*</value>
+ <description>
+ ACL for delete-key operations.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.ROLLOVER</name>
+ <value>*</value>
+ <description>
+ ACL for rollover-key operations.
+ If the user does is not in the GET ACL, the key material is not returned
+ as part of the response.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.GET</name>
+ <value>*</value>
+ <description>
+ ACL for get-key-version and get-current-key operations.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.GET_KEYS</name>
+ <value>*</value>
+ <description>
+ ACL for get-keys operation.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.GET_METADATA</name>
+ <value>*</value>
+ <description>
+ ACL for get-key-metadata an get-keys-metadata operations.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.SET_KEY_MATERIAL</name>
+ <value>*</value>
+ <description>
+ Complimentary ACL for CREATE and ROLLOVER operation to allow the client
+ to provide the key material when creating or rolling a key.
+ </description>
+ </property>
+
+</configuration>
Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh?rev=1619518&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh (added)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh Thu Aug 21 18:58:53 2014
@@ -0,0 +1,45 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+# Set kms specific environment variables here.
+
+# Settings for the Embedded Tomcat that runs KMS
+# Java System properties for KMS should be specified in this variable
+#
+# export CATALINA_OPTS=
+
+# KMS logs directory
+#
+# export KMS_LOG=${KMS_HOME}/logs
+
+# KMS temporary directory
+#
+# export KMS_TEMP=${KMS_HOME}/temp
+
+# The HTTP port used by KMS
+#
+# export KMS_HTTP_PORT=16000
+
+# The Admin port used by KMS
+#
+# export KMS_ADMIN_PORT=`expr ${KMS_HTTP_PORT} + 1`
+
+# The location of the SSL keystore if using SSL
+#
+# export KMS_SSL_KEYSTORE_FILE=${HOME}/.keystore
+
+# The password of the SSL keystore if using SSL
+#
+# export KMS_SSL_KEYSTORE_PASS=password
Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-log4j.properties?rev=1619518&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-log4j.properties (added)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-log4j.properties Thu Aug 21 18:58:53 2014
@@ -0,0 +1,38 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+# If the Java System property 'kms.log.dir' is not defined at KMS start up time
+# Setup sets its value to '${kms.home}/logs'
+
+log4j.appender.kms=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.kms.DatePattern='.'yyyy-MM-dd
+log4j.appender.kms.File=${kms.log.dir}/kms.log
+log4j.appender.kms.Append=true
+log4j.appender.kms.layout=org.apache.log4j.PatternLayout
+log4j.appender.kms.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n
+
+log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.kms-audit.DatePattern='.'yyyy-MM-dd
+log4j.appender.kms-audit.File=${kms.log.dir}/kms-audit.log
+log4j.appender.kms-audit.Append=true
+log4j.appender.kms-audit.layout=org.apache.log4j.PatternLayout
+log4j.appender.kms-audit.layout.ConversionPattern=%d{ISO8601} %m%n
+
+log4j.logger.kms-audit=INFO, kms-audit
+log4j.additivity.kms-audit=false
+
+log4j.rootLogger=ALL, kms
+log4j.logger.org.apache.hadoop.conf=ERROR
+log4j.logger.org.apache.hadoop=INFO
+log4j.logger.com.sun.jersey.server.wadl.generators.WadlGeneratorJAXBGrammarGenerator=OFF
\ No newline at end of file
Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml?rev=1619518&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml (added)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml Thu Aug 21 18:58:53 2014
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+ <!-- KMS Backend KeyProvider -->
+ <property>
+ <name>hadoop.security.key.provider.path</name>
+ <value>jceks://file@/${user.home}/kms.keystore</value>
+ <description>
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.keystore.JavaKeyStoreProvider.password</name>
+ <value>none</value>
+ <description>
+ </description>
+ </property>
+
+ <!-- KMS Cache -->
+ <property>
+ <name>hadoop.kms.cache.timeout.ms</name>
+ <value>10000</value>
+ <description>
+ </description>
+ </property>
+
+ <!-- KMS Security -->
+
+ <property>
+ <name>hadoop.kms.authentication.type</name>
+ <value>simple</value>
+ <description>
+ simple or kerberos
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.kerberos.keytab</name>
+ <value>${user.home}/kms.keytab</value>
+ <description>
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.kerberos.principal</name>
+ <value>HTTP/localhost</value>
+ <description>
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.kerberos.name.rules</name>
+ <value>DEFAULT</value>
+ <description>
+ </description>
+ </property>
+
+</configuration>
Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java?rev=1619518&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java (added)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java Thu Aug 21 18:58:53 2014
@@ -0,0 +1,305 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
+import org.apache.hadoop.util.StringUtils;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.Principal;
+import java.text.MessageFormat;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Class providing the REST bindings, via Jersey, for the KMS.
+ */
+@Path(KMSRESTConstants.SERVICE_VERSION)
+@InterfaceAudience.Private
+public class KMS {
+ private static final String CREATE_KEY = "CREATE_KEY";
+ private static final String DELETE_KEY = "DELETE_KEY";
+ private static final String ROLL_NEW_VERSION = "ROLL_NEW_VERSION";
+ private static final String GET_KEYS = "GET_KEYS";
+ private static final String GET_KEYS_METADATA = "GET_KEYS_METADATA";
+ private static final String GET_KEY_VERSION = "GET_KEY_VERSION";
+ private static final String GET_CURRENT_KEY = "GET_CURRENT_KEY";
+ private static final String GET_KEY_VERSIONS = "GET_KEY_VERSIONS";
+ private static final String GET_METADATA = "GET_METADATA";
+
+ private KeyProvider provider;
+
+ public KMS() throws Exception {
+ provider = KMSWebApp.getKeyProvider();
+ }
+
+ private static Principal getPrincipal(SecurityContext securityContext)
+ throws AuthenticationException{
+ Principal user = securityContext.getUserPrincipal();
+ if (user == null) {
+ throw new AuthenticationException("User must be authenticated");
+ }
+ return user;
+ }
+
+ private static void assertAccess(KMSACLs.Type aclType, Principal principal,
+ String operation, String key) throws AccessControlException {
+ if (!KMSWebApp.getACLs().hasAccess(aclType, principal.getName())) {
+ KMSWebApp.getUnauthorizedCallsMeter().mark();
+ KMSAudit.unauthorized(principal, operation, key);
+ throw new AuthorizationException(MessageFormat.format(
+ "User:{0} not allowed to do ''{1}'' on ''{2}''",
+ principal.getName(), operation, key));
+ }
+ }
+
+ private static KeyProvider.KeyVersion removeKeyMaterial(
+ KeyProvider.KeyVersion keyVersion) {
+ return new KMSClientProvider.KMSKeyVersion(keyVersion.getVersionName(),
+ null);
+ }
+
+ private static URI getKeyURI(String name) throws URISyntaxException {
+ return new URI(KMSRESTConstants.SERVICE_VERSION + "/" +
+ KMSRESTConstants.KEY_RESOURCE + "/" + name);
+ }
+
+ @POST
+ @Path(KMSRESTConstants.KEYS_RESOURCE)
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response createKey(@Context SecurityContext securityContext,
+ Map jsonKey) throws Exception {
+ KMSWebApp.getAdminCallsMeter().mark();
+ Principal user = getPrincipal(securityContext);
+ String name = (String) jsonKey.get(KMSRESTConstants.NAME_FIELD);
+ KMSClientProvider.checkNotEmpty(name, KMSRESTConstants.NAME_FIELD);
+ assertAccess(KMSACLs.Type.CREATE, user, CREATE_KEY, name);
+ String cipher = (String) jsonKey.get(KMSRESTConstants.CIPHER_FIELD);
+ String material = (String) jsonKey.get(KMSRESTConstants.MATERIAL_FIELD);
+ int length = (jsonKey.containsKey(KMSRESTConstants.LENGTH_FIELD))
+ ? (Integer) jsonKey.get(KMSRESTConstants.LENGTH_FIELD) : 0;
+ String description = (String)
+ jsonKey.get(KMSRESTConstants.DESCRIPTION_FIELD);
+
+ if (material != null) {
+ assertAccess(KMSACLs.Type.SET_KEY_MATERIAL, user,
+ CREATE_KEY + " with user provided material", name);
+ }
+ KeyProvider.Options options = new KeyProvider.Options(
+ KMSWebApp.getConfiguration());
+ if (cipher != null) {
+ options.setCipher(cipher);
+ }
+ if (length != 0) {
+ options.setBitLength(length);
+ }
+ options.setDescription(description);
+
+ KeyProvider.KeyVersion keyVersion = (material != null)
+ ? provider.createKey(name, Base64.decodeBase64(material), options)
+ : provider.createKey(name, options);
+
+ provider.flush();
+
+ KMSAudit.ok(user, CREATE_KEY, name, "UserProvidedMaterial:" +
+ (material != null) + " Description:" + description);
+
+ if (!KMSWebApp.getACLs().hasAccess(KMSACLs.Type.GET, user.getName())) {
+ keyVersion = removeKeyMaterial(keyVersion);
+ }
+ Map json = KMSServerJSONUtils.toJSON(keyVersion);
+ String requestURL = KMSMDCFilter.getURL();
+ int idx = requestURL.lastIndexOf(KMSRESTConstants.KEYS_RESOURCE);
+ requestURL = requestURL.substring(0, idx);
+ String keyURL = requestURL + KMSRESTConstants.KEY_RESOURCE + "/" + name;
+ return Response.created(getKeyURI(name)).type(MediaType.APPLICATION_JSON).
+ header("Location", keyURL).entity(json).build();
+ }
+
+ @DELETE
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}")
+ public Response deleteKey(@Context SecurityContext securityContext,
+ @PathParam("name") String name) throws Exception {
+ KMSWebApp.getAdminCallsMeter().mark();
+ Principal user = getPrincipal(securityContext);
+ assertAccess(KMSACLs.Type.DELETE, user, DELETE_KEY, name);
+ KMSClientProvider.checkNotEmpty(name, "name");
+ provider.deleteKey(name);
+ provider.flush();
+
+ KMSAudit.ok(user, DELETE_KEY, name, "");
+
+ return Response.ok().build();
+ }
+
+ @POST
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}")
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response rolloverKey(@Context SecurityContext securityContext,
+ @PathParam("name") String name, Map jsonMaterial)
+ throws Exception {
+ KMSWebApp.getAdminCallsMeter().mark();
+ Principal user = getPrincipal(securityContext);
+ assertAccess(KMSACLs.Type.ROLLOVER, user, ROLL_NEW_VERSION, name);
+ KMSClientProvider.checkNotEmpty(name, "name");
+ String material = (String)
+ jsonMaterial.get(KMSRESTConstants.MATERIAL_FIELD);
+ if (material != null) {
+ assertAccess(KMSACLs.Type.SET_KEY_MATERIAL, user,
+ ROLL_NEW_VERSION + " with user provided material", name);
+ }
+ KeyProvider.KeyVersion keyVersion = (material != null)
+ ? provider.rollNewVersion(name, Base64.decodeBase64(material))
+ : provider.rollNewVersion(name);
+
+ provider.flush();
+
+ KMSAudit.ok(user, ROLL_NEW_VERSION, name, "UserProvidedMaterial:" +
+ (material != null) + " NewVersion:" + keyVersion.getVersionName());
+
+ if (!KMSWebApp.getACLs().hasAccess(KMSACLs.Type.GET, user.getName())) {
+ keyVersion = removeKeyMaterial(keyVersion);
+ }
+ Map json = KMSServerJSONUtils.toJSON(keyVersion);
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEYS_METADATA_RESOURCE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getKeysMetadata(@Context SecurityContext securityContext,
+ @QueryParam(KMSRESTConstants.KEY_OP) List<String> keyNamesList)
+ throws Exception {
+ KMSWebApp.getAdminCallsMeter().mark();
+ Principal user = getPrincipal(securityContext);
+ String[] keyNames = keyNamesList.toArray(new String[keyNamesList.size()]);
+ String names = StringUtils.arrayToString(keyNames);
+ assertAccess(KMSACLs.Type.GET_METADATA, user, GET_KEYS_METADATA, names);
+ KeyProvider.Metadata[] keysMeta = provider.getKeysMetadata(keyNames);
+ Object json = KMSServerJSONUtils.toJSON(keyNames, keysMeta);
+ KMSAudit.ok(user, GET_KEYS_METADATA, names, "");
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEYS_NAMES_RESOURCE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getKeyNames(@Context SecurityContext securityContext)
+ throws Exception {
+ KMSWebApp.getAdminCallsMeter().mark();
+ Principal user = getPrincipal(securityContext);
+ assertAccess(KMSACLs.Type.GET_KEYS, user, GET_KEYS, "*");
+ Object json = provider.getKeys();
+ KMSAudit.ok(user, GET_KEYS, "*", "");
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}")
+ public Response getKey(@Context SecurityContext securityContext,
+ @PathParam("name") String name)
+ throws Exception {
+ return getMetadata(securityContext, name);
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" +
+ KMSRESTConstants.METADATA_SUB_RESOURCE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getMetadata(@Context SecurityContext securityContext,
+ @PathParam("name") String name)
+ throws Exception {
+ Principal user = getPrincipal(securityContext);
+ KMSClientProvider.checkNotEmpty(name, "name");
+ KMSWebApp.getAdminCallsMeter().mark();
+ assertAccess(KMSACLs.Type.GET_METADATA, user, GET_METADATA, name);
+ Object json = KMSServerJSONUtils.toJSON(name, provider.getMetadata(name));
+ KMSAudit.ok(user, GET_METADATA, name, "");
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" +
+ KMSRESTConstants.CURRENT_VERSION_SUB_RESOURCE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getCurrentVersion(@Context SecurityContext securityContext,
+ @PathParam("name") String name)
+ throws Exception {
+ Principal user = getPrincipal(securityContext);
+ KMSClientProvider.checkNotEmpty(name, "name");
+ KMSWebApp.getKeyCallsMeter().mark();
+ assertAccess(KMSACLs.Type.GET, user, GET_CURRENT_KEY, name);
+ Object json = KMSServerJSONUtils.toJSON(provider.getCurrentKey(name));
+ KMSAudit.ok(user, GET_CURRENT_KEY, name, "");
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEY_VERSION_RESOURCE + "/{versionName:.*}")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getKeyVersion(@Context SecurityContext securityContext,
+ @PathParam("versionName") String versionName)
+ throws Exception {
+ Principal user = getPrincipal(securityContext);
+ KMSClientProvider.checkNotEmpty(versionName, "versionName");
+ KMSWebApp.getKeyCallsMeter().mark();
+ assertAccess(KMSACLs.Type.GET, user, GET_KEY_VERSION, versionName);
+ Object json = KMSServerJSONUtils.toJSON(provider.getKeyVersion(versionName));
+ KMSAudit.ok(user, GET_KEY_VERSION, versionName, "");
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+ @GET
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" +
+ KMSRESTConstants.VERSIONS_SUB_RESOURCE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getKeyVersions(@Context SecurityContext securityContext,
+ @PathParam("name") String name)
+ throws Exception {
+ Principal user = getPrincipal(securityContext);
+ KMSClientProvider.checkNotEmpty(name, "name");
+ KMSWebApp.getKeyCallsMeter().mark();
+ assertAccess(KMSACLs.Type.GET, user, GET_KEY_VERSIONS, name);
+ Object json = KMSServerJSONUtils.toJSON(provider.getKeyVersions(name));
+ KMSAudit.ok(user, GET_KEY_VERSIONS, name, "");
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ }
+
+}
Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java?rev=1619518&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java (added)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java Thu Aug 21 18:58:53 2014
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * Provides access to the <code>AccessControlList</code>s used by KMS,
+ * hot-reloading them if the <code>kms-acls.xml</code> file where the ACLs
+ * are defined has been updated.
+ */
+public class KMSACLs implements Runnable {
+ private static final Logger LOG = LoggerFactory.getLogger(KMSACLs.class);
+
+
+ public enum Type {
+ CREATE, DELETE, ROLLOVER, GET, GET_KEYS, GET_METADATA, SET_KEY_MATERIAL;
+
+ public String getConfigKey() {
+ return KMSConfiguration.CONFIG_PREFIX + "acl." + this.toString();
+ }
+ }
+
+ public static final String ACL_DEFAULT = AccessControlList.WILDCARD_ACL_VALUE;
+
+ public static final int RELOADER_SLEEP_MILLIS = 1000;
+
+ Map<Type, AccessControlList> acls;
+ private ReadWriteLock lock;
+ private ScheduledExecutorService executorService;
+ private long lastReload;
+
+ KMSACLs(Configuration conf) {
+ lock = new ReentrantReadWriteLock();
+ if (conf == null) {
+ conf = loadACLs();
+ }
+ setACLs(conf);
+ }
+
+ public KMSACLs() {
+ this(null);
+ }
+
+ private void setACLs(Configuration conf) {
+ lock.writeLock().lock();
+ try {
+ acls = new HashMap<Type, AccessControlList>();
+ for (Type aclType : Type.values()) {
+ String aclStr = conf.get(aclType.getConfigKey(), ACL_DEFAULT);
+ acls.put(aclType, new AccessControlList(aclStr));
+ LOG.info("'{}' ACL '{}'", aclType, aclStr);
+ }
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ @Override
+ public void run() {
+ try {
+ if (KMSConfiguration.isACLsFileNewer(lastReload)) {
+ setACLs(loadACLs());
+ }
+ } catch (Exception ex) {
+ LOG.warn("Could not reload ACLs file: " + ex.toString(), ex);
+ }
+ }
+
+ public synchronized void startReloader() {
+ if (executorService == null) {
+ executorService = Executors.newScheduledThreadPool(1);
+ executorService.scheduleAtFixedRate(this, RELOADER_SLEEP_MILLIS,
+ RELOADER_SLEEP_MILLIS, TimeUnit.MILLISECONDS);
+ }
+ }
+
+ public synchronized void stopReloader() {
+ if (executorService != null) {
+ executorService.shutdownNow();
+ executorService = null;
+ }
+ }
+
+ private Configuration loadACLs() {
+ LOG.debug("Loading ACLs file");
+ lastReload = System.currentTimeMillis();
+ Configuration conf = KMSConfiguration.getACLsConf();
+ // triggering the resource loading.
+ conf.get(Type.CREATE.getConfigKey());
+ return conf;
+ }
+
+ public boolean hasAccess(Type type, String user) {
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
+ AccessControlList acl = null;
+ lock.readLock().lock();
+ try {
+ acl = acls.get(type);
+ } finally {
+ lock.readLock().unlock();
+ }
+ return acl.isUserAllowed(ugi);
+ }
+
+}
Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java?rev=1619518&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java (added)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java Thu Aug 21 18:58:53 2014
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.security.Principal;
+
+/**
+ * Provides convenience methods for audit logging consistently the different
+ * types of events.
+ */
+public class KMSAudit {
+ public static final String KMS_LOGGER_NAME = "kms-audit";
+
+ private static Logger AUDIT_LOG = LoggerFactory.getLogger(KMS_LOGGER_NAME);
+
+ private static void op(String status, String op, Principal user, String key,
+ String extraMsg) {
+ AUDIT_LOG.info("Status:{} User:{} Op:{} Name:{}{}", status, user.getName(),
+ op, key, extraMsg);
+ }
+
+ public static void ok(Principal user, String op, String key,
+ String extraMsg) {
+ op("OK", op, user, key, extraMsg);
+ }
+
+ public static void unauthorized(Principal user, String op, String key) {
+ op("UNAUTHORIZED", op, user, key, "");
+ }
+
+ public static void error(Principal user, String method, String url,
+ String extraMsg) {
+ AUDIT_LOG.info("Status:ERROR User:{} Method:{} URL:{} Exception:'{}'",
+ user.getName(), method, url, extraMsg);
+ }
+
+ public static void unauthenticated(String remoteHost, String method,
+ String url, String extraMsg) {
+ AUDIT_LOG.info(
+ "Status:UNAUTHENTICATED RemoteHost:{} Method:{} URL:{} ErrorMsg:'{}'",
+ remoteHost, method, url, extraMsg);
+ }
+
+}
Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java?rev=1619518&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java (added)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java Thu Aug 21 18:58:53 2014
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
+
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.servlet.http.HttpServletResponseWrapper;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Authentication filter that takes the configuration from the KMS configuration
+ * file.
+ */
+@InterfaceAudience.Private
+public class KMSAuthenticationFilter extends AuthenticationFilter {
+ private static final String CONF_PREFIX = KMSConfiguration.CONFIG_PREFIX +
+ "authentication.";
+
+ @Override
+ protected Properties getConfiguration(String configPrefix,
+ FilterConfig filterConfig) {
+ Properties props = new Properties();
+ Configuration conf = KMSWebApp.getConfiguration();
+ for (Map.Entry<String, String> entry : conf) {
+ String name = entry.getKey();
+ if (name.startsWith(CONF_PREFIX)) {
+ String value = conf.get(name);
+ name = name.substring(CONF_PREFIX.length());
+ props.setProperty(name, value);
+ }
+ }
+ return props;
+ }
+
+ private static class KMSResponse extends HttpServletResponseWrapper {
+ public int statusCode;
+ public String msg;
+
+ public KMSResponse(ServletResponse response) {
+ super((HttpServletResponse)response);
+ }
+
+ @Override
+ public void setStatus(int sc) {
+ statusCode = sc;
+ super.setStatus(sc);
+ }
+
+ @Override
+ public void sendError(int sc, String msg) throws IOException {
+ statusCode = sc;
+ this.msg = msg;
+ super.sendError(sc, msg);
+ }
+
+ @Override
+ public void sendError(int sc) throws IOException {
+ statusCode = sc;
+ super.sendError(sc);
+ }
+
+ @Override
+ public void setStatus(int sc, String sm) {
+ statusCode = sc;
+ msg = sm;
+ super.setStatus(sc, sm);
+ }
+ }
+
+ @Override
+ public void doFilter(ServletRequest request, ServletResponse response,
+ FilterChain filterChain) throws IOException, ServletException {
+ KMSResponse kmsResponse = new KMSResponse(response);
+ super.doFilter(request, kmsResponse, filterChain);
+
+ if (kmsResponse.statusCode != HttpServletResponse.SC_OK &&
+ kmsResponse.statusCode != HttpServletResponse.SC_CREATED &&
+ kmsResponse.statusCode != HttpServletResponse.SC_UNAUTHORIZED) {
+ KMSWebApp.getInvalidCallsMeter().mark();
+ }
+
+ // HttpServletResponse.SC_UNAUTHORIZED is because the request does not
+ // belong to an authenticated user.
+ if (kmsResponse.statusCode == HttpServletResponse.SC_UNAUTHORIZED) {
+ KMSWebApp.getUnauthenticatedCallsMeter().mark();
+ String method = ((HttpServletRequest) request).getMethod();
+ StringBuffer requestURL = ((HttpServletRequest) request).getRequestURL();
+ String queryString = ((HttpServletRequest) request).getQueryString();
+ if (queryString != null) {
+ requestURL.append("?").append(queryString);
+ }
+ KMSAudit.unauthenticated(request.getRemoteHost(), method,
+ requestURL.toString(), kmsResponse.msg);
+ }
+ }
+
+}