You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cm...@apache.org on 2014/08/20 01:50:11 UTC
svn commit: r1619012 [2/14] - in
/hadoop/common/branches/HADOOP-10388/hadoop-common-project: ./ hadoop-auth/
hadoop-auth/dev-support/
hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/
hadoop-auth/src/main/java/org/apache/hadoo...
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java Tue Aug 19 23:49:39 2014
@@ -17,18 +17,27 @@
*/
package org.apache.hadoop.security.authentication.util;
+import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
+
+import java.io.File;
+import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
import java.util.Locale;
+import java.util.Set;
+import java.util.regex.Pattern;
+import org.apache.directory.server.kerberos.shared.keytab.Keytab;
+import org.apache.directory.server.kerberos.shared.keytab.KeytabEntry;
import org.ietf.jgss.GSSException;
import org.ietf.jgss.Oid;
-import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
-
public class KerberosUtil {
/* Return the Kerberos login module name */
@@ -103,4 +112,48 @@ public class KerberosUtil {
// with uppercase characters.
return service + "/" + fqdn.toLowerCase(Locale.US);
}
+
+ /**
+ * Get all the unique principals present in the keytabfile.
+ *
+ * @param keytabFileName
+ * Name of the keytab file to be read.
+ * @return list of unique principals in the keytab.
+ * @throws IOException
+ * If keytab entries cannot be read from the file.
+ */
+ static final String[] getPrincipalNames(String keytabFileName) throws IOException {
+ Keytab keytab = Keytab.read(new File(keytabFileName));
+ Set<String> principals = new HashSet<String>();
+ List<KeytabEntry> entries = keytab.getEntries();
+ for (KeytabEntry entry: entries){
+ principals.add(entry.getPrincipalName().replace("\\", "/"));
+ }
+ return principals.toArray(new String[0]);
+ }
+
+ /**
+ * Get all the unique principals from keytabfile which matches a pattern.
+ *
+ * @param keytab
+ * Name of the keytab file to be read.
+ * @param pattern
+ * pattern to be matched.
+ * @return list of unique principals which matches the pattern.
+ * @throws IOException
+ */
+ public static final String[] getPrincipalNames(String keytab,
+ Pattern pattern) throws IOException {
+ String[] principals = getPrincipalNames(keytab);
+ if (principals.length != 0) {
+ List<String> matchingPrincipals = new ArrayList<String>();
+ for (String principal : principals) {
+ if (pattern.matcher(principal).matches()) {
+ matchingPrincipals.add(principal);
+ }
+ }
+ principals = matchingPrincipals.toArray(new String[0]);
+ }
+ return principals;
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/Signer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/Signer.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/Signer.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/Signer.java Tue Aug 19 23:49:39 2014
@@ -24,18 +24,19 @@ import java.security.NoSuchAlgorithmExce
public class Signer {
private static final String SIGNATURE = "&s=";
- private byte[] secret;
+ private SignerSecretProvider secretProvider;
/**
- * Creates a Signer instance using the specified secret.
+ * Creates a Signer instance using the specified SignerSecretProvider. The
+ * SignerSecretProvider should already be initialized.
*
- * @param secret secret to use for creating the digest.
+ * @param secretProvider The SignerSecretProvider to use
*/
- public Signer(byte[] secret) {
- if (secret == null) {
- throw new IllegalArgumentException("secret cannot be NULL");
+ public Signer(SignerSecretProvider secretProvider) {
+ if (secretProvider == null) {
+ throw new IllegalArgumentException("secretProvider cannot be NULL");
}
- this.secret = secret.clone();
+ this.secretProvider = secretProvider;
}
/**
@@ -47,11 +48,12 @@ public class Signer {
*
* @return the signed string.
*/
- public String sign(String str) {
+ public synchronized String sign(String str) {
if (str == null || str.length() == 0) {
throw new IllegalArgumentException("NULL or empty string to sign");
}
- String signature = computeSignature(str);
+ byte[] secret = secretProvider.getCurrentSecret();
+ String signature = computeSignature(secret, str);
return str + SIGNATURE + signature;
}
@@ -71,21 +73,19 @@ public class Signer {
}
String originalSignature = signedStr.substring(index + SIGNATURE.length());
String rawValue = signedStr.substring(0, index);
- String currentSignature = computeSignature(rawValue);
- if (!originalSignature.equals(currentSignature)) {
- throw new SignerException("Invalid signature");
- }
+ checkSignatures(rawValue, originalSignature);
return rawValue;
}
/**
* Returns then signature of a string.
*
+ * @param secret The secret to use
* @param str string to sign.
*
* @return the signature for the string.
*/
- protected String computeSignature(String str) {
+ protected String computeSignature(byte[] secret, String str) {
try {
MessageDigest md = MessageDigest.getInstance("SHA");
md.update(str.getBytes());
@@ -97,4 +97,22 @@ public class Signer {
}
}
+ protected void checkSignatures(String rawValue, String originalSignature)
+ throws SignerException {
+ boolean isValid = false;
+ byte[][] secrets = secretProvider.getAllSecrets();
+ for (int i = 0; i < secrets.length; i++) {
+ byte[] secret = secrets[i];
+ if (secret != null) {
+ String currentSignature = computeSignature(secret, rawValue);
+ if (originalSignature.equals(currentSignature)) {
+ isValid = true;
+ break;
+ }
+ }
+ }
+ if (!isValid) {
+ throw new SignerException("Invalid signature");
+ }
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/site/apt/BuildingIt.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/site/apt/BuildingIt.apt.vm?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/site/apt/BuildingIt.apt.vm (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/site/apt/BuildingIt.apt.vm Tue Aug 19 23:49:39 2014
@@ -18,8 +18,6 @@
Hadoop Auth, Java HTTP SPNEGO ${project.version} - Building It
- \[ {{{./index.html}Go Back}} \]
-
* Requirements
* Java 6+
@@ -70,6 +68,3 @@ $ mvn package -Pdocs
The generated documentation is available at
<<<hadoop-auth/target/site/>>>.
-
- \[ {{{./index.html}Go Back}} \]
-
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/site/apt/Configuration.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/site/apt/Configuration.apt.vm?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/site/apt/Configuration.apt.vm (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/site/apt/Configuration.apt.vm Tue Aug 19 23:49:39 2014
@@ -20,8 +20,6 @@
Hadoop Auth, Java HTTP SPNEGO ${project.version} - Server Side
Configuration
- \[ {{{./index.html}Go Back}} \]
-
* Server Side Configuration Setup
The AuthenticationFilter filter is Hadoop Auth's server side component.
@@ -241,5 +239,3 @@ Configuration
...
</web-app>
+---+
-
- \[ {{{./index.html}Go Back}} \]
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/site/apt/Examples.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/site/apt/Examples.apt.vm?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/site/apt/Examples.apt.vm (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/site/apt/Examples.apt.vm Tue Aug 19 23:49:39 2014
@@ -18,8 +18,6 @@
Hadoop Auth, Java HTTP SPNEGO ${project.version} - Examples
- \[ {{{./index.html}Go Back}} \]
-
* Accessing a Hadoop Auth protected URL Using a browser
<<IMPORTANT:>> The browser must support HTTP Kerberos SPNEGO. For example,
@@ -133,5 +131,3 @@ You are: user[tucu] principal[tucu@LOCAL
....
+---+
-
- \[ {{{./index.html}Go Back}} \]
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java Tue Aug 19 23:49:39 2014
@@ -33,36 +33,6 @@ public class TestAuthenticatedURL {
token = new AuthenticatedURL.Token("foo");
Assert.assertTrue(token.isSet());
Assert.assertEquals("foo", token.toString());
-
- AuthenticatedURL.Token token1 = new AuthenticatedURL.Token();
- AuthenticatedURL.Token token2 = new AuthenticatedURL.Token();
- Assert.assertEquals(token1.hashCode(), token2.hashCode());
- Assert.assertTrue(token1.equals(token2));
-
- token1 = new AuthenticatedURL.Token();
- token2 = new AuthenticatedURL.Token("foo");
- Assert.assertNotSame(token1.hashCode(), token2.hashCode());
- Assert.assertFalse(token1.equals(token2));
-
- token1 = new AuthenticatedURL.Token("foo");
- token2 = new AuthenticatedURL.Token();
- Assert.assertNotSame(token1.hashCode(), token2.hashCode());
- Assert.assertFalse(token1.equals(token2));
-
- token1 = new AuthenticatedURL.Token("foo");
- token2 = new AuthenticatedURL.Token("foo");
- Assert.assertEquals(token1.hashCode(), token2.hashCode());
- Assert.assertTrue(token1.equals(token2));
-
- token1 = new AuthenticatedURL.Token("bar");
- token2 = new AuthenticatedURL.Token("foo");
- Assert.assertNotSame(token1.hashCode(), token2.hashCode());
- Assert.assertFalse(token1.equals(token2));
-
- token1 = new AuthenticatedURL.Token("foo");
- token2 = new AuthenticatedURL.Token("bar");
- Assert.assertNotSame(token1.hashCode(), token2.hashCode());
- Assert.assertFalse(token1.equals(token2));
}
@Test
@@ -137,4 +107,12 @@ public class TestAuthenticatedURL {
Mockito.verify(connConf).configure(Mockito.<HttpURLConnection>any());
}
+ @Test
+ public void testGetAuthenticator() throws Exception {
+ Authenticator authenticator = Mockito.mock(Authenticator.class);
+
+ AuthenticatedURL aURL = new AuthenticatedURL(authenticator);
+ Assert.assertEquals(authenticator, aURL.getAuthenticator());
+ }
+
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java Tue Aug 19 23:49:39 2014
@@ -23,6 +23,7 @@ import java.util.Vector;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
+import javax.servlet.ServletContext;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
@@ -33,6 +34,8 @@ import javax.servlet.http.HttpServletRes
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.util.Signer;
+import org.apache.hadoop.security.authentication.util.SignerSecretProvider;
+import org.apache.hadoop.security.authentication.util.StringSignerSecretProvider;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
@@ -74,6 +77,8 @@ public class TestAuthenticationFilter {
Assert.fail();
} catch (ServletException ex) {
// Expected
+ Assert.assertEquals("Authentication type must be specified: simple|kerberos|<class>",
+ ex.getMessage());
} catch (Exception ex) {
Assert.fail();
} finally {
@@ -155,9 +160,14 @@ public class TestAuthenticationFilter {
Mockito.when(config.getInitParameterNames()).thenReturn(
new Vector<String>(Arrays.asList(AuthenticationFilter.AUTH_TYPE,
AuthenticationFilter.AUTH_TOKEN_VALIDITY)).elements());
+ ServletContext context = Mockito.mock(ServletContext.class);
+ Mockito.when(context.getAttribute(
+ AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null);
+ Mockito.when(config.getServletContext()).thenReturn(context);
filter.init(config);
Assert.assertEquals(PseudoAuthenticationHandler.class, filter.getAuthenticationHandler().getClass());
Assert.assertTrue(filter.isRandomSecret());
+ Assert.assertFalse(filter.isCustomSignerSecretProvider());
Assert.assertNull(filter.getCookieDomain());
Assert.assertNull(filter.getCookiePath());
Assert.assertEquals(TOKEN_VALIDITY_SEC, filter.getValidity());
@@ -165,6 +175,26 @@ public class TestAuthenticationFilter {
filter.destroy();
}
+ // string secret
+ filter = new AuthenticationFilter();
+ try {
+ FilterConfig config = Mockito.mock(FilterConfig.class);
+ Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn("simple");
+ Mockito.when(config.getInitParameter(AuthenticationFilter.SIGNATURE_SECRET)).thenReturn("secret");
+ Mockito.when(config.getInitParameterNames()).thenReturn(
+ new Vector<String>(Arrays.asList(AuthenticationFilter.AUTH_TYPE,
+ AuthenticationFilter.SIGNATURE_SECRET)).elements());
+ ServletContext context = Mockito.mock(ServletContext.class);
+ Mockito.when(context.getAttribute(
+ AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null);
+ Mockito.when(config.getServletContext()).thenReturn(context);
+ filter.init(config);
+ Assert.assertFalse(filter.isRandomSecret());
+ Assert.assertFalse(filter.isCustomSignerSecretProvider());
+ } finally {
+ filter.destroy();
+ }
+
// custom secret
filter = new AuthenticationFilter();
try {
@@ -174,8 +204,26 @@ public class TestAuthenticationFilter {
Mockito.when(config.getInitParameterNames()).thenReturn(
new Vector<String>(Arrays.asList(AuthenticationFilter.AUTH_TYPE,
AuthenticationFilter.SIGNATURE_SECRET)).elements());
+ ServletContext context = Mockito.mock(ServletContext.class);
+ Mockito.when(context.getAttribute(
+ AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(
+ new SignerSecretProvider() {
+ @Override
+ public void init(Properties config, long tokenValidity) {
+ }
+ @Override
+ public byte[] getCurrentSecret() {
+ return null;
+ }
+ @Override
+ public byte[][] getAllSecrets() {
+ return null;
+ }
+ });
+ Mockito.when(config.getServletContext()).thenReturn(context);
filter.init(config);
Assert.assertFalse(filter.isRandomSecret());
+ Assert.assertTrue(filter.isCustomSignerSecretProvider());
} finally {
filter.destroy();
}
@@ -191,6 +239,10 @@ public class TestAuthenticationFilter {
new Vector<String>(Arrays.asList(AuthenticationFilter.AUTH_TYPE,
AuthenticationFilter.COOKIE_DOMAIN,
AuthenticationFilter.COOKIE_PATH)).elements());
+ ServletContext context = Mockito.mock(ServletContext.class);
+ Mockito.when(context.getAttribute(
+ AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null);
+ Mockito.when(config.getServletContext()).thenReturn(context);
filter.init(config);
Assert.assertEquals(".foo.com", filter.getCookieDomain());
Assert.assertEquals("/bar", filter.getCookiePath());
@@ -211,6 +263,10 @@ public class TestAuthenticationFilter {
new Vector<String>(
Arrays.asList(AuthenticationFilter.AUTH_TYPE,
"management.operation.return")).elements());
+ ServletContext context = Mockito.mock(ServletContext.class);
+ Mockito.when(context.getAttribute(
+ AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null);
+ Mockito.when(config.getServletContext()).thenReturn(context);
filter.init(config);
Assert.assertTrue(DummyAuthenticationHandler.init);
} finally {
@@ -233,6 +289,31 @@ public class TestAuthenticationFilter {
filter.destroy();
}
}
+
+ @Test
+ public void testInitCaseSensitivity() throws Exception {
+ // minimal configuration & simple auth handler (Pseudo)
+ AuthenticationFilter filter = new AuthenticationFilter();
+ try {
+ FilterConfig config = Mockito.mock(FilterConfig.class);
+ Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn("SimPle");
+ Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TOKEN_VALIDITY)).thenReturn(
+ (new Long(TOKEN_VALIDITY_SEC)).toString());
+ Mockito.when(config.getInitParameterNames()).thenReturn(
+ new Vector<String>(Arrays.asList(AuthenticationFilter.AUTH_TYPE,
+ AuthenticationFilter.AUTH_TOKEN_VALIDITY)).elements());
+ ServletContext context = Mockito.mock(ServletContext.class);
+ Mockito.when(context.getAttribute(
+ AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null);
+ Mockito.when(config.getServletContext()).thenReturn(context);
+
+ filter.init(config);
+ Assert.assertEquals(PseudoAuthenticationHandler.class,
+ filter.getAuthenticationHandler().getClass());
+ } finally {
+ filter.destroy();
+ }
+ }
@Test
public void testGetRequestURL() throws Exception {
@@ -247,6 +328,10 @@ public class TestAuthenticationFilter {
new Vector<String>(
Arrays.asList(AuthenticationFilter.AUTH_TYPE,
"management.operation.return")).elements());
+ ServletContext context = Mockito.mock(ServletContext.class);
+ Mockito.when(context.getAttribute(
+ AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null);
+ Mockito.when(config.getServletContext()).thenReturn(context);
filter.init(config);
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
@@ -274,11 +359,15 @@ public class TestAuthenticationFilter {
Arrays.asList(AuthenticationFilter.AUTH_TYPE,
AuthenticationFilter.SIGNATURE_SECRET,
"management.operation.return")).elements());
+ ServletContext context = Mockito.mock(ServletContext.class);
+ Mockito.when(context.getAttribute(
+ AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null);
+ Mockito.when(config.getServletContext()).thenReturn(context);
filter.init(config);
AuthenticationToken token = new AuthenticationToken("u", "p", DummyAuthenticationHandler.TYPE);
token.setExpires(System.currentTimeMillis() + TOKEN_VALIDITY_SEC);
- Signer signer = new Signer("secret".getBytes());
+ Signer signer = new Signer(new StringSignerSecretProvider("secret"));
String tokenSigned = signer.sign(token.toString());
Cookie cookie = new Cookie(AuthenticatedURL.AUTH_COOKIE, tokenSigned);
@@ -307,12 +396,16 @@ public class TestAuthenticationFilter {
Arrays.asList(AuthenticationFilter.AUTH_TYPE,
AuthenticationFilter.SIGNATURE_SECRET,
"management.operation.return")).elements());
+ ServletContext context = Mockito.mock(ServletContext.class);
+ Mockito.when(context.getAttribute(
+ AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null);
+ Mockito.when(config.getServletContext()).thenReturn(context);
filter.init(config);
AuthenticationToken token =
new AuthenticationToken("u", "p", DummyAuthenticationHandler.TYPE);
token.setExpires(System.currentTimeMillis() - TOKEN_VALIDITY_SEC);
- Signer signer = new Signer("secret".getBytes());
+ Signer signer = new Signer(new StringSignerSecretProvider("secret"));
String tokenSigned = signer.sign(token.toString());
Cookie cookie = new Cookie(AuthenticatedURL.AUTH_COOKIE, tokenSigned);
@@ -348,11 +441,15 @@ public class TestAuthenticationFilter {
Arrays.asList(AuthenticationFilter.AUTH_TYPE,
AuthenticationFilter.SIGNATURE_SECRET,
"management.operation.return")).elements());
+ ServletContext context = Mockito.mock(ServletContext.class);
+ Mockito.when(context.getAttribute(
+ AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null);
+ Mockito.when(config.getServletContext()).thenReturn(context);
filter.init(config);
AuthenticationToken token = new AuthenticationToken("u", "p", "invalidtype");
token.setExpires(System.currentTimeMillis() + TOKEN_VALIDITY_SEC);
- Signer signer = new Signer("secret".getBytes());
+ Signer signer = new Signer(new StringSignerSecretProvider("secret"));
String tokenSigned = signer.sign(token.toString());
Cookie cookie = new Cookie(AuthenticatedURL.AUTH_COOKIE, tokenSigned);
@@ -386,6 +483,10 @@ public class TestAuthenticationFilter {
new Vector<String>(
Arrays.asList(AuthenticationFilter.AUTH_TYPE,
"management.operation.return")).elements());
+ ServletContext context = Mockito.mock(ServletContext.class);
+ Mockito.when(context.getAttribute(
+ AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null);
+ Mockito.when(config.getServletContext()).thenReturn(context);
filter.init(config);
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
@@ -435,6 +536,10 @@ public class TestAuthenticationFilter {
AuthenticationFilter.AUTH_TOKEN_VALIDITY,
AuthenticationFilter.SIGNATURE_SECRET, "management.operation" +
".return", "expired.token")).elements());
+ ServletContext context = Mockito.mock(ServletContext.class);
+ Mockito.when(context.getAttribute(
+ AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null);
+ Mockito.when(config.getServletContext()).thenReturn(context);
if (withDomainPath) {
Mockito.when(config.getInitParameter(AuthenticationFilter
@@ -488,7 +593,7 @@ public class TestAuthenticationFilter {
Mockito.verify(chain).doFilter(Mockito.any(ServletRequest.class),
Mockito.any(ServletResponse.class));
- Signer signer = new Signer("secret".getBytes());
+ Signer signer = new Signer(new StringSignerSecretProvider("secret"));
String value = signer.verifyAndExtract(v);
AuthenticationToken token = AuthenticationToken.parse(value);
assertThat(token.getExpires(), not(0L));
@@ -508,21 +613,17 @@ public class TestAuthenticationFilter {
private static void parseCookieMap(String cookieHeader, HashMap<String,
String> cookieMap) {
- for (String pair : cookieHeader.split(";")) {
- String p = pair.trim();
- int idx = p.indexOf('=');
- final String k, v;
- if (idx == -1) {
- k = p;
- v = null;
- } else if (idx == p.length()) {
- k = p.substring(0, idx - 1);
- v = null;
- } else {
- k = p.substring(0, idx);
- v = p.substring(idx + 1);
+ List<HttpCookie> cookies = HttpCookie.parse(cookieHeader);
+ for (HttpCookie cookie : cookies) {
+ if (AuthenticatedURL.AUTH_COOKIE.equals(cookie.getName())) {
+ cookieMap.put(cookie.getName(), cookie.getValue());
+ if (cookie.getPath() != null) {
+ cookieMap.put("Path", cookie.getPath());
+ }
+ if (cookie.getDomain() != null) {
+ cookieMap.put("Domain", cookie.getDomain());
+ }
}
- cookieMap.put(k, v);
}
}
@@ -559,6 +660,10 @@ public class TestAuthenticationFilter {
new Vector<String>(
Arrays.asList(AuthenticationFilter.AUTH_TYPE,
"management.operation.return")).elements());
+ ServletContext context = Mockito.mock(ServletContext.class);
+ Mockito.when(context.getAttribute(
+ AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null);
+ Mockito.when(config.getServletContext()).thenReturn(context);
filter.init(config);
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
@@ -566,7 +671,7 @@ public class TestAuthenticationFilter {
AuthenticationToken token = new AuthenticationToken("u", "p", "t");
token.setExpires(System.currentTimeMillis() + TOKEN_VALIDITY_SEC);
- Signer signer = new Signer("secret".getBytes());
+ Signer signer = new Signer(new StringSignerSecretProvider("secret"));
String tokenSigned = signer.sign(token.toString());
Cookie cookie = new Cookie(AuthenticatedURL.AUTH_COOKIE, tokenSigned);
@@ -609,6 +714,10 @@ public class TestAuthenticationFilter {
new Vector<String>(
Arrays.asList(AuthenticationFilter.AUTH_TYPE,
"management.operation.return")).elements());
+ ServletContext context = Mockito.mock(ServletContext.class);
+ Mockito.when(context.getAttribute(
+ AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null);
+ Mockito.when(config.getServletContext()).thenReturn(context);
filter.init(config);
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
@@ -672,6 +781,10 @@ public class TestAuthenticationFilter {
Arrays.asList(AuthenticationFilter.AUTH_TYPE,
AuthenticationFilter.SIGNATURE_SECRET,
"management.operation.return")).elements());
+ ServletContext context = Mockito.mock(ServletContext.class);
+ Mockito.when(context.getAttribute(
+ AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null);
+ Mockito.when(config.getServletContext()).thenReturn(context);
filter.init(config);
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
@@ -679,7 +792,7 @@ public class TestAuthenticationFilter {
AuthenticationToken token = new AuthenticationToken("u", "p", DummyAuthenticationHandler.TYPE);
token.setExpires(System.currentTimeMillis() - TOKEN_VALIDITY_SEC);
- Signer signer = new Signer(secret.getBytes());
+ Signer signer = new Signer(new StringSignerSecretProvider(secret));
String tokenSigned = signer.sign(token.toString());
Cookie cookie = new Cookie(AuthenticatedURL.AUTH_COOKIE, tokenSigned);
@@ -739,6 +852,10 @@ public class TestAuthenticationFilter {
Arrays.asList(AuthenticationFilter.AUTH_TYPE,
AuthenticationFilter.SIGNATURE_SECRET,
"management.operation.return")).elements());
+ ServletContext context = Mockito.mock(ServletContext.class);
+ Mockito.when(context.getAttribute(
+ AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null);
+ Mockito.when(config.getServletContext()).thenReturn(context);
filter.init(config);
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
@@ -746,7 +863,7 @@ public class TestAuthenticationFilter {
AuthenticationToken token = new AuthenticationToken("u", "p", "invalidtype");
token.setExpires(System.currentTimeMillis() + TOKEN_VALIDITY_SEC);
- Signer signer = new Signer(secret.getBytes());
+ Signer signer = new Signer(new StringSignerSecretProvider(secret));
String tokenSigned = signer.sign(token.toString());
Cookie cookie = new Cookie(AuthenticatedURL.AUTH_COOKIE, tokenSigned);
@@ -774,6 +891,10 @@ public class TestAuthenticationFilter {
new Vector<String>(
Arrays.asList(AuthenticationFilter.AUTH_TYPE,
"management.operation.return")).elements());
+ ServletContext context = Mockito.mock(ServletContext.class);
+ Mockito.when(context.getAttribute(
+ AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null);
+ Mockito.when(config.getServletContext()).thenReturn(context);
filter.init(config);
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
@@ -793,7 +914,7 @@ public class TestAuthenticationFilter {
AuthenticationToken token = new AuthenticationToken("u", "p", "t");
token.setExpires(System.currentTimeMillis() + TOKEN_VALIDITY_SEC);
- Signer signer = new Signer("secret".getBytes());
+ Signer signer = new Signer(new StringSignerSecretProvider("secret"));
String tokenSigned = signer.sign(token.toString());
Cookie cookie = new Cookie(AuthenticatedURL.AUTH_COOKIE, tokenSigned);
Mockito.when(request.getCookies()).thenReturn(new Cookie[]{cookie});
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java Tue Aug 19 23:49:39 2014
@@ -18,6 +18,7 @@ import org.apache.hadoop.security.authen
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.security.authentication.util.KerberosUtil;
import org.ietf.jgss.GSSContext;
@@ -30,10 +31,18 @@ import org.junit.Test;
import org.mockito.Mockito;
import org.ietf.jgss.Oid;
+import javax.security.auth.Subject;
+import javax.security.auth.kerberos.KerberosPrincipal;
+import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+
import java.io.File;
+import java.security.Principal;
+import java.util.Arrays;
+import java.util.List;
import java.util.Properties;
+import java.util.Set;
import java.util.concurrent.Callable;
public class TestKerberosAuthenticationHandler
@@ -110,8 +119,65 @@ public class TestKerberosAuthenticationH
@Test(timeout=60000)
public void testInit() throws Exception {
- Assert.assertEquals(KerberosTestUtils.getServerPrincipal(), handler.getPrincipal());
Assert.assertEquals(KerberosTestUtils.getKeytabFile(), handler.getKeytab());
+ Set<KerberosPrincipal> principals = handler.getPrincipals();
+ Principal expectedPrincipal =
+ new KerberosPrincipal(KerberosTestUtils.getServerPrincipal());
+ Assert.assertTrue(principals.contains(expectedPrincipal));
+ Assert.assertEquals(1, principals.size());
+ }
+
+ // dynamic configuration of HTTP principals
+ @Test(timeout=60000)
+ public void testDynamicPrincipalDiscovery() throws Exception {
+ String[] keytabUsers = new String[]{
+ "HTTP/host1", "HTTP/host2", "HTTP2/host1", "XHTTP/host"
+ };
+ String keytab = KerberosTestUtils.getKeytabFile();
+ getKdc().createPrincipal(new File(keytab), keytabUsers);
+
+ // destroy handler created in setUp()
+ handler.destroy();
+ Properties props = new Properties();
+ props.setProperty(KerberosAuthenticationHandler.KEYTAB, keytab);
+ props.setProperty(KerberosAuthenticationHandler.PRINCIPAL, "*");
+ handler = getNewAuthenticationHandler();
+ handler.init(props);
+
+ Assert.assertEquals(KerberosTestUtils.getKeytabFile(), handler.getKeytab());
+
+ Set<KerberosPrincipal> loginPrincipals = handler.getPrincipals();
+ for (String user : keytabUsers) {
+ Principal principal = new KerberosPrincipal(
+ user + "@" + KerberosTestUtils.getRealm());
+ boolean expected = user.startsWith("HTTP/");
+ Assert.assertEquals("checking for "+user, expected,
+ loginPrincipals.contains(principal));
+ }
+ }
+
+ // dynamic configuration of HTTP principals
+ @Test(timeout=60000)
+ public void testDynamicPrincipalDiscoveryMissingPrincipals() throws Exception {
+ String[] keytabUsers = new String[]{"hdfs/localhost"};
+ String keytab = KerberosTestUtils.getKeytabFile();
+ getKdc().createPrincipal(new File(keytab), keytabUsers);
+
+ // destroy handler created in setUp()
+ handler.destroy();
+ Properties props = new Properties();
+ props.setProperty(KerberosAuthenticationHandler.KEYTAB, keytab);
+ props.setProperty(KerberosAuthenticationHandler.PRINCIPAL, "*");
+ handler = getNewAuthenticationHandler();
+ try {
+ handler.init(props);
+ Assert.fail("init should have failed");
+ } catch (ServletException ex) {
+ Assert.assertEquals("Principals do not exist in the keytab",
+ ex.getCause().getMessage());
+ } catch (Throwable t) {
+ Assert.fail("wrong exception: "+t);
+ }
}
@Test(timeout=60000)
@@ -190,7 +256,8 @@ public class TestKerberosAuthenticationH
Mockito.when(request.getHeader(KerberosAuthenticator.AUTHORIZATION))
.thenReturn(KerberosAuthenticator.NEGOTIATE + " " + token);
-
+ Mockito.when(request.getServerName()).thenReturn("localhost");
+
AuthenticationToken authToken = handler.authenticate(request, response);
if (authToken != null) {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java Tue Aug 19 23:49:39 2014
@@ -91,6 +91,22 @@ public class TestKerberosName {
checkBadTranslation("root/joe@FOO.COM");
}
+ @Test
+ public void testToLowerCase() throws Exception {
+ String rules =
+ "RULE:[1:$1]/L\n" +
+ "RULE:[2:$1]/L\n" +
+ "RULE:[2:$1;$2](^.*;admin$)s/;admin$///L\n" +
+ "RULE:[2:$1;$2](^.*;guest$)s/;guest$//g/L\n" +
+ "DEFAULT";
+ KerberosName.setRules(rules);
+ KerberosName.printRules();
+ checkTranslation("Joe@FOO.COM", "joe");
+ checkTranslation("Joe/root@FOO.COM", "joe");
+ checkTranslation("Joe/admin@FOO.COM", "joe");
+ checkTranslation("Joe/guestguest@FOO.COM", "joe");
+ }
+
@After
public void clear() {
System.clearProperty("java.security.krb5.realm");
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosUtil.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosUtil.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosUtil.java Tue Aug 19 23:49:39 2014
@@ -16,13 +16,39 @@
*/
package org.apache.hadoop.security.authentication.util;
-import org.junit.Assert;
-
+import java.io.File;
import java.io.IOException;
-
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import org.apache.directory.server.kerberos.shared.keytab.Keytab;
+import org.apache.directory.server.kerberos.shared.keytab.KeytabEntry;
+import org.apache.directory.shared.kerberos.KerberosTime;
+import org.apache.directory.shared.kerberos.codec.types.EncryptionType;
+import org.apache.directory.shared.kerberos.components.EncryptionKey;
+import org.junit.After;
+import org.junit.Assert;
import org.junit.Test;
public class TestKerberosUtil {
+ static String testKeytab = "test.keytab";
+ static String[] testPrincipals = new String[]{
+ "HTTP@testRealm",
+ "test/testhost@testRealm",
+ "HTTP/testhost@testRealm",
+ "HTTP1/testhost@testRealm",
+ "HTTP/testhostanother@testRealm"
+ };
+
+ @After
+ public void deleteKeytab() {
+ File keytabFile = new File(testKeytab);
+ if (keytabFile.exists()){
+ keytabFile.delete();
+ }
+ }
@Test
public void testGetServerPrincipal() throws IOException {
@@ -51,4 +77,84 @@ public class TestKerberosUtil {
service + "/" + testHost.toLowerCase(),
KerberosUtil.getServicePrincipal(service, testHost.toLowerCase()));
}
+
+ @Test
+ public void testGetPrincipalNamesMissingKeytab() {
+ try {
+ KerberosUtil.getPrincipalNames(testKeytab);
+ Assert.fail("Exception should have been thrown");
+ } catch (IOException e) {
+ //expects exception
+ }
+ }
+
+ @Test
+ public void testGetPrincipalNamesMissingPattern() throws IOException {
+ createKeyTab(testKeytab, new String[]{"test/testhost@testRealm"});
+ try {
+ KerberosUtil.getPrincipalNames(testKeytab, null);
+ Assert.fail("Exception should have been thrown");
+ } catch (Exception e) {
+ //expects exception
+ }
+ }
+
+ @Test
+ public void testGetPrincipalNamesFromKeytab() throws IOException {
+ createKeyTab(testKeytab, testPrincipals);
+ // read all principals in the keytab file
+ String[] principals = KerberosUtil.getPrincipalNames(testKeytab);
+ Assert.assertNotNull("principals cannot be null", principals);
+
+ int expectedSize = 0;
+ List<String> principalList = Arrays.asList(principals);
+ for (String principal : testPrincipals) {
+ Assert.assertTrue("missing principal "+principal,
+ principalList.contains(principal));
+ expectedSize++;
+ }
+ Assert.assertEquals(expectedSize, principals.length);
+ }
+
+ @Test
+ public void testGetPrincipalNamesFromKeytabWithPattern() throws IOException {
+ createKeyTab(testKeytab, testPrincipals);
+ // read the keytab file
+ // look for principals with HTTP as the first part
+ Pattern httpPattern = Pattern.compile("HTTP/.*");
+ String[] httpPrincipals =
+ KerberosUtil.getPrincipalNames(testKeytab, httpPattern);
+ Assert.assertNotNull("principals cannot be null", httpPrincipals);
+
+ int expectedSize = 0;
+ List<String> httpPrincipalList = Arrays.asList(httpPrincipals);
+ for (String principal : testPrincipals) {
+ if (httpPattern.matcher(principal).matches()) {
+ Assert.assertTrue("missing principal "+principal,
+ httpPrincipalList.contains(principal));
+ expectedSize++;
+ }
+ }
+ Assert.assertEquals(expectedSize, httpPrincipals.length);
+ }
+
+ private void createKeyTab(String fileName, String[] principalNames)
+ throws IOException {
+ //create a test keytab file
+ List<KeytabEntry> lstEntries = new ArrayList<KeytabEntry>();
+ for (String principal : principalNames){
+ // create 3 versions of the key to ensure methods don't return
+ // duplicate principals
+ for (int kvno=1; kvno <= 3; kvno++) {
+ EncryptionKey key = new EncryptionKey(
+ EncryptionType.UNKNOWN, "samplekey1".getBytes(), kvno);
+ KeytabEntry keytabEntry = new KeytabEntry(
+ principal, 1 , new KerberosTime(), (byte) 1, key);
+ lstEntries.add(keytabEntry);
+ }
+ }
+ Keytab keytab = Keytab.getInstance();
+ keytab.setEntries(lstEntries);
+ keytab.write(new File(testKeytab));
+ }
}
\ No newline at end of file
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestSigner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestSigner.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestSigner.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestSigner.java Tue Aug 19 23:49:39 2014
@@ -13,24 +13,15 @@
*/
package org.apache.hadoop.security.authentication.util;
+import java.util.Properties;
import org.junit.Assert;
import org.junit.Test;
public class TestSigner {
@Test
- public void testNoSecret() throws Exception {
- try {
- new Signer(null);
- Assert.fail();
- }
- catch (IllegalArgumentException ex) {
- }
- }
-
- @Test
public void testNullAndEmptyString() throws Exception {
- Signer signer = new Signer("secret".getBytes());
+ Signer signer = new Signer(new StringSignerSecretProvider("secret"));
try {
signer.sign(null);
Assert.fail();
@@ -51,17 +42,17 @@ public class TestSigner {
@Test
public void testSignature() throws Exception {
- Signer signer = new Signer("secret".getBytes());
+ Signer signer = new Signer(new StringSignerSecretProvider("secret"));
String s1 = signer.sign("ok");
String s2 = signer.sign("ok");
String s3 = signer.sign("wrong");
Assert.assertEquals(s1, s2);
- Assert.assertNotSame(s1, s3);
+ Assert.assertNotEquals(s1, s3);
}
@Test
public void testVerify() throws Exception {
- Signer signer = new Signer("secret".getBytes());
+ Signer signer = new Signer(new StringSignerSecretProvider("secret"));
String t = "test";
String s = signer.sign(t);
String e = signer.verifyAndExtract(s);
@@ -70,7 +61,7 @@ public class TestSigner {
@Test
public void testInvalidSignedText() throws Exception {
- Signer signer = new Signer("secret".getBytes());
+ Signer signer = new Signer(new StringSignerSecretProvider("secret"));
try {
signer.verifyAndExtract("test");
Assert.fail();
@@ -83,7 +74,7 @@ public class TestSigner {
@Test
public void testTampering() throws Exception {
- Signer signer = new Signer("secret".getBytes());
+ Signer signer = new Signer(new StringSignerSecretProvider("secret"));
String t = "test";
String s = signer.sign(t);
s += "x";
@@ -96,4 +87,66 @@ public class TestSigner {
Assert.fail();
}
}
+
+ @Test
+ public void testMultipleSecrets() throws Exception {
+ TestSignerSecretProvider secretProvider = new TestSignerSecretProvider();
+ Signer signer = new Signer(secretProvider);
+ secretProvider.setCurrentSecret("secretB");
+ String t1 = "test";
+ String s1 = signer.sign(t1);
+ String e1 = signer.verifyAndExtract(s1);
+ Assert.assertEquals(t1, e1);
+ secretProvider.setPreviousSecret("secretA");
+ String t2 = "test";
+ String s2 = signer.sign(t2);
+ String e2 = signer.verifyAndExtract(s2);
+ Assert.assertEquals(t2, e2);
+ Assert.assertEquals(s1, s2); //check is using current secret for signing
+ secretProvider.setCurrentSecret("secretC");
+ secretProvider.setPreviousSecret("secretB");
+ String t3 = "test";
+ String s3 = signer.sign(t3);
+ String e3 = signer.verifyAndExtract(s3);
+ Assert.assertEquals(t3, e3);
+ Assert.assertNotEquals(s1, s3); //check not using current secret for signing
+ String e1b = signer.verifyAndExtract(s1);
+ Assert.assertEquals(t1, e1b); // previous secret still valid
+ secretProvider.setCurrentSecret("secretD");
+ secretProvider.setPreviousSecret("secretC");
+ try {
+ signer.verifyAndExtract(s1); // previous secret no longer valid
+ Assert.fail();
+ } catch (SignerException ex) {
+ // Expected
+ }
+ }
+
+ class TestSignerSecretProvider extends SignerSecretProvider {
+
+ private byte[] currentSecret;
+ private byte[] previousSecret;
+
+ @Override
+ public void init(Properties config, long tokenValidity) {
+ }
+
+ @Override
+ public byte[] getCurrentSecret() {
+ return currentSecret;
+ }
+
+ @Override
+ public byte[][] getAllSecrets() {
+ return new byte[][]{currentSecret, previousSecret};
+ }
+
+ public void setCurrentSecret(String secretStr) {
+ currentSecret = secretStr.getBytes();
+ }
+
+ public void setPreviousSecret(String previousSecretStr) {
+ previousSecret = previousSecretStr.getBytes();
+ }
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/CHANGES.txt Tue Aug 19 23:49:39 2014
@@ -7,8 +7,27 @@ Trunk (Unreleased)
HADOOP-8124. Remove the deprecated FSDataOutputStream constructor,
FSDataOutputStream.sync() and Syncable.sync(). (szetszwo)
+ HADOOP-10474 Move o.a.h.record to hadoop-streaming. (wheat9)
+
+ HADOOP-9902. Shell script rewrite (aw)
+
NEW FEATURES
+
+ HADOOP-10433. Key Management Server based on KeyProvider API. (tucu)
+
+ HADOOP-9629. Support Windows Azure Storage - Blob as a file system in Hadoop.
+ (Dexter Bradshaw, Mostafa Elhemali, Xi Fang, Johannes Klein, David Lao,
+ Mike Liddell, Chuan Liu, Lengning Liu, Ivan Mitic, Michael Rys,
+ Alexander Stojanovic, Brian Swan, and Min Wei via cnauroth)
+
+ HADOOP-10728. Metrics system for Windows Azure Storage Filesystem.
+ (Dexter Bradshaw, Mostafa Elhemali, Xi Fang, Johannes Klein, David Lao,
+ Mike Liddell, Chuan Liu, Lengning Liu, Ivan Mitic, Michael Rys,
+ Alexander Stojanovich, Brian Swan, and Min Wei via cnauroth)
+ HADOOP-10719. Add generateEncryptedKey and decryptEncryptedKey
+ methods to KeyProvider. (asuresh via tucu)
+
IMPROVEMENTS
HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution
@@ -19,12 +38,8 @@ Trunk (Unreleased)
HADOOP-7595. Upgrade dependency to Avro 1.5.3. (Alejandro Abdelnur via atm)
- HADOOP-7664. Remove warmings when overriding final parameter configuration
- if the override value is same as the final parameter value.
- (Ravi Prakash via suresh)
-
- HADOOP-8078. Add capability to turn on security in unit tests. (Jaimin Jetly
- via jitendra)
+ HADOOP-8078. Add capability to turn on security in unit tests. (Jaimin
+ Jetly via jitendra)
HADOOP-7757. Test file reference count is at least 3x actual value (Jon
Eagles via bobby)
@@ -121,9 +136,6 @@ Trunk (Unreleased)
HADOOP-10342. Add a new method to UGI to use a Kerberos login subject to
build a new UGI. (Larry McCay via omalley)
- HADOOP-9968. Makes ProxyUsers to work with NetGroups (Benoy Antony via
- ddas)
-
HADOOP-10237. JavaKeyStoreProvider needs to set keystore permissions
correctly. (Larry McCay via omalley)
@@ -141,6 +153,61 @@ Trunk (Unreleased)
HADOOP-10430. KeyProvider Metadata should have an optional description,
there should be a method to retrieve the metadata from all keys. (tucu)
+ HADOOP-10534. KeyProvider getKeysMetadata should take a list of names
+ rather than returning all keys. (omalley)
+
+ HADOOP-10563. Remove the dependency of jsp in trunk. (wheat9)
+
+ HADOOP-10485. Remove dead classes in hadoop-streaming. (wheat9)
+
+ HADOOP-10696. Add optional attributes to KeyProvider Options and Metadata.
+ (tucu)
+
+ HADOOP-10695. KMSClientProvider should respect a configurable timeout.
+ (yoderme via tucu)
+
+ HADOOP-10757. KeyProvider KeyVersion should provide the key name.
+ (asuresh via tucu)
+
+ HADOOP-10769. Create KeyProvider extension to handle delegation tokens.
+ (Arun Suresh via atm)
+
+ HADOOP-10812. Delegate KeyProviderExtension#toString to underlying
+ KeyProvider. (wang)
+
+ HADOOP-10736. Add key attributes to the key shell. (Mike Yoder via wang)
+
+ HADOOP-10824. Refactor KMSACLs to avoid locking. (Benoy Antony via umamahesh)
+
+ HADOOP-10841. EncryptedKeyVersion should have a key name property.
+ (asuresh via tucu)
+
+ HADOOP-10842. CryptoExtension generateEncryptedKey method should
+ receive the key name. (asuresh via tucu)
+
+ HADOOP-10750. KMSKeyProviderCache should be in hadoop-common.
+ (asuresh via tucu)
+
+ HADOOP-10720. KMS: Implement generateEncryptedKey and decryptEncryptedKey
+ in the REST API. (asuresh via tucu)
+
+ HADOOP-10891. Add EncryptedKeyVersion factory method to
+ KeyProviderCryptoExtension. (wang)
+
+ HADOOP-10756. KMS audit log should consolidate successful similar requests.
+ (asuresh via tucu)
+
+ HADOOP-10793. KeyShell args should use single-dash style. (wang)
+
+ HADOOP-10936. Change default KeyProvider bitlength to 128. (wang)
+
+ HADOOP-10224. JavaKeyStoreProvider has to protect against corrupting
+ underlying store. (asuresh via tucu)
+
+ HADOOP-10770. KMS add delegation token support. (tucu)
+
+ HADOOP-10698. KMS, add proxyuser support. (tucu)
+
BUG FIXES
HADOOP-9451. Fault single-layer config if node group topology is enabled.
@@ -289,9 +356,6 @@ Trunk (Unreleased)
HADOOP-9394. Port findHangingTest.sh from HBase to Hadoop. (Andrew Wang
via atm)
- HADOOP-9099. NetUtils.normalizeHostName fails on domains where
- UnknownHost resolves to an IP address. (Ivan Mitic via suresh)
-
HADOOP-9431 TestSecurityUtil#testLocalHostNameForNullOrWild on systems where hostname
contains capital letters (Chris Nauroth via sanjay)
@@ -319,22 +383,264 @@ Trunk (Unreleased)
HADOOP-10431. Change visibility of KeyStore.Options getter methods to public. (tucu)
+ HADOOP-10583. bin/hadoop key throws NPE with no args and assorted other fixups. (clamb via tucu)
+
+ HADOOP-10586. KeyShell doesn't allow setting Options via CLI. (clamb via tucu)
+
+ HADOOP-10625. Trim configuration names when putting/getting them
+ to properties. (Wangda Tan via xgong)
+
+ HADOOP-10645. TestKMS fails because race condition writing acl files. (tucu)
+
+ HADOOP-10611. KMS, keyVersion name should not be assumed to be
+ keyName@versionNumber. (tucu)
+
+ HADOOP-10717. HttpServer2 should load jsp DTD from local jars instead of
+ going remote. (Dapeng Sun via wheat9)
+
+ HADOOP-10689. InputStream is not closed in
+ AzureNativeFileSystemStore#retrieve(). (Chen He via cnauroth)
+
+ HADOOP-10690. Lack of synchronization on access to InputStream in
+ NativeAzureFileSystem#NativeAzureFsInputStream#close().
+ (Chen He via cnauroth)
+
+ HADOOP-10831. UserProvider is not thread safe. (Benoy Antony via umamahesh)
+
+ HADOOP-10834. Typo in CredentialShell usage. (Benoy Antony via umamahesh)
+
+ HADOOP-10816. KeyShell returns -1 on error to the shell, should be 1.
+ (Mike Yoder via wang)
+
+ HADOOP-10840. Fix OutOfMemoryError caused by metrics system in Azure File
+ System. (Shanyu Zhao via cnauroth)
+
+ HADOOP-10826. Iteration on KeyProviderFactory.serviceLoader is
+ thread-unsafe. (benoyantony viat tucu)
+
+ HADOOP-10881. Clarify usage of encryption and encrypted encryption
+ key in KeyProviderCryptoExtension. (wang)
+
+ HADOOP-10920. site plugin couldn't parse hadoop-kms index.apt.vm.
+ (Akira Ajisaka via wang)
+
+ HADOOP-10925. Compilation fails in native link0 function on Windows.
+ (cnauroth)
+
+ HADOOP-10939. Fix TestKeyProviderFactory testcases to use default 128 bit
+ length keys. (Arun Suresh via wang)
+
+ HADOOP-10862. Miscellaneous trivial corrections to KMS classes.
+ (asuresh via tucu)
+
+ HADOOP-10967. Improve DefaultCryptoExtension#generateEncryptedKey
+ performance. (hitliuyi via tucu)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
HADOOP-8589. ViewFs tests fail when tests and home dirs are nested (sanjay Radia)
-Release 2.5.0 - UNRELEASED
+Release 2.6.0 - UNRELEASED
INCOMPATIBLE CHANGES
- HADOOP-10474 Move o.a.h.record to hadoop-streaming. (wheat9)
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ HADOOP-10808. Remove unused native code for munlock. (cnauroth)
+
+ HADOOP-10815. Implement Windows equivalent of mlock. (cnauroth)
+
+ HADOOP-7664. Remove warmings when overriding final parameter configuration
+ if the override value is same as the final parameter value.
+ (Ravi Prakash via suresh)
+
+ HADOOP-10673. Update rpc metrics when the call throws an exception. (Ming Ma
+ via jing9)
+
+ HADOOP-10845. Add common tests for ACLs in combination with viewfs.
+ (Stephen Chu via cnauroth)
+
+ HADOOP-10839. Add unregisterSource() to MetricsSystem API.
+ (Shanyu Zhao via cnauroth)
+
+ HADOOP-10607. Create an API to separate credentials/password storage
+ from applications (Larry McCay via omalley)
+
+ HADOOP-10732. Fix locking in credential update. (Ted Yu via omalley)
+
+ HADOOP-10733. Fix potential null dereference in CredShell. (Ted Yu via
+ omalley)
+
+ HADOOP-10610. Upgrade S3n s3.fs.buffer.dir to support multi directories.
+ (Ted Malaska via atm)
+
+ HADOOP-10817. ProxyUsers configuration should support configurable
+ prefixes. (tucu)
+
+ HADOOP-10755. Support negative caching of user-group mapping.
+ (Lei Xu via wang)
+
+ HADOOP-10855. Allow Text to be read with a known Length. (todd)
+
+ HADOOP-10887. Add XAttrs to ViewFs and make XAttrs + ViewFileSystem
+ internal dir behavior consistent. (Stephen Chu via wang)
+
+ HADOOP-10882. Move DirectBufferPool into common util. (todd)
+
+ HADOOP-8069. Enable TCP_NODELAY by default for IPC. (Todd Lipcon via
+ Arpit Agarwal)
+
+ HADOOP-10902. Deletion of directories with snapshots will not output
+ reason for trash move failure. (Stephen Chu via wang)
+
+ HADOOP-10900. CredentialShell args should use single-dash style. (wang)
+
+ HADOOP-10903. Enhance hadoop classpath command to expand wildcards or write
+ classpath into jar manifest. (cnauroth)
+
+ HADOOP-10791. AuthenticationFilter should support externalizing the
+ secret for signing and provide rotation support. (rkanter via tucu)
+
+ HADOOP-10771. Refactor HTTP delegation support out of httpfs to common.
+ (tucu)
+
+ HADOOP-10835. Implement HTTP proxyuser support in HTTP authentication
+ client/server libraries. (tucu)
+
+ HADOOP-10820. Throw an exception in GenericOptionsParser when passed
+ an empty Path. (Alex Holmes and Zhihai Xu via wang)
+
+ HADOOP-10281. Create a scheduler, which assigns schedulables a priority
+ level. (Chris Li via Arpit Agarwal)
+
+ HADOOP-8944. Shell command fs -count should include human readable option
+ (Jonathan Allen via aw)
+
+ HADOOP-10231. Add some components in Native Libraries document (Akira
+ AJISAKA via aw)
+
+ HADOOP-10650. Add ability to specify a reverse ACL (black list) of users
+ and groups. (Benoy Antony via Arpit Agarwal)
+
+ HADOOP-10335. An ip whilelist based implementation to resolve Sasl
+ properties per connection. (Benoy Antony via Arpit Agarwal)
+
+ HADOOP-10975. org.apache.hadoop.util.DataChecksum should support calculating
+ checksums in native code (James Thomas via Colin Patrick McCabe)
+
+ OPTIMIZATIONS
+
+ HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
+
+ BUG FIXES
+
+ HADOOP-10781. Unportable getgrouplist() usage breaks FreeBSD (Dmitry
+ Sivachenko via Colin Patrick McCabe)
+
+ HADOOP-10507. FsShell setfacl can throw ArrayIndexOutOfBoundsException when
+ no perm is specified. (Stephen Chu and Sathish Gurram via cnauroth)
+
+ HADOOP-10780. hadoop_user_info_alloc fails on FreeBSD due to incorrect
+ sysconf use (Dmitry Sivachenko via Colin Patrick McCabe)
+
+ HADOOP-10810. Clean up native code compilation warnings. (cnauroth)
+
+ HADOOP-9921. daemon scripts should remove pid file on stop call after stop
+ or process is found not running ( vinayakumarb )
+
+ HADOOP-10591. Compression codecs must used pooled direct buffers or
+ deallocate direct buffers when stream is closed (cmccabe)
+
+ HADOOP-10857. Native Libraries Guide doen't mention a dependency on
+ openssl-development package (ozawa via cmccabe)
+
+ HADOOP-10866. RawLocalFileSystem fails to read symlink targets via the stat
+ command when the format of the stat command uses non-curly quotes (yzhang
+ via cmccabe)
+
+ HADOOP-10830. Missing lock in JavaKeyStoreProvider.createCredentialEntry.
+ (Benoy Antony via umamahesh)
+
+ HADOOP-10928. Incorrect usage on `hadoop credential list`.
+ (Josh Elser via wang)
+
+ HADOOP-10927. Fix CredentialShell help behavior and error codes.
+ (Josh Elser via wang)
+
+ HADOOP-10937. Need to set version name correctly before decrypting EEK.
+ (Arun Suresh via wang)
+
+ HADOOP-10918. JMXJsonServlet fails when used within Tomcat. (tucu)
+
+ HADOOP-10933. FileBasedKeyStoresFactory Should use Configuration.getPassword
+ for SSL Passwords. (lmccay via tucu)
+
+ HADOOP-10759. Remove hardcoded JAVA_HEAP_MAX. (Sam Liu via Eric Yang)
+
+ HADOOP-10905. LdapGroupsMapping Should use configuration.getPassword for SSL
+ and LDAP Passwords. (lmccay via brandonli)
+
+ HADOOP-10931 compile error on tools/hadoop-openstack (xukun via stevel)
+
+ HADOOP-10929. Typo in Configuration.getPasswordFromCredentialProviders
+ (lmccay via brandonli)
+
+ HADOOP-10402. Configuration.getValByRegex does not substitute for
+ variables. (Robert Kanter via kasha)
+
+ HADOOP-10851. NetgroupCache does not remove group memberships. (Benoy
+ Antony via Arpit Agarwal)
+
+ HADOOP-10962. Flags for posix_fadvise are not valid in some architectures
+ (David Villegas via Colin Patrick McCabe)
+
+ HADOOP-10966. Hadoop Common native compilation broken in windows.
+ (David Villegas via Arpit Agarwal)
+
+ HADOOP-10843. TestGridmixRecord unit tests failure on PowerPC (Jinghui Wang
+ via Colin Patrick McCabe)
+
+ HADOOP-10121. Fix javadoc spelling for HadoopArchives#writeTopLevelDirs
+ (Akira AJISAKA via aw)
+
+ HADOOP-10964. Small fix for NetworkTopologyWithNodeGroup#sortByDistance.
+ (Yi Liu via wang)
+
+ HADOOP-10059. RPC authentication and authorization metrics overflow to
+ negative values on busy clusters (Tsuyoshi OZAWA and Akira AJISAKA
+ via jlowe)
+
+ HADOOP-10973. Native Libraries Guide contains format error. (Peter Klavins
+ via Arpit Agarwal)
+
+ HADOOP-10972. Native Libraries Guide contains mis-spelt build line (Peter
+ Klavins via aw)
+
+ HADOOP-10873. Fix dead link in Configuration javadoc (Akira AJISAKA
+ via aw)
+
+ HADOOP-10968. hadoop native build fails to detect java_libarch on
+ ppc64le (Dinar Valeev via Colin Patrick McCabe)
+
+Release 2.5.0 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
NEW FEATURES
HADOOP-10498. Add support for proxy server. (daryn)
+ HADOOP-9704. Write metrics sink plugin for Hadoop/Graphite (Chu Tong, Alex Newman and Babak Behzad via raviprak)
+
+ HADOOP-8943. Support multiple group mapping providers. (Kai Zheng via brandonli)
+
+ HADOOP-9361 Strictly define the expected behavior of filesystem APIs and
+ write tests to verify compliance (stevel)
+
IMPROVEMENTS
HADOOP-10451. Remove unused field and imports from SaslRpcServer.
@@ -348,10 +654,116 @@ Release 2.5.0 - UNRELEASED
HADOOP-10104. Update jackson to 1.9.13 (Akira Ajisaka via stevel)
- HADOOP-10485. Remove dead classes in hadoop-streaming. (wheat9)
+ HADOOP-10503. Move junit up to v 4.11. (cnauroth)
+
+ HADOOP-10535. Make the retry numbers in ActiveStandbyElector configurable.
+ (jing9)
+
+ HADOOP-10322. Add ability to read principal names from a keytab.
+ (Benoy Antony and Daryn Sharp via kihwal)
+
+ HADOOP-10549. MAX_SUBST and varPat should be final in Configuration.java.
+ (Gera Shegalov via cnauroth)
+
+ HADOOP-10471. Reduce the visibility of constants in ProxyUsers.
+ (Benoy Antony via wheat9)
+
+ HADOOP-10556. Add toLowerCase support to auth_to_local rules
+ for service name. (tucu)
+
+ HADOOP-10467. Enable proxyuser specification to support list of users in
+ addition to list of groups (Benoy Antony via Arpit Agarwal)
+
+ HADOOP-10158. SPNEGO should work with multiple interfaces/SPNs.
+ (daryn via kihwal)
+
+ HADOOP-10566. Refactor proxyservers out of ProxyUsers.
+ (Benoy Antony via suresh)
+
+ HADOOP-10572. Example NFS mount command must pass noacl as it isn't
+ supported by the server yet. (Harsh J via brandonli)
+
+ HADOOP-10609. .gitignore should ignore .orig and .rej files. (kasha)
+
+ HADOOP-10614. CBZip2InputStream is not threadsafe (Xiangrui Meng via
+ Sandy Ryza)
+
+ HADOOP-10618. Remove SingleNodeSetup.apt.vm. (Akira Ajisaka via
+ Arpit Agarwal)
+
+ HADOOP-9968. Makes ProxyUsers to work with NetGroups (Benoy Antony via
+ ddas)
+
+ HADOOP-10448. Support pluggable mechanism to specify proxy user settings.
+ (Benoy Antony via Arpit Agarwal)
+
+ HADOOP-9555. HA functionality that uses ZooKeeper may experience inadvertent
+ TCP RST and miss session expiration event due to bug in client connection
+ management. (cnauroth)
+
+ HADOOP-10376. Refactor refresh*Protocols into a single generic
+ refreshConfigProtocol. (Chris Li via Arpit Agarwal)
+
+ HADOOP-6350. Documenting Hadoop metrics. (Akira Ajisaka via Arpit Agarwal)
+
+ HADOOP-10691. Improve the readability of 'hadoop fs -help'.
+ (Lei Xu via wang)
+
+ HADOOP-10688. Expose thread-level FileSystem StatisticsData (Sandy Ryza)
+
+ HADOOP-10657. Have RetryInvocationHandler log failover attempt at INFO
+ level. (Ming Ma via jing9)
+
+ HADOOP-10666. Remove Copyright /d/d/d/d Apache Software Foundation from
+ the source files license header. (Henry Saputra via wang)
+
+ HADOOP-10557. FsShell -cp -pa option for preserving extended ACLs.
+ (Akira Ajisaka via cnauroth)
+
+ HADOOP-10279. Create multiplexer, a requirement for the fair queue.
+ (Chris Li via Arpit Agarwal)
+
+ HADOOP-10659. Refactor AccessControlList to reuse utility functions
+ and to improve performance. (Benoy Antony via Arpit Agarwal)
+
+ HADOOP-10665. Make Hadoop Authentication Handler loads case in-sensitive
+ (Benoy Antony via vinayakumarb)
+
+ HADOOP-10652. Refactor Proxyusers to use AccessControlList. (Benoy
+ Antony via Arpit Agarwal)
+
+ HADOOP-10747. Support configurable retries on SASL connection failures in
+ RPC client. (cnauroth)
+
+ HADOOP-10754. Reenable several HA ZooKeeper-related tests on Windows.
+ (cnauroth)
+
+ HADOOP-10565. Support IP ranges (CIDR) in proxyuser.hosts. (Benoy Antony
+ via Arpit Agarwal)
+
+ HADOOP-10649. Allow overriding the default ACL for service authorization
+ (Benoy Antony via Arpit Agarwal)
+
+ HADOOP-10767. Clean up unused code in Ls shell command. (cnauroth)
+
+ HADOOP-9651 Filesystems to throw FileAlreadyExistsException in
+ createFile(path, overwrite=false) when the file exists (stevel)
+
+ HADOOP-9495 Define behaviour of Seekable.seek(), write tests,
+ fix all hadoop implementations for compliance
+
+ HADOOP-10312 Shell.ExitCodeException to have more useful toString (stevel)
+
+ HADOOP-10782. Fix typo in DataChecksum class. (Jingguo Yao via suresh)
+
+ HADOOP-10896. Update compatibility doc to capture visibility of
+ un-annotated classes/ methods. (kasha)
OPTIMIZATIONS
+ HADOOP-10674. Improve PureJavaCrc32 performance and use java.util.zip.CRC32
+ for Java 7 and above. (szetszwo)
+
BUG FIXES
HADOOP-10378. Typo in help printed by hdfs dfs -help.
@@ -393,7 +805,167 @@ Release 2.5.0 - UNRELEASED
HADOOP-10499. Remove unused parameter from ProxyUsers.authorize().
(Benoy Antony via cnauroth)
-Release 2.4.1 - UNRELEASED
+ HADOOP-9919. Update hadoop-metrics2.properties examples to Yarn.
+ (Akira AJISAKA via suresh)
+
+ HADOOP-10526. Chance for Stream leakage in CompressorStream. (Rushabh
+ Shah via kihwal)
+
+ HADOOP-10251. Both NameNodes could be in STANDBY State if SNN network is unstable
+ (Vinayakumar B via umamahesh)
+
+ HADOOP-10531. hadoop-config.sh - bug in --hosts argument.
+ (Sebastien Barrier via wang)
+
+ HADOOP-10539. Provide backward compatibility for ProxyUsers.authorize()
+ call. (Benoy Antony via cnauroth)
+
+ HADOOP-10540. Datanode upgrade in Windows fails with hardlink error.
+ (Chris Nauroth and Arpit Agarwal)
+
+ HADOOP-10508. RefreshCallQueue fails when authorization is enabled.
+ (Chris Li via wheat9)
+
+ HADOOP-10547. Give SaslPropertiesResolver.getDefaultProperties() public
+ scope. (Benoy Antony via Arpit Agarwal)
+
+ HADOOP-10543. RemoteException's unwrapRemoteException method failed for
+ PathIOException. (Yongjun Zhang via atm)
+
+ HADOOP-10568. Add s3 server-side encryption. (David S. Wang via atm)
+
+ HADOOP-10541. InputStream in MiniKdc#initKDCServer for minikdc.ldiff is not
+ closed. (Swarnim Kulkarni via cnauroth)
+
+ HADOOP-10517. InputStream is not closed in two methods of JarFinder.
+ (Ted Yu via cnauroth)
+
+ HADOOP-10581. TestUserGroupInformation#testGetServerSideGroups fails
+ because groups stored in Set and ArrayList are compared.
+ (Mit Desai via kihwal)
+
+ HADOOP-10585. Retry polices ignore interrupted exceptions (Daryn Sharp via
+ jeagles)
+
+ HADOOP-10401. ShellBasedUnixGroupsMapping#getGroups does not always return
+ primary group first (Akira AJISAKA via Colin Patrick McCabe)
+
+ HADOOP-10489. UserGroupInformation#getTokens and UserGroupInformation
+ #addToken can lead to ConcurrentModificationException (Robert Kanter via atm)
+
+ HADOOP-10602. Documentation has broken "Go Back" hyperlinks.
+ (Akira AJISAKA via cnauroth)
+
+ HADOOP-10639. FileBasedKeyStoresFactory initialization is not using default
+ for SSL_REQUIRE_CLIENT_CERT_KEY. (tucu)
+
+ HADOOP-10638. Updating hadoop-daemon.sh to work as expected when nfs is
+ started as a privileged user. (Manikandan Narayanaswamy via atm)
+
+ HADOOP-10630. Possible race condition in RetryInvocationHandler. (jing9)
+
+ HADOOP-10658. SSLFactory expects truststores being configured. (tucu via atm)
+
+ HADOOP-10647. String Format Exception in SwiftNativeFileSystemStore.java.
+ (Gene Kim via stevel)
+
+ HADOOP-9099. NetUtils.normalizeHostName fails on domains where
+ UnknownHost resolves to an IP address. (Ivan Mitic via suresh)
+
+ HADOOP-10664. TestNetUtils.testNormalizeHostName fails. (atm)
+
+ HADOOP-10656. The password keystore file is not picked by LDAP group mapping
+ (brandonli)
+
+ HADOOP-10622. Shell.runCommand can deadlock (Gera Shegalov via jlowe)
+
+ HADOOP-10686. Writables are not always configured.
+ (Abraham Elmahrek via kasha)
+
+ HADOOP-10678. SecurityUtil has unnecessary synchronization on collection
+ used for only tests. (Benoy Antony via cnauroth)
+
+ HADOOP-10683. Users authenticated with KERBEROS are recorded as being
+ authenticated with SIMPLE. (Benoy Antony via cnauroth)
+
+ HADOOP-10702. KerberosAuthenticationHandler does not log the principal names
+ correctly. (Benoy Antony via cnauroth)
+
+ HADOOP-10699. Fix build native library on mac osx (Binglin Chang via
+ jlowe)
+
+ HADOOP-10660. GraphiteSink should implement Closeable (Chen He and Ted Yu via raviprak)
+
+ HADOOP-10716. Cannot use more than 1 har filesystem.
+ (Rushabh Shah via cnauroth)
+
+ HADOOP-9559. When metrics system is restarted MBean names get incorrectly
+ flagged as dupes. (Mostafa Elhemali and Mike Liddell via cnauroth)
+
+ HADOOP-10746. TestSocketIOWithTimeout#testSocketIOWithTimeout fails on
+ Power PC. (Jinghui Wang via Arpit Agarwal)
+
+ HADOOP-9705. FsShell cp -p does not preserve directory attibutes.
+ (Akira AJISAKA via cnauroth)
+
+ HADOOP-10739. Renaming a file into a directory containing the same
+ filename results in a confusing I/O error (chang li via jlowe)
+
+ HADOOP-10533 S3 input stream NPEs in MapReduce join (stevel)
+
+ HADOOP-10419 BufferedFSInputStream NPEs on getPos() on a closed stream
+ (stevel)
+
+ HADOOP-10801 dead link in site.xml (Akira AJISAKA via stevel)
+
+ HADOOP-10590. ServiceAuthorizationManager is not threadsafe. (Benoy Antony via vinayakumarb)
+
+ HADOOP-10711. Cleanup some extra dependencies from hadoop-auth. (rkanter via tucu)
+
+ HADOOP-10479. Fix new findbugs warnings in hadoop-minikdc.
+ (Swarnim Kulkarni via wheat9)
+
+ HADOOP-10715. Remove public GraphiteSink#setWriter (Babak Behzad via raviprak)
+
+ HADOOP-10710. hadoop.auth cookie is not properly constructed according to
+ RFC2109. (Juan Yu via tucu)
+
+ HADOOP-10864. Tool documentenation is broken. (Akira Ajisaka
+ via Arpit Agarwal)
+
+ HADOOP-10872. TestPathData fails intermittently with "Mkdirs failed
+ to create d1". (Yongjun Zhang via Arpit Agarwal)
+
+ HADOOP-10890. TestDFVariations.testMount fails intermittently. (Yongjun
+ Zhang via Arpit Agarwal)
+
+ HADOOP-10894. Fix dead link in ToolRunner documentation. (Akira Ajisaka
+ via Arpit Agarwal)
+
+ HADOOP-10910. Increase findbugs maxHeap size. (wang)
+
+ BREAKDOWN OF HADOOP-10514 SUBTASKS AND RELATED JIRAS
+
+ HADOOP-10520. Extended attributes definition and FileSystem APIs for
+ extended attributes. (Yi Liu via wang)
+
+ HADOOP-10546. Javadoc and other small fixes for extended attributes in
+ hadoop-common. (Charles Lamb via wang)
+
+ HADOOP-10521. FsShell commands for extended attributes. (Yi Liu via wang)
+
+ HADOOP-10548. Improve FsShell xattr error handling and other fixes. (Charles Lamb via umamahesh)
+
+ HADOOP-10567. Shift XAttr value encoding code out for reuse. (Yi Liu via umamahesh)
+
+ HADOOP-10621. Remove CRLF for xattr value base64 encoding for better display.(Yi Liu via umamahesh)
+
+ HADOOP-10575. Small fixes for XAttrCommands and test. (Yi Liu via umamahesh)
+
+ HADOOP-10561. Copy command with preserve option should handle Xattrs.
+ (Yi Liu via cnauroth)
+
+Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
@@ -420,6 +992,17 @@ Release 2.4.1 - UNRELEASED
HADOOP-10490. TestMapFile and TestBloomMapFile leak file descriptors.
(cnauroth)
+ HADOOP-10522. JniBasedUnixGroupMapping mishandles errors. (kihwal)
+
+ HADOOP-10527. Fix incorrect return code and allow more retries on EINTR.
+ (kihwal)
+
+ HADOOP-10612. NFS failed to refresh the user group id mapping table (brandonli)
+
+ HADOOP-10562. Namenode exits on exception without printing stack trace
+ in AbstractDelegationTokenSecretManager. (Suresh Srinivas via Arpit
+ Agarwal)
+
Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/CHANGES.txt:r1588992-1596568
Merged /hadoop/common/branches/branch-2.5/hadoop-common-project/hadoop-common/CHANGES.txt:r1607590,1607618,1609091
Merged /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt:r1600970,1616481
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1588388-1619000
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml Tue Aug 19 23:49:39 2014
@@ -287,6 +287,10 @@
<!-- protobuf generated code -->
<Class name="~org\.apache\.hadoop\.ipc\.proto\.RefreshCallQueueProtocolProtos.*"/>
</Match>
+ <Match>
+ <!-- protobuf generated code -->
+ <Class name="~org\.apache\.hadoop\.ipc\.proto\.GenericRefreshProtocolProtos.*"/>
+ </Match>
<!--
Manually checked, misses child thread manually syncing on parent's intrinsic lock.
@@ -357,4 +361,10 @@
<Bug code="NP" />
</Match>
+ <Match>
+ <Class name="org.apache.hadoop.crypto.key.kms.KMSClientProvider"/>
+ <Method name="validateResponse"/>
+ <Bug pattern="REC_CATCH_EXCEPTION"/>
+ </Match>
+
</FindBugsFilter>
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/pom.xml?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/pom.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/pom.xml Tue Aug 19 23:49:39 2014
@@ -104,6 +104,11 @@
<scope>compile</scope>
</dependency>
<dependency>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
<scope>compile</scope>
@@ -120,26 +125,6 @@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- <scope>runtime</scope>
- </dependency>
- <dependency>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- <scope>runtime</scope>
- </dependency>
- <dependency>
- <groupId>javax.servlet.jsp</groupId>
- <artifactId>jsp-api</artifactId>
- <scope>runtime</scope>
- </dependency>
- <dependency>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- <scope>runtime</scope>
- </dependency>
- <dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<scope>compile</scope>
@@ -219,6 +204,17 @@
<scope>compile</scope>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minikdc</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
</dependency>
@@ -338,6 +334,7 @@
<include>RefreshAuthorizationPolicyProtocol.proto</include>
<include>RefreshUserMappingsProtocol.proto</include>
<include>RefreshCallQueueProtocol.proto</include>
+ <include>GenericRefreshProtocol.proto</include>
</includes>
</source>
<output>${project.build.directory}/generated-sources/java</output>
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/JNIFlags.cmake
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/JNIFlags.cmake?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/JNIFlags.cmake (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/JNIFlags.cmake Tue Aug 19 23:49:39 2014
@@ -78,6 +78,12 @@ IF("${CMAKE_SYSTEM}" MATCHES "Linux")
SET(_java_libarch "amd64")
ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm")
SET(_java_libarch "arm")
+ ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64le")
+ IF(EXISTS "${_JAVA_HOME}/jre/lib/ppc64le")
+ SET(_java_libarch "ppc64le")
+ ELSE()
+ SET(_java_libarch "ppc64")
+ ENDIF()
ELSE()
SET(_java_libarch ${CMAKE_SYSTEM_PROCESSOR})
ENDIF()