You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2010/02/25 15:14:18 UTC
svn commit: r916290 - in /hadoop/common/trunk: ./ .eclipse.templates/ ivy/
src/java/org/apache/hadoop/security/token/
src/test/core/org/apache/hadoop/io/compress/
src/test/core/org/apache/hadoop/security/token/
Author: omalley
Date: Thu Feb 25 14:14:18 2010
New Revision: 916290
URL: http://svn.apache.org/viewvc?rev=916290&view=rev
Log:
HADOOP-6579. Provide a mechanism for encoding/decoding Tokens from
a url-safe string and change the commons-code library to 1.4. (omalley)
Modified:
hadoop/common/trunk/.eclipse.templates/.classpath
hadoop/common/trunk/CHANGES.txt
hadoop/common/trunk/ivy/hadoop-core-template.xml
hadoop/common/trunk/ivy/libraries.properties
hadoop/common/trunk/src/java/org/apache/hadoop/security/token/Token.java
hadoop/common/trunk/src/test/core/org/apache/hadoop/io/compress/TestCodec.java
hadoop/common/trunk/src/test/core/org/apache/hadoop/security/token/TestToken.java
Modified: hadoop/common/trunk/.eclipse.templates/.classpath
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/.eclipse.templates/.classpath?rev=916290&r1=916289&r2=916290&view=diff
==============================================================================
--- hadoop/common/trunk/.eclipse.templates/.classpath (original)
+++ hadoop/common/trunk/.eclipse.templates/.classpath Thu Feb 25 14:14:18 2010
@@ -7,7 +7,7 @@
<classpathentry kind="var" path="ANT_HOME/lib/ant.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/Hadoop-Core/common/avro-1.2.0.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/Hadoop-Core/common/commons-cli-1.2.jar"/>
- <classpathentry kind="lib" path="build/ivy/lib/Hadoop-Core/common/commons-codec-1.3.jar"/>
+ <classpathentry kind="lib" path="build/ivy/lib/Hadoop-Core/common/commons-codec-1.4.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/Hadoop-Core/common/commons-el-1.0.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/Hadoop-Core/common/commons-httpclient-3.0.1.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/Hadoop-Core/common/commons-logging-1.1.1.jar"/>
Modified: hadoop/common/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=916290&r1=916289&r2=916290&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Thu Feb 25 14:14:18 2010
@@ -161,6 +161,9 @@
HADOOP-6543. Allows secure clients to talk to unsecure clusters.
(Kan Zhang via ddas)
+ HADOOP-6579. Provide a mechanism for encoding/decoding Tokens from
+ a url-safe string and change the commons-code library to 1.4. (omalley)
+
OPTIMIZATIONS
HADOOP-6467. Improve the performance on HarFileSystem.listStatus(..).
Modified: hadoop/common/trunk/ivy/hadoop-core-template.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/ivy/hadoop-core-template.xml?rev=916290&r1=916289&r2=916290&view=diff
==============================================================================
--- hadoop/common/trunk/ivy/hadoop-core-template.xml (original)
+++ hadoop/common/trunk/ivy/hadoop-core-template.xml Thu Feb 25 14:14:18 2010
@@ -41,7 +41,7 @@
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
- <version>1.3</version>
+ <version>1.4</version>
</dependency>
<dependency>
<groupId>commons-net</groupId>
Modified: hadoop/common/trunk/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/ivy/libraries.properties?rev=916290&r1=916289&r2=916290&view=diff
==============================================================================
--- hadoop/common/trunk/ivy/libraries.properties (original)
+++ hadoop/common/trunk/ivy/libraries.properties Thu Feb 25 14:14:18 2010
@@ -23,7 +23,7 @@
commons-cli.version=1.2
commons-cli2.version=2.0-mahout
-commons-codec.version=1.3
+commons-codec.version=1.4
commons-collections.version=3.1
commons-httpclient.version=3.0.1
commons-lang.version=2.4
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/security/token/Token.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/security/token/Token.java?rev=916290&r1=916289&r2=916290&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/security/token/Token.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/security/token/Token.java Thu Feb 25 14:14:18 2010
@@ -21,9 +21,15 @@
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.Arrays;
+import org.apache.commons.codec.binary.Base64;
+
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.io.WritableUtils;
/**
@@ -47,7 +53,21 @@
kind = id.getKind();
service = new Text();
}
-
+
+ /**
+ * Construct a token from the components.
+ * @param identifier the token identifier
+ * @param password the token's password
+ * @param kind the kind of token
+ * @param service the service for this token
+ */
+ public Token(byte[] identifier, byte[] password, Text kind, Text service) {
+ this.identifier = identifier;
+ this.password = password;
+ this.kind = kind;
+ this.service = service;
+ }
+
/**
* Default constructor
*/
@@ -123,4 +143,103 @@
kind.write(out);
service.write(out);
}
+
+ /**
+ * Generate a string with the url-quoted base64 encoded serialized form
+ * of the Writable.
+ * @param obj the object to serialize
+ * @return the encoded string
+ * @throws IOException
+ */
+ private static String encodeWritable(Writable obj) throws IOException {
+ DataOutputBuffer buf = new DataOutputBuffer();
+ obj.write(buf);
+ Base64 encoder = new Base64(0, null, true);
+ byte[] raw = new byte[buf.getLength()];
+ System.arraycopy(buf.getData(), 0, raw, 0, buf.getLength());
+ return encoder.encodeToString(raw);
+ }
+
+ /**
+ * Modify the writable to the value from the newValue
+ * @param obj the object to read into
+ * @param newValue the string with the url-safe base64 encoded bytes
+ * @throws IOException
+ */
+ private static void decodeWritable(Writable obj,
+ String newValue) throws IOException {
+ Base64 decoder = new Base64(0, null, true);
+ DataInputBuffer buf = new DataInputBuffer();
+ byte[] decoded = decoder.decode(newValue);
+ buf.reset(decoded, decoded.length);
+ obj.readFields(buf);
+ }
+
+ /**
+ * Encode this token as a url safe string
+ * @return the encoded string
+ * @throws IOException
+ */
+ public String encodeToUrlString() throws IOException {
+ return encodeWritable(this);
+ }
+
+ /**
+ * Decode the given url safe string into this token.
+ * @param newValue the encoded string
+ * @throws IOException
+ */
+ public void decodeFromUrlString(String newValue) throws IOException {
+ decodeWritable(this, newValue);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public boolean equals(Object right) {
+ if (this == right) {
+ return true;
+ } else if (right == null || getClass() != right.getClass()) {
+ return false;
+ } else {
+ Token<T> r = (Token<T>) right;
+ return Arrays.equals(identifier, r.identifier) &&
+ Arrays.equals(password, r.password) &&
+ kind.equals(r.kind) &&
+ service.equals(r.service);
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return WritableComparator.hashBytes(identifier, identifier.length);
+ }
+
+ private static void addBinaryBuffer(StringBuilder buffer, byte[] bytes) {
+ for (int idx = 0; idx < bytes.length; idx++) {
+ // if not the first, put a blank separator in
+ if (idx != 0) {
+ buffer.append(' ');
+ }
+ String num = Integer.toHexString(0xff & bytes[idx]);
+ // if it is only one digit, add a leading 0.
+ if (num.length() < 2) {
+ buffer.append('0');
+ }
+ buffer.append(num);
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder buffer = new StringBuilder();
+ buffer.append("Ident: ");
+ addBinaryBuffer(buffer, identifier);
+ buffer.append(", Pass: ");
+ addBinaryBuffer(buffer, password);
+ buffer.append(", Kind: ");
+ buffer.append(kind.toString());
+ buffer.append(", Service: ");
+ buffer.append(service.toString());
+ return buffer.toString();
+ }
}
Modified: hadoop/common/trunk/src/test/core/org/apache/hadoop/io/compress/TestCodec.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/io/compress/TestCodec.java?rev=916290&r1=916289&r2=916290&view=diff
==============================================================================
--- hadoop/common/trunk/src/test/core/org/apache/hadoop/io/compress/TestCodec.java (original)
+++ hadoop/common/trunk/src/test/core/org/apache/hadoop/io/compress/TestCodec.java Thu Feb 25 14:14:18 2010
@@ -245,7 +245,7 @@
codec.getClass().getSimpleName());
final Path file = new Path(wd, "test" + codec.getDefaultExtension());
final byte[] b = new byte[REC_SIZE];
- final Base64 b64 = new Base64();
+ final Base64 b64 = new Base64(0, null);
DataOutputStream fout = null;
Compressor cmp = CodecPool.getCompressor(codec);
try {
Modified: hadoop/common/trunk/src/test/core/org/apache/hadoop/security/token/TestToken.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/security/token/TestToken.java?rev=916290&r1=916289&r2=916290&view=diff
==============================================================================
--- hadoop/common/trunk/src/test/core/org/apache/hadoop/security/token/TestToken.java (original)
+++ hadoop/common/trunk/src/test/core/org/apache/hadoop/security/token/TestToken.java Thu Feb 25 14:14:18 2010
@@ -22,6 +22,7 @@
import java.util.Arrays;
import org.apache.hadoop.io.*;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import junit.framework.TestCase;
@@ -58,4 +59,40 @@
destToken.readFields(in);
assertTrue(checkEqual(sourceToken, destToken));
}
+
+ private static void checkUrlSafe(String str) throws Exception {
+ int len = str.length();
+ for(int i=0; i < len; ++i) {
+ char ch = str.charAt(i);
+ if (ch == '-') continue;
+ if (ch == '_') continue;
+ if (ch >= '0' && ch <= '9') continue;
+ if (ch >= 'A' && ch <= 'Z') continue;
+ if (ch >= 'a' && ch <= 'z') continue;
+ fail("Encoded string " + str +
+ " has invalid character at position " + i);
+ }
+ }
+
+ public static void testEncodeWritable() throws Exception {
+ String[] values = new String[]{"", "a", "bb", "ccc", "dddd", "eeeee",
+ "ffffff", "ggggggg", "hhhhhhhh", "iiiiiiiii",
+ "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLM" +
+ "NOPQRSTUVWXYZ01234567890!@#$%^&*()-=_+[]{}|;':,./<>?"};
+ Token<AbstractDelegationTokenIdentifier> orig;
+ Token<AbstractDelegationTokenIdentifier> copy =
+ new Token<AbstractDelegationTokenIdentifier>();
+ // ensure that for each string the input and output values match
+ for(int i=0; i< values.length; ++i) {
+ String val = values[i];
+ System.out.println("Input = " + val);
+ orig = new Token<AbstractDelegationTokenIdentifier>(val.getBytes(),
+ val.getBytes(), new Text(val), new Text(val));
+ String encode = orig.encodeToUrlString();
+ copy.decodeFromUrlString(encode);
+ assertEquals(orig, copy);
+ checkUrlSafe(encode);
+ }
+ }
+
}