You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2013/03/31 06:34:02 UTC
svn commit: r1462877 - in /hadoop/common/branches/branch-1.2: ./ bin/
src/core/org/apache/hadoop/io/compress/
src/docs/src/documentation/content/xdocs/ src/hdfs/org/apache/hadoop/hdfs/
src/hdfs/org/apache/hadoop/hdfs/protocol/ src/hdfs/org/apache/hadoo...
Author: suresh
Date: Sun Mar 31 04:34:01 2013
New Revision: 1462877
URL: http://svn.apache.org/r1462877
Log:
HDFS-4651. Merge changes 1462875 and 1462876 from branch-1
Added:
hadoop/common/branches/branch-1.2/src/docs/src/documentation/content/xdocs/hdfs_imageviewer.xml
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/docs/src/documentation/content/xdocs/hdfs_imageviewer.xml
hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/protocol/LayoutVersion.java
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/protocol/LayoutVersion.java
hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/
- copied from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/
hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/DelimitedImageVisitor.java
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/DelimitedImageVisitor.java
hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/DepthCounter.java
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/DepthCounter.java
hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java
hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoader.java
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoader.java
hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java
hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/IndentedImageVisitor.java
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/IndentedImageVisitor.java
hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsImageVisitor.java
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsImageVisitor.java
hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/NameDistributionVisitor.java
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/NameDistributionVisitor.java
hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/TextWriterImageVisitor.java
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/TextWriterImageVisitor.java
hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/XmlImageVisitor.java
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/XmlImageVisitor.java
hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/protocol/
- copied from r1462876, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/protocol/
hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java
- copied unchanged from r1462876, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java
hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/tools/
- copied from r1462875, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/
hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/
- copied from r1462875, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/
hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/SpotCheckImageVisitor.java
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/SpotCheckImageVisitor.java
hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestDelimitedImageVisitor.java
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestDelimitedImageVisitor.java
hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOIVCanReadOldVersions.java
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOIVCanReadOldVersions.java
hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18
hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19
- copied unchanged from r1462875, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19
Modified:
hadoop/common/branches/branch-1.2/bin/hadoop
hadoop/common/branches/branch-1.2/build.xml
hadoop/common/branches/branch-1.2/src/core/org/apache/hadoop/io/compress/CompressionCodecFactory.java
hadoop/common/branches/branch-1.2/src/docs/src/documentation/content/xdocs/site.xml
hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Modified: hadoop/common/branches/branch-1.2/bin/hadoop
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/bin/hadoop?rev=1462877&r1=1462876&r2=1462877&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/bin/hadoop (original)
+++ hadoop/common/branches/branch-1.2/bin/hadoop Sun Mar 31 04:34:01 2013
@@ -81,6 +81,7 @@ print_usage()
echo " fsck run a DFS filesystem checking utility"
echo " fs run a generic filesystem user client"
echo " balancer run a cluster balancing utility"
+ echo " oiv apply the offline fsimage viewer to an fsimage"
echo " fetchdt fetch a delegation token from the NameNode"
echo " jobtracker run the MapReduce job Tracker node"
echo " pipes run a Pipes job"
@@ -287,6 +288,8 @@ elif [ "$COMMAND" = "fsck" ] ; then
elif [ "$COMMAND" = "balancer" ] ; then
CLASS=org.apache.hadoop.hdfs.server.balancer.Balancer
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_BALANCER_OPTS"
+elif [ "$COMMAND" = "oiv" ] ; then
+ CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer
elif [ "$COMMAND" = "fetchdt" ] ; then
CLASS=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
elif [ "$COMMAND" = "jobtracker" ] ; then
Modified: hadoop/common/branches/branch-1.2/build.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/build.xml?rev=1462877&r1=1462876&r2=1462877&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/build.xml (original)
+++ hadoop/common/branches/branch-1.2/build.xml Sun Mar 31 04:34:01 2013
@@ -936,6 +936,8 @@
<copy file="${test.src.dir}/org/apache/hadoop/cli/clitest_data/data30bytes" todir="${test.cache.data}"/>
<copy file="${test.src.dir}/org/apache/hadoop/cli/clitest_data/data60bytes" todir="${test.cache.data}"/>
<copy file="${test.src.dir}/org/apache/hadoop/cli/clitest_data/data120bytes" todir="${test.cache.data}"/>
+ <copy file="${test.src.dir}/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18" todir="${test.cache.data}"/>
+ <copy file="${test.src.dir}/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19" todir="${test.cache.data}"/>
</target>
<!-- ================================================================== -->
Modified: hadoop/common/branches/branch-1.2/src/core/org/apache/hadoop/io/compress/CompressionCodecFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/src/core/org/apache/hadoop/io/compress/CompressionCodecFactory.java?rev=1462877&r1=1462876&r2=1462877&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/src/core/org/apache/hadoop/io/compress/CompressionCodecFactory.java (original)
+++ hadoop/common/branches/branch-1.2/src/core/org/apache/hadoop/io/compress/CompressionCodecFactory.java Sun Mar 31 04:34:01 2013
@@ -39,10 +39,30 @@ public class CompressionCodecFactory {
* automatically supports finding the longest matching suffix.
*/
private SortedMap<String, CompressionCodec> codecs = null;
+
+ /**
+ * A map from the reversed filename suffixes to the codecs.
+ * This is probably overkill, because the maps should be small, but it
+ * automatically supports finding the longest matching suffix.
+ */
+ private Map<String, CompressionCodec> codecsByName = null;
+
+ /**
+ * A map from class names to the codecs
+ */
+ private HashMap<String, CompressionCodec> codecsByClassName = null;
private void addCodec(CompressionCodec codec) {
String suffix = codec.getDefaultExtension();
codecs.put(new StringBuffer(suffix).reverse().toString(), codec);
+ codecsByClassName.put(codec.getClass().getCanonicalName(), codec);
+
+ String codecName = codec.getClass().getSimpleName();
+ codecsByName.put(codecName.toLowerCase(), codec);
+ if (codecName.endsWith("Codec")) {
+ codecName = codecName.substring(0, codecName.length() - "Codec".length());
+ codecsByName.put(codecName.toLowerCase(), codec);
+ }
}
/**
@@ -131,6 +151,8 @@ public class CompressionCodecFactory {
*/
public CompressionCodecFactory(Configuration conf) {
codecs = new TreeMap<String, CompressionCodec>();
+ codecsByClassName = new HashMap<String, CompressionCodec>();
+ codecsByName = new HashMap<String, CompressionCodec>();
List<Class<? extends CompressionCodec>> codecClasses = getCodecClasses(conf);
if (codecClasses == null) {
addCodec(new GzipCodec());
@@ -168,6 +190,68 @@ public class CompressionCodecFactory {
}
/**
+ * Find the relevant compression codec for the codec's canonical class name.
+ * @param classname the canonical class name of the codec
+ * @return the codec object
+ */
+ public CompressionCodec getCodecByClassName(String classname) {
+ if (codecsByClassName == null) {
+ return null;
+ }
+ return codecsByClassName.get(classname);
+ }
+
+ /**
+ * Find the relevant compression codec for the codec's canonical class name
+ * or by codec alias.
+ * <p/>
+ * Codec aliases are case insensitive.
+ * <p/>
+ * The code alias is the short class name (without the package name).
+ * If the short class name ends with 'Codec', then there are two aliases for
+ * the codec, the complete short class name and the short class name without
+ * the 'Codec' ending. For example for the 'GzipCodec' codec class name the
+ * alias are 'gzip' and 'gzipcodec'.
+ *
+ * @param codecName the canonical class name of the codec
+ * @return the codec object
+ */
+ public CompressionCodec getCodecByName(String codecName) {
+ if (codecsByClassName == null) {
+ return null;
+ }
+ CompressionCodec codec = getCodecByClassName(codecName);
+ if (codec == null) {
+ // trying to get the codec by name in case the name was specified instead a class
+ codec = codecsByName.get(codecName.toLowerCase());
+ }
+ return codec;
+ }
+
+ /**
+ * Find the relevant compression codec for the codec's canonical class name
+ * or by codec alias and returns its implemetation class.
+ * <p/>
+ * Codec aliases are case insensitive.
+ * <p/>
+ * The code alias is the short class name (without the package name).
+ * If the short class name ends with 'Codec', then there are two aliases for
+ * the codec, the complete short class name and the short class name without
+ * the 'Codec' ending. For example for the 'GzipCodec' codec class name the
+ * alias are 'gzip' and 'gzipcodec'.
+ *
+ * @param codecName the canonical class name of the codec
+ * @return the codec class
+ */
+ public Class<? extends CompressionCodec> getCodecClassByName(String codecName) {
+ CompressionCodec codec = getCodecByName(codecName);
+ if (codec == null) {
+ return null;
+ }
+ return codec.getClass();
+ }
+
+ /**
* Removes a suffix from a filename, if it has it.
* @param filename the filename to strip
* @param suffix the suffix to remove
Modified: hadoop/common/branches/branch-1.2/src/docs/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/src/docs/src/documentation/content/xdocs/site.xml?rev=1462877&r1=1462876&r2=1462877&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/src/docs/src/documentation/content/xdocs/site.xml (original)
+++ hadoop/common/branches/branch-1.2/src/docs/src/documentation/content/xdocs/site.xml Sun Mar 31 04:34:01 2013
@@ -63,6 +63,7 @@ See http://forrest.apache.org/docs/linki
<hdfs_perm label="Permissions" href="hdfs_permissions_guide.html" />
<hdfs_quotas label="Quotas" href="hdfs_quota_admin_guide.html" />
<hdfs_SLG label="Synthetic Load Generator" href="SLG_user_guide.html" />
+ <hdfs_imageviewer label="Offline Image Viewer" href="hdfs_imageviewer.html" />
<hftp label="HFTP" href="hftp.html"/>
<webhdfs label="WebHDFS REST API" href="webhdfs.html" />
<hdfs_libhdfs label="C API libhdfs" href="libhdfs.html" />
Modified: hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1462877&r1=1462876&r2=1462877&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java Sun Mar 31 04:34:01 2013
@@ -130,6 +130,8 @@ public class DFSConfigKeys extends Commo
public static final long DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 24*60*60*1000;
public static final String DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY = "dfs.namenode.delegation.token.max-lifetime";
public static final long DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7*24*60*60*1000;
+ public static final String DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY = "dfs.namenode.delegation.token.always-use"; // for tests
+ public static final boolean DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT = false;
//Following keys have no defaults
public static final String DFS_DATANODE_DATA_DIR_KEY = "dfs.datanode.data.dir";
Modified: hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1462877&r1=1462876&r2=1462877&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original)
+++ hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java Sun Mar 31 04:34:01 2013
@@ -1820,7 +1820,7 @@ public class FSImage extends Storage {
}
static private final UTF8 U_STR = new UTF8();
- static String readString(DataInputStream in) throws IOException {
+ public static String readString(DataInputStream in) throws IOException {
U_STR.readFields(in);
return U_STR.toString();
}
@@ -1830,7 +1830,7 @@ public class FSImage extends Storage {
return s.isEmpty()? null: s;
}
- static byte[] readBytes(DataInputStream in) throws IOException {
+ public static byte[] readBytes(DataInputStream in) throws IOException {
U_STR.readFields(in);
int len = U_STR.getLength();
byte[] bytes = new byte[len];
Modified: hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1462877&r1=1462876&r2=1462877&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java Sun Mar 31 04:34:01 2013
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT;
+
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -294,8 +297,15 @@ public class NameNode implements ClientP
myMetrics = NameNodeInstrumentation.create(conf);
this.namesystem = new FSNamesystem(this, conf);
+
+ // For testing purposes, allow the DT secret manager to be started regardless
+ // of whether security is enabled.
+ boolean alwaysUseDelegationTokensForTests =
+ conf.getBoolean(DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY,
+ DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT);
- if (UserGroupInformation.isSecurityEnabled()) {
+ if (UserGroupInformation.isSecurityEnabled() ||
+ alwaysUseDelegationTokensForTests) {
namesystem.activateSecretManager();
}