You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2012/12/23 03:08:51 UTC
svn commit: r1425390 - in
/hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt
src/main/bin/hadoop
src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java
Author: suresh
Date: Sun Dec 23 02:08:50 2012
New Revision: 1425390
URL: http://svn.apache.org/viewvc?rev=1425390&view=rev
Log:
HADOOP-9162. Add utility to check native library availability. Contributed by Binglin Chang.
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/bin/hadoop
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1425390&r1=1425389&r2=1425390&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Sun Dec 23 02:08:50 2012
@@ -143,6 +143,9 @@ Trunk (Unreleased)
HADOOP-9140 Cleanup rpc PB protos (sanjay Radia)
+ HADOOP-9162. Add utility to check native library availability.
+ (Binglin Chang via suresh)
+
BUG FIXES
HADOOP-9041. FsUrlStreamHandlerFactory could cause an infinite loop in
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/bin/hadoop
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/bin/hadoop?rev=1425390&r1=1425389&r2=1425390&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/bin/hadoop (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/bin/hadoop Sun Dec 23 02:08:50 2012
@@ -31,6 +31,7 @@ function print_usage(){
echo " fs run a generic filesystem user client"
echo " version print the version"
echo " jar <jar> run a jar file"
+ echo " checknative [-a|-h] check native hadoop and compression libraries availability"
echo " distcp <srcurl> <desturl> copy file or directories recursively"
echo " archive -archiveName NAME -p <parent path> <src>* <dest> create a hadoop archive"
echo " classpath prints the class path needed to get the"
@@ -100,6 +101,8 @@ case $COMMAND in
CLASS=org.apache.hadoop.util.VersionInfo
elif [ "$COMMAND" = "jar" ] ; then
CLASS=org.apache.hadoop.util.RunJar
+ elif [ "$COMMAND" = "checknative" ] ; then
+ CLASS=org.apache.hadoop.util.NativeLibraryChecker
elif [ "$COMMAND" = "distcp" ] ; then
CLASS=org.apache.hadoop.tools.DistCp
CLASSPATH=${CLASSPATH}:${TOOL_PATH}
Added: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java?rev=1425390&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java (added)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java Sun Dec 23 02:08:50 2012
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.util.NativeCodeLoader;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.compress.SnappyCodec;
+import org.apache.hadoop.io.compress.zlib.ZlibFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class NativeLibraryChecker {
+ /**
+ * A tool to test native library availability,
+ */
+ public static void main(String[] args) {
+ String usage = "NativeLibraryChecker [-a|-h]\n"
+ + " -a use -a to check all libraries are available\n"
+ + " by default just check hadoop library is available\n"
+ + " exit with error code if check failed\n"
+ + " -h print this message\n";
+ if (args.length > 1 ||
+ (args.length == 1 &&
+ !(args[0].equals("-a") || args[0].equals("-h")))) {
+ System.err.println(usage);
+ ExitUtil.terminate(1);
+ }
+ boolean checkAll = false;
+ if (args.length == 1) {
+ if (args[0].equals("-h")) {
+ System.out.println(usage);
+ return;
+ }
+ checkAll = true;
+ }
+ boolean nativeHadoopLoaded = NativeCodeLoader.isNativeCodeLoaded();
+ boolean zlibLoaded = false;
+ boolean snappyLoaded = false;
+ // lz4 is linked within libhadoop
+ boolean lz4Loaded = nativeHadoopLoaded;
+ if (nativeHadoopLoaded) {
+ zlibLoaded = ZlibFactory.isNativeZlibLoaded(new Configuration());
+ snappyLoaded = NativeCodeLoader.buildSupportsSnappy() &&
+ SnappyCodec.isNativeCodeLoaded();
+ }
+ System.out.println("Native library checking:");
+ System.out.printf("hadoop: %b\n", nativeHadoopLoaded);
+ System.out.printf("zlib: %b\n", zlibLoaded);
+ System.out.printf("snappy: %b\n", snappyLoaded);
+ System.out.printf("lz4: %b\n", lz4Loaded);
+ if ((!nativeHadoopLoaded) ||
+ (checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded))) {
+ // return 1 to indicated check failed
+ ExitUtil.terminate(1);
+ }
+ }
+}
Added: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java?rev=1425390&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java (added)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java Sun Dec 23 02:08:50 2012
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.util.ExitUtil.ExitException;
+import org.junit.Test;
+
+public class TestNativeLibraryChecker extends TestCase {
+ private void expectExit(String [] args) {
+ try {
+ // should throw exit exception
+ NativeLibraryChecker.main(args);
+ fail("should call exit");
+ } catch (ExitException e) {
+ // pass
+ ExitUtil.resetFirstExitException();
+ }
+ }
+
+ @Test
+ public void testNativeLibraryChecker() {
+ ExitUtil.disableSystemExit();
+ // help should return normally
+ NativeLibraryChecker.main(new String[] {"-h"});
+ // illegal argmuments should exit
+ expectExit(new String[] {"-a", "-h"});
+ expectExit(new String[] {"aaa"});
+ if (NativeCodeLoader.isNativeCodeLoaded()) {
+ // no argument should return normally
+ NativeLibraryChecker.main(new String[0]);
+ } else {
+ // no argument should exit
+ expectExit(new String[0]);
+ }
+ }
+
+}