You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2015/12/23 19:51:39 UTC
ignite git commit: IGNITE-2218: fix for native libs loading
Repository: ignite
Updated Branches:
refs/heads/ignite-igfs-fixes bd05fa90b -> b8bcb3f8c
IGNITE-2218: fix for native libs loading
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b8bcb3f8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b8bcb3f8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b8bcb3f8
Branch: refs/heads/ignite-igfs-fixes
Commit: b8bcb3f8cba15f778218f9defb623ad7436de385
Parents: bd05fa9
Author: dmagda <ma...@gmail.com>
Authored: Wed Dec 23 21:51:30 2015 +0300
Committer: dmagda <ma...@gmail.com>
Committed: Wed Dec 23 21:51:30 2015 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopClassLoader.java | 6 ++
.../v2/HadoopJniBasedUnixGroupsMapping.java | 98 ++++++++++++++++++++
.../hadoop/v2/HadoopNativeCodeLoader.java | 57 ++++++++++--
3 files changed, 155 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b8bcb3f8/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index f12af46..cf11b38 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -33,6 +33,7 @@ import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.processors.hadoop.v2.HadoopDaemon;
+import org.apache.ignite.internal.processors.hadoop.v2.HadoopJniBasedUnixGroupsMapping;
import org.apache.ignite.internal.processors.hadoop.v2.HadoopNativeCodeLoader;
import org.apache.ignite.internal.processors.hadoop.v2.HadoopShutdownHookManager;
import org.apache.ignite.internal.util.typedef.F;
@@ -67,6 +68,9 @@ public class HadoopClassLoader extends URLClassLoader {
public static final String HADOOP_DAEMON_CLASS_NAME = "org.apache.hadoop.util.Daemon";
/** */
+ private static final String HADOOP_UNIX_MAPPING_CLASS_NAME = "org.apache.hadoop.security.JniBasedUnixGroupsMapping";
+
+ /** */
private static final URLClassLoader APP_CLS_LDR = (URLClassLoader)HadoopClassLoader.class.getClassLoader();
/** */
@@ -147,6 +151,8 @@ public class HadoopClassLoader extends URLClassLoader {
return loadFromBytes(name, HadoopShutdownHookManager.class.getName());
else if (name.endsWith(".util.NativeCodeLoader"))
return loadFromBytes(name, HadoopNativeCodeLoader.class.getName());
+ else if (name.equals(HADOOP_UNIX_MAPPING_CLASS_NAME))
+ return loadFromBytes(name, HadoopJniBasedUnixGroupsMapping.class.getName());
else if (name.equals(HADOOP_DAEMON_CLASS_NAME))
// We replace this in order to be able to forcibly stop some daemon threads
// that otherwise never stop (e.g. PeerCache runnables):
http://git-wip-us.apache.org/repos/asf/ignite/blob/b8bcb3f8/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopJniBasedUnixGroupsMapping.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopJniBasedUnixGroupsMapping.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopJniBasedUnixGroupsMapping.java
new file mode 100644
index 0000000..d917e2b
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopJniBasedUnixGroupsMapping.java
@@ -0,0 +1,98 @@
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.GroupMappingServiceProvider;
+import org.apache.hadoop.security.JniBasedUnixGroupsMapping;
+
+/**
+ * A JNI-based implementation of {@link GroupMappingServiceProvider}
+ * that invokes libC calls to get the group
+ * memberships of a given user.
+ */
+@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+@InterfaceStability.Evolving
+public class HadoopJniBasedUnixGroupsMapping implements GroupMappingServiceProvider {
+
+ private static final Log LOG =
+ LogFactory.getLog(JniBasedUnixGroupsMapping.class);
+
+ static {
+ if (!HadoopNativeCodeLoader.isNativeCodeLoaded()) {
+ throw new RuntimeException("Bailing out since native library couldn't " +
+ "be loaded");
+ }
+ anchorNative();
+ LOG.debug("Using JniBasedUnixGroupsMapping for Group resolution");
+ }
+
+ /**
+ * Set up our JNI resources.
+ *
+ * @throws RuntimeException if setup fails.
+ */
+ static void anchorNative() {
+ try {
+ ClassLoader parent = Configuration.class.getClassLoader().getParent();
+
+ Class cls = Class.forName(JniBasedUnixGroupsMapping.class.getName(), true, parent);
+
+ Method m = cls.getDeclaredMethod("anchorNative");
+
+ m.setAccessible(true);
+
+ m.invoke(null);
+ }
+ catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Get the set of groups associated with a user.
+ *
+ * @param username The user name
+ *
+ * @return The set of groups associated with a user.
+ */
+ native static String[] getGroupsForUser(String username);
+
+ /**
+ * Log an error message about a group. Used from JNI.
+ */
+ static private void logError(int groupId, String error) {
+ LOG.error("error looking up the name of group " + groupId + ": " + error);
+ }
+
+ @Override
+ public List<String> getGroups(String user) throws IOException {
+ String[] groups = new String[0];
+ try {
+ groups = getGroupsForUser(user);
+ } catch (Exception e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Error getting groups for " + user, e);
+ } else {
+ LOG.info("Error getting groups for " + user + ": " + e.getMessage());
+ }
+ }
+ return Arrays.asList(groups);
+ }
+
+ @Override
+ public void cacheGroupsRefresh() throws IOException {
+ // does nothing in this provider of user to groups mapping
+ }
+
+ @Override
+ public void cacheGroupsAdd(List<String> groups) throws IOException {
+ // does nothing in this provider of user to groups mapping
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b8bcb3f8/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopNativeCodeLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopNativeCodeLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopNativeCodeLoader.java
index 4c4840d..f3ed66c 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopNativeCodeLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopNativeCodeLoader.java
@@ -17,9 +17,12 @@
package org.apache.ignite.internal.processors.hadoop.v2;
+import java.lang.reflect.Method;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.util.NativeCodeLoader;
/**
* A fake helper to load the native hadoop code i.e. libhadoop.so.
@@ -34,23 +37,62 @@ public class HadoopNativeCodeLoader {
* else <code>false</code>
*/
public static boolean isNativeCodeLoaded() {
- return false;
+ try {
+ ClassLoader parent = Configuration.class.getClassLoader().getParent();
+
+ Class cls = Class.forName(NativeCodeLoader.class.getName(), true, parent);
+
+ Method m = cls.getMethod("isNativeCodeLoaded");
+
+ boolean value = (boolean)m.invoke(null);
+
+ return value;
+ }
+ catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
/**
* Returns true only if this build was compiled with support for snappy.
*/
public static boolean buildSupportsSnappy() {
- return false;
+ try {
+ ClassLoader parent = Configuration.class.getClassLoader().getParent();
+
+ Class cls = Class.forName(NativeCodeLoader.class.getName(), true, parent);
+
+ boolean value = (boolean)cls.getMethod("buildSupportsSnappy").invoke(null);
+
+ return value;
+ }
+ catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
/**
* @return Library name.
*/
public static String getLibraryName() {
- throw new IllegalStateException();
+ try {
+ ClassLoader parent = Configuration.class.getClassLoader().getParent();
+
+ Class cls = Class.forName(NativeCodeLoader.class.getName(), true, parent);
+
+ Method m = cls.getMethod("getLibraryName");
+
+ String value = (String)m.invoke(null);
+
+ return value;
+ }
+ catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
+ // --------------- copied methods:
+
/**
* Return if native hadoop libraries, if present, can be used for this job.
* @param conf configuration
@@ -59,7 +101,8 @@ public class HadoopNativeCodeLoader {
* used for this job; <code>false</code> otherwise.
*/
public boolean getLoadNativeLibraries(Configuration conf) {
- return false;
+ return conf.getBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,
+ CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT);
}
/**
@@ -68,7 +111,9 @@ public class HadoopNativeCodeLoader {
* @param conf configuration
* @param loadNativeLibraries can native hadoop libraries be loaded
*/
- public void setLoadNativeLibraries(Configuration conf, boolean loadNativeLibraries) {
- // No-op.
+ public void setLoadNativeLibraries(Configuration conf,
+ boolean loadNativeLibraries) {
+ conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,
+ loadNativeLibraries);
}
}