You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zh...@apache.org on 2015/01/12 19:18:46 UTC
[04/34] hadoop git commit: HDFS-7564. NFS gateway dynamically reload
UID/GID mapping file /etc/nfs.map. Contributed by Yongjun Zhang
HDFS-7564. NFS gateway dynamically reload UID/GID mapping file /etc/nfs.map. Contributed by Yongjun Zhang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2fbde44c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2fbde44c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2fbde44c
Branch: refs/heads/HDFS-EC
Commit: 2fbde44c9ec1fef1a3114ba8f67ff84ede8fece6
Parents: 7a04642
Author: Brandon Li <br...@apache.org>
Authored: Tue Jan 6 16:19:39 2015 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Mon Jan 12 10:17:59 2015 -0800
----------------------------------------------------------------------
.../hadoop/security/ShellBasedIdMapping.java | 75 +++++++++++------
.../security/TestShellBasedIdMapping.java | 88 ++++++++++++++++++--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
3 files changed, 137 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fbde44c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
index 28ab248..fd362d0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
@@ -75,6 +75,10 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
private final File staticMappingFile;
private StaticMapping staticMapping = null;
+ // Last time the static map was modified, measured time difference in
+ // milliseconds since midnight, January 1, 1970 UTC
+ private long lastModificationTimeStaticMap = 0;
+
private boolean constructFullMapAtInit = false;
// Used for parsing the static mapping file.
@@ -88,7 +92,6 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
// Maps for id to name map. Guarded by this object monitor lock
private BiMap<Integer, String> uidNameMap = HashBiMap.create();
private BiMap<Integer, String> gidNameMap = HashBiMap.create();
-
private long lastUpdateTime = 0; // Last time maps were updated
/*
@@ -118,7 +121,7 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
conf.get(IdMappingConstant.STATIC_ID_MAPPING_FILE_KEY,
IdMappingConstant.STATIC_ID_MAPPING_FILE_DEFAULT);
staticMappingFile = new File(staticFilePath);
-
+ updateStaticMapping();
updateMaps();
}
@@ -290,20 +293,42 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
return true;
}
- private synchronized void initStaticMapping() throws IOException {
- staticMapping = new StaticMapping(
- new HashMap<Integer, Integer>(), new HashMap<Integer, Integer>());
+ private synchronized void updateStaticMapping() throws IOException {
+ final boolean init = (staticMapping == null);
+ //
+ // if the static mapping file
+ // - was modified after last update, load the map again;
+ // - did not exist but was added since last update, load the map;
+ // - existed before but deleted since last update, clear the map
+ //
if (staticMappingFile.exists()) {
- LOG.info("Using '" + staticMappingFile + "' for static UID/GID mapping...");
- staticMapping = parseStaticMap(staticMappingFile);
+ // check modification time, reload the file if the last modification
+ // time changed since prior load.
+ long lmTime = staticMappingFile.lastModified();
+ if (lmTime != lastModificationTimeStaticMap) {
+ LOG.info(init? "Using " : "Reloading " + "'" + staticMappingFile
+ + "' for static UID/GID mapping...");
+ lastModificationTimeStaticMap = lmTime;
+ staticMapping = parseStaticMap(staticMappingFile);
+ }
} else {
- LOG.info("Not doing static UID/GID mapping because '" + staticMappingFile
- + "' does not exist.");
+ if (init) {
+ staticMapping = new StaticMapping(new HashMap<Integer, Integer>(),
+ new HashMap<Integer, Integer>());
+ }
+ if (lastModificationTimeStaticMap != 0 || init) {
+ // print the following log at initialization or when the static
+ // mapping file was deleted after prior load
+ LOG.info("Not doing static UID/GID mapping because '"
+ + staticMappingFile + "' does not exist.");
+ }
+ lastModificationTimeStaticMap = 0;
+ staticMapping.clear();
}
- }
+ }
/*
- * Reset the maps to empty.
+ * Refresh static map, and reset the other maps to empty.
* For testing code, a full map may be re-constructed here when the object
* was created with constructFullMapAtInit being set to true.
*/
@@ -314,15 +339,16 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
if (constructFullMapAtInit) {
loadFullMaps();
+ // set constructFullMapAtInit to false to allow testing code to
+ // do incremental update to maps after initial construction
+ constructFullMapAtInit = false;
} else {
+ updateStaticMapping();
clearNameMaps();
}
}
synchronized private void loadFullUserMap() throws IOException {
- if (staticMapping == null) {
- initStaticMapping();
- }
BiMap<Integer, String> uMap = HashBiMap.create();
if (OS.startsWith("Mac")) {
updateMapInternal(uMap, "user", MAC_GET_ALL_USERS_CMD, "\\s+",
@@ -336,9 +362,6 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
}
synchronized private void loadFullGroupMap() throws IOException {
- if (staticMapping == null) {
- initStaticMapping();
- }
BiMap<Integer, String> gMap = HashBiMap.create();
if (OS.startsWith("Mac")) {
@@ -353,7 +376,6 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
}
synchronized private void loadFullMaps() throws IOException {
- initStaticMapping();
loadFullUserMap();
loadFullGroupMap();
}
@@ -443,9 +465,7 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
}
boolean updated = false;
- if (staticMapping == null) {
- initStaticMapping();
- }
+ updateStaticMapping();
if (OS.startsWith("Linux")) {
if (isGrp) {
@@ -481,9 +501,7 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
}
boolean updated = false;
- if (staticMapping == null) {
- initStaticMapping();
- }
+ updateStaticMapping();
if (OS.startsWith("Linux")) {
if (isGrp) {
@@ -547,6 +565,15 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
this.uidMapping = new PassThroughMap<Integer>(uidMapping);
this.gidMapping = new PassThroughMap<Integer>(gidMapping);
}
+
+ public void clear() {
+ uidMapping.clear();
+ gidMapping.clear();
+ }
+
+ public boolean isNonEmpty() {
+ return uidMapping.size() > 0 || gidMapping.size() > 0;
+ }
}
static StaticMapping parseStaticMap(File staticMapFile)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fbde44c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java
index 857c706..e6e1d73 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java
@@ -41,6 +41,13 @@ public class TestShellBasedIdMapping {
private static final Map<Integer, Integer> EMPTY_PASS_THROUGH_MAP =
new PassThroughMap<Integer>();
+ private void createStaticMapFile(final File smapFile, final String smapStr)
+ throws IOException {
+ OutputStream out = new FileOutputStream(smapFile);
+ out.write(smapStr.getBytes());
+ out.close();
+ }
+
@Test
public void testStaticMapParsing() throws IOException {
File tempStaticMapFile = File.createTempFile("nfs-", ".map");
@@ -58,11 +65,10 @@ public class TestShellBasedIdMapping {
"gid 12 202\n" +
"uid 4294967294 123\n" +
"gid 4294967295 321";
- OutputStream out = new FileOutputStream(tempStaticMapFile);
- out.write(staticMapFileContents.getBytes());
- out.close();
- StaticMapping parsedMap = ShellBasedIdMapping.parseStaticMap(tempStaticMapFile);
-
+ createStaticMapFile(tempStaticMapFile, staticMapFileContents);
+
+ StaticMapping parsedMap =
+ ShellBasedIdMapping.parseStaticMap(tempStaticMapFile);
assertEquals(10, (int)parsedMap.uidMapping.get(100));
assertEquals(11, (int)parsedMap.uidMapping.get(201));
assertEquals(12, (int)parsedMap.uidMapping.get(301));
@@ -120,6 +126,78 @@ public class TestShellBasedIdMapping {
assertEquals(498, (int)gMap.inverse().get("mapred2"));
}
+ // Test staticMap refreshing
+ @Test
+ public void testStaticMapUpdate() throws IOException {
+ File tempStaticMapFile = File.createTempFile("nfs-", ".map");
+ tempStaticMapFile.delete();
+ Configuration conf = new Configuration();
+ conf.setLong(IdMappingConstant.USERGROUPID_UPDATE_MILLIS_KEY, 1000);
+ conf.set(IdMappingConstant.STATIC_ID_MAPPING_FILE_KEY,
+ tempStaticMapFile.getPath());
+
+ ShellBasedIdMapping refIdMapping =
+ new ShellBasedIdMapping(conf, true);
+ ShellBasedIdMapping incrIdMapping = new ShellBasedIdMapping(conf);
+
+ BiMap<Integer, String> uidNameMap = refIdMapping.getUidNameMap();
+ BiMap<Integer, String> gidNameMap = refIdMapping.getGidNameMap();
+
+ // Force empty map, to see effect of incremental map update of calling
+ // getUid()
+ incrIdMapping.clearNameMaps();
+ uidNameMap = refIdMapping.getUidNameMap();
+ {
+ BiMap.Entry<Integer, String> me = uidNameMap.entrySet().iterator().next();
+ Integer id = me.getKey();
+ String name = me.getValue();
+
+ // The static map is empty, so the id found for "name" would be
+ // the same as "id"
+ Integer nid = incrIdMapping.getUid(name);
+ assertEquals(id, nid);
+
+ // Clear map and update staticMap file
+ incrIdMapping.clearNameMaps();
+ Integer rid = id + 10000;
+ String smapStr = "uid " + rid + " " + id;
+ createStaticMapFile(tempStaticMapFile, smapStr);
+
+ // Now the id found for "name" should be the id specified by
+ // the staticMap
+ nid = incrIdMapping.getUid(name);
+ assertEquals(rid, nid);
+ }
+
+ // Force empty map, to see effect of incremental map update of calling
+ // getGid()
+ incrIdMapping.clearNameMaps();
+ gidNameMap = refIdMapping.getGidNameMap();
+ {
+ BiMap.Entry<Integer, String> me = gidNameMap.entrySet().iterator().next();
+ Integer id = me.getKey();
+ String name = me.getValue();
+
+ // The static map is empty, so the id found for "name" would be
+ // the same as "id"
+ Integer nid = incrIdMapping.getGid(name);
+ assertEquals(id, nid);
+
+ // Clear map and update staticMap file
+ incrIdMapping.clearNameMaps();
+ Integer rid = id + 10000;
+ String smapStr = "gid " + rid + " " + id;
+ // Sleep a bit to avoid that two changes have the same modification time
+ try {Thread.sleep(1000);} catch (InterruptedException e) {}
+ createStaticMapFile(tempStaticMapFile, smapStr);
+
+ // Now the id found for "name" should be the id specified by
+ // the staticMap
+ nid = incrIdMapping.getGid(name);
+ assertEquals(rid, nid);
+ }
+ }
+
@Test
public void testDuplicates() throws IOException {
assumeTrue(!Shell.WINDOWS);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fbde44c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e9d6ad2..a76f2a8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -644,6 +644,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7583. Fix findbug in TransferFsImage.java (vinayakumarb)
+ HDFS-7564. NFS gateway dynamically reload UID/GID mapping file /etc/nfs.map
+ (Yongjun Zhang via brandonli)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES