You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2013/06/19 00:15:27 UTC
svn commit: r1494341 - in
/hadoop/common/trunk/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/io/nativeio/
src/main/native/src/org/apache/hadoop/io/nativeio/ src/main/winutils/
src/main/winutils/include/ src/test/java/org/apach...
Author: cnauroth
Date: Tue Jun 18 22:15:26 2013
New Revision: 1494341
URL: http://svn.apache.org/r1494341
Log:
HADOOP-9637. Adding Native Fstat for Windows as needed by YARN. Contributed by Chuan Liu.
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/chmod.c
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/chown.c
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/ls.c
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1494341&r1=1494340&r2=1494341&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Jun 18 22:15:26 2013
@@ -809,6 +809,9 @@ Release 2.1.0-beta - UNRELEASED
HADOOP-9599. hadoop-config.cmd doesn't set JAVA_LIBRARY_PATH correctly.
(Mostafa Elhemali via ivanmi)
+ HADOOP-9637. Adding Native Fstat for Windows as needed by YARN. (Chuan Liu
+ via cnauroth)
+
Release 2.0.5-alpha - 06/06/2013
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java?rev=1494341&r1=1494340&r2=1494341&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java Tue Jun 18 22:15:26 2013
@@ -247,7 +247,21 @@ public class NativeIO {
this.groupId = groupId;
this.mode = mode;
}
-
+
+ Stat(String owner, String group, int mode) {
+ if (!Shell.WINDOWS) {
+ this.owner = owner;
+ } else {
+ this.owner = stripDomain(owner);
+ }
+ if (!Shell.WINDOWS) {
+ this.group = group;
+ } else {
+ this.group = stripDomain(group);
+ }
+ this.mode = mode;
+ }
+
@Override
public String toString() {
return "Stat(owner='" + owner + "', group='" + group + "'" +
@@ -273,9 +287,25 @@ public class NativeIO {
* @throws IOException thrown if there was an IO error while obtaining the file stat.
*/
public static Stat getFstat(FileDescriptor fd) throws IOException {
- Stat stat = fstat(fd);
- stat.owner = getName(IdCache.USER, stat.ownerId);
- stat.group = getName(IdCache.GROUP, stat.groupId);
+ Stat stat = null;
+ if (!Shell.WINDOWS) {
+ stat = fstat(fd);
+ stat.owner = getName(IdCache.USER, stat.ownerId);
+ stat.group = getName(IdCache.GROUP, stat.groupId);
+ } else {
+ try {
+ stat = fstat(fd);
+ } catch (NativeIOException nioe) {
+ if (nioe.getErrorCode() == 6) {
+ throw new NativeIOException("The handle is invalid.",
+ Errno.EBADF);
+ } else {
+ LOG.warn(String.format("NativeIO.getFstat error (%d): %s",
+ nioe.getErrorCode(), nioe.getMessage()));
+ throw new NativeIOException("Unknown error", Errno.UNKNOWN);
+ }
+ }
+ }
return stat;
}
@@ -448,14 +478,27 @@ public class NativeIO {
new ConcurrentHashMap<Long, CachedUid>();
private static long cacheTimeout;
private static boolean initialized = false;
+
+ /**
+ * The Windows logon name has two part, NetBIOS domain name and
+ * user account name, of the format DOMAIN\UserName. This method
+ * will remove the domain part of the full logon name.
+ *
+ * @param the full principal name containing the domain
+ * @return name with domain removed
+ */
+ private static String stripDomain(String name) {
+ int i = name.indexOf('\\');
+ if (i != -1)
+ name = name.substring(i + 1);
+ return name;
+ }
public static String getOwner(FileDescriptor fd) throws IOException {
ensureInitialized();
if (Shell.WINDOWS) {
String owner = Windows.getOwner(fd);
- int i = owner.indexOf('\\');
- if (i != -1)
- owner = owner.substring(i + 1);
+ owner = stripDomain(owner);
return owner;
} else {
long uid = POSIX.getUIDforFDOwnerforOwner(fd);
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c?rev=1494341&r1=1494340&r2=1494341&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c Tue Jun 18 22:15:26 2013
@@ -50,6 +50,7 @@
// the NativeIO$POSIX$Stat inner class and its constructor
static jclass stat_clazz;
static jmethodID stat_ctor;
+static jmethodID stat_ctor2;
// the NativeIOException class and its constructor
static jclass nioe_clazz;
@@ -84,10 +85,12 @@ static int workaround_non_threadsafe_cal
return result;
}
-#ifdef UNIX
static void stat_init(JNIEnv *env, jclass nativeio_class) {
+ jclass clazz = NULL;
+ jclass obj_class = NULL;
+ jmethodID obj_ctor = NULL;
// Init Stat
- jclass clazz = (*env)->FindClass(env, "org/apache/hadoop/io/nativeio/NativeIO$POSIX$Stat");
+ clazz = (*env)->FindClass(env, "org/apache/hadoop/io/nativeio/NativeIO$POSIX$Stat");
if (!clazz) {
return; // exception has been raised
}
@@ -100,12 +103,16 @@ static void stat_init(JNIEnv *env, jclas
if (!stat_ctor) {
return; // exception has been raised
}
-
- jclass obj_class = (*env)->FindClass(env, "java/lang/Object");
+ stat_ctor2 = (*env)->GetMethodID(env, stat_clazz, "<init>",
+ "(Ljava/lang/String;Ljava/lang/String;I)V");
+ if (!stat_ctor2) {
+ return; // exception has been raised
+ }
+ obj_class = (*env)->FindClass(env, "java/lang/Object");
if (!obj_class) {
return; // exception has been raised
}
- jmethodID obj_ctor = (*env)->GetMethodID(env, obj_class,
+ obj_ctor = (*env)->GetMethodID(env, obj_class,
"<init>", "()V");
if (!obj_ctor) {
return; // exception has been raised
@@ -130,7 +137,6 @@ static void stat_deinit(JNIEnv *env) {
pw_lock_object = NULL;
}
}
-#endif
static void nioe_init(JNIEnv *env) {
// Init NativeIOException
@@ -168,10 +174,8 @@ static void nioe_deinit(JNIEnv *env) {
JNIEXPORT void JNICALL
Java_org_apache_hadoop_io_nativeio_NativeIO_initNative(
JNIEnv *env, jclass clazz) {
-#ifdef UNIX
stat_init(env, clazz);
PASS_EXCEPTIONS_GOTO(env, error);
-#endif
nioe_init(env);
PASS_EXCEPTIONS_GOTO(env, error);
fd_init(env);
@@ -229,9 +233,44 @@ cleanup:
#endif
#ifdef WINDOWS
- THROW(env, "java/io/IOException",
- "The function POSIX.fstat() is not supported on Windows");
- return NULL;
+ LPWSTR owner = NULL;
+ LPWSTR group = NULL;
+ int mode;
+ jstring jstr_owner = NULL;
+ jstring jstr_group = NULL;
+ int rc;
+ jobject ret = NULL;
+ HANDLE hFile = (HANDLE) fd_get(env, fd_object);
+ PASS_EXCEPTIONS_GOTO(env, cleanup);
+
+ rc = FindFileOwnerAndPermissionByHandle(hFile, &owner, &group, &mode);
+ if (rc != ERROR_SUCCESS) {
+ throw_ioe(env, rc);
+ goto cleanup;
+ }
+
+ jstr_owner = (*env)->NewString(env, owner, (jsize) wcslen(owner));
+ if (jstr_owner == NULL) goto cleanup;
+
+ jstr_group = (*env)->NewString(env, group, (jsize) wcslen(group));;
+ if (jstr_group == NULL) goto cleanup;
+
+ ret = (*env)->NewObject(env, stat_clazz, stat_ctor2,
+ jstr_owner, jstr_group, (jint)mode);
+
+cleanup:
+ if (ret == NULL) {
+ if (jstr_owner != NULL)
+ (*env)->ReleaseStringChars(env, jstr_owner, owner);
+
+ if (jstr_group != NULL)
+ (*env)->ReleaseStringChars(env, jstr_group, group);
+ }
+
+ LocalFree(owner);
+ LocalFree(group);
+
+ return ret;
#endif
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/chmod.c
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/chmod.c?rev=1494341&r1=1494340&r2=1494341&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/chmod.c (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/chmod.c Tue Jun 18 22:15:26 2013
@@ -561,22 +561,11 @@ static BOOL ConvertActionsToMask(__in LP
{
MODE_CHANGE_ACTION const *curr = NULL;
- BY_HANDLE_FILE_INFORMATION fileInformation;
DWORD dwErrorCode = ERROR_SUCCESS;
INT mode = 0;
- dwErrorCode = GetFileInformationByName(path, FALSE, &fileInformation);
- if (dwErrorCode != ERROR_SUCCESS)
- {
- ReportErrorCode(L"GetFileInformationByName", dwErrorCode);
- return FALSE;
- }
- if (IsDirFileInfo(&fileInformation))
- {
- mode |= UX_DIRECTORY;
- }
- dwErrorCode = FindFileOwnerAndPermission(path, NULL, NULL, &mode);
+ dwErrorCode = FindFileOwnerAndPermission(path, FALSE, NULL, NULL, &mode);
if (dwErrorCode != ERROR_SUCCESS)
{
ReportErrorCode(L"FindFileOwnerAndPermission", dwErrorCode);
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/chown.c
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/chown.c?rev=1494341&r1=1494340&r2=1494341&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/chown.c (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/chown.c Tue Jun 18 22:15:26 2013
@@ -52,7 +52,7 @@ static DWORD ChangeFileOwnerBySid(__in L
// Get a pointer to the existing owner information and DACL
//
- dwRtnCode = FindFileOwnerAndPermission(longPathName, NULL, NULL, &oldMode);
+ dwRtnCode = FindFileOwnerAndPermission(longPathName, FALSE, NULL, NULL, &oldMode);
if (dwRtnCode != ERROR_SUCCESS)
{
goto ChangeFileOwnerByNameEnd;
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h?rev=1494341&r1=1494340&r2=1494341&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h Tue Jun 18 22:15:26 2013
@@ -63,6 +63,7 @@ enum UnixAclMask
UX_U_WRITE = 00200, // S_IWUSR
UX_U_READ = 00400, // S_IRUSR
UX_DIRECTORY = 0040000, // S_IFDIR
+ UX_REGULAR = 0100000, // S_IFREG
UX_SYMLINK = 0120000, // S_IFLNK
};
@@ -130,6 +131,13 @@ BOOL IsDirFileInfo(const BY_HANDLE_FILE_
DWORD FindFileOwnerAndPermission(
__in LPCWSTR pathName,
+ __in BOOL followLink,
+ __out_opt LPWSTR *pOwnerName,
+ __out_opt LPWSTR *pGroupName,
+ __out_opt PINT pMask);
+
+DWORD FindFileOwnerAndPermissionByHandle(
+ __in HANDLE fileHandle,
__out_opt LPWSTR *pOwnerName,
__out_opt LPWSTR *pGroupName,
__out_opt PINT pMask);
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c?rev=1494341&r1=1494340&r2=1494341&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c Tue Jun 18 22:15:26 2013
@@ -707,6 +707,71 @@ CheckAccessEnd:
return dwRtnCode;
}
+
+//----------------------------------------------------------------------------
+// Function: FindFileOwnerAndPermissionByHandle
+//
+// Description:
+// Find the owner, primary group and permissions of a file object given the
+// the file object handle. The function will always follow symbolic links.
+//
+// Returns:
+// ERROR_SUCCESS: on success
+// Error code otherwise
+//
+// Notes:
+// - Caller needs to destroy the memeory of owner and group names by calling
+// LocalFree() function.
+//
+// - If the user or group name does not exist, the user or group SID will be
+// returned as the name.
+//
+DWORD FindFileOwnerAndPermissionByHandle(
+ __in HANDLE fileHandle,
+ __out_opt LPWSTR *pOwnerName,
+ __out_opt LPWSTR *pGroupName,
+ __out_opt PINT pMask)
+{
+ LPWSTR path = NULL;
+ DWORD cchPathLen = 0;
+ DWORD dwRtnCode = ERROR_SUCCESS;
+
+ DWORD ret = ERROR_SUCCESS;
+
+ dwRtnCode = GetFinalPathNameByHandle(fileHandle, path, cchPathLen, 0);
+ if (dwRtnCode == 0)
+ {
+ ret = GetLastError();
+ goto FindFileOwnerAndPermissionByHandleEnd;
+ }
+ cchPathLen = dwRtnCode;
+ path = (LPWSTR) LocalAlloc(LPTR, cchPathLen * sizeof(WCHAR));
+ if (path == NULL)
+ {
+ ret = GetLastError();
+ goto FindFileOwnerAndPermissionByHandleEnd;
+ }
+
+ dwRtnCode = GetFinalPathNameByHandle(fileHandle, path, cchPathLen, 0);
+ if (dwRtnCode != cchPathLen - 1)
+ {
+ ret = GetLastError();
+ goto FindFileOwnerAndPermissionByHandleEnd;
+ }
+
+ dwRtnCode = FindFileOwnerAndPermission(path, TRUE, pOwnerName, pGroupName, pMask);
+ if (dwRtnCode != ERROR_SUCCESS)
+ {
+ ret = dwRtnCode;
+ goto FindFileOwnerAndPermissionByHandleEnd;
+ }
+
+FindFileOwnerAndPermissionByHandleEnd:
+ LocalFree(path);
+ return ret;
+}
+
+
//----------------------------------------------------------------------------
// Function: FindFileOwnerAndPermission
//
@@ -726,6 +791,7 @@ CheckAccessEnd:
//
DWORD FindFileOwnerAndPermission(
__in LPCWSTR pathName,
+ __in BOOL followLink,
__out_opt LPWSTR *pOwnerName,
__out_opt LPWSTR *pGroupName,
__out_opt PINT pMask)
@@ -740,6 +806,9 @@ DWORD FindFileOwnerAndPermission(
DWORD cbSid = SECURITY_MAX_SID_SIZE;
PACL pDacl = NULL;
+ BOOL isSymlink;
+ BY_HANDLE_FILE_INFORMATION fileInformation;
+
ACCESS_MASK ownerAccessRights = 0;
ACCESS_MASK groupAccessRights = 0;
ACCESS_MASK worldAccessRights = 0;
@@ -801,6 +870,28 @@ DWORD FindFileOwnerAndPermission(
if (pMask == NULL) goto FindFileOwnerAndPermissionEnd;
+ dwRtnCode = GetFileInformationByName(pathName,
+ followLink, &fileInformation);
+ if (dwRtnCode != ERROR_SUCCESS)
+ {
+ ret = dwRtnCode;
+ goto FindFileOwnerAndPermissionEnd;
+ }
+
+ dwRtnCode = SymbolicLinkCheck(pathName, &isSymlink);
+ if (dwRtnCode != ERROR_SUCCESS)
+ {
+ ret = dwRtnCode;
+ goto FindFileOwnerAndPermissionEnd;
+ }
+
+ if (isSymlink)
+ *pMask |= UX_SYMLINK;
+ else if (IsDirFileInfo(&fileInformation))
+ *pMask |= UX_DIRECTORY;
+ else
+ *pMask |= UX_REGULAR;
+
if ((dwRtnCode = GetEffectiveRightsForSid(pSd,
psidOwner, &ownerAccessRights)) != ERROR_SUCCESS)
{
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/ls.c
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/ls.c?rev=1494341&r1=1494340&r2=1494341&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/ls.c (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/ls.c Tue Jun 18 22:15:26 2013
@@ -253,8 +253,6 @@ int Ls(__in int argc, __in_ecount(argc)
LARGE_INTEGER fileSize;
- BOOL isSymlink = FALSE;
-
int ret = EXIT_FAILURE;
int optionsMask = 0;
@@ -290,19 +288,8 @@ int Ls(__in int argc, __in_ecount(argc)
goto LsEnd;
}
- dwErrorCode = SymbolicLinkCheck(longPathName, &isSymlink);
- if (dwErrorCode != ERROR_SUCCESS)
- {
- ReportErrorCode(L"IsSymbolicLink", dwErrorCode);
- goto LsEnd;
- }
-
- if (isSymlink)
- unixAccessMode |= UX_SYMLINK;
- else if (IsDirFileInfo(&fileInformation))
- unixAccessMode |= UX_DIRECTORY;
-
dwErrorCode = FindFileOwnerAndPermission(longPathName,
+ optionsMask & CmdLineOptionFollowSymlink,
&ownerName, &groupName, &unixAccessMode);
if (dwErrorCode != ERROR_SUCCESS)
{
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java?rev=1494341&r1=1494340&r2=1494341&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java Tue Jun 18 22:15:26 2013
@@ -26,6 +26,7 @@ import java.io.FileWriter;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicReference;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import org.junit.Assert;
@@ -42,6 +43,7 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.util.Time;
@@ -64,17 +66,23 @@ public class TestNativeIO {
@Test (timeout = 30000)
public void testFstat() throws Exception {
- if (Path.WINDOWS) {
- return;
- }
-
FileOutputStream fos = new FileOutputStream(
new File(TEST_DIR, "testfstat"));
NativeIO.POSIX.Stat stat = NativeIO.POSIX.getFstat(fos.getFD());
fos.close();
LOG.info("Stat: " + String.valueOf(stat));
- assertEquals(System.getProperty("user.name"), stat.getOwner());
+ String owner = stat.getOwner();
+ String expectedOwner = System.getProperty("user.name");
+ if (Path.WINDOWS) {
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser(expectedOwner);
+ final String adminsGroupString = "Administrators";
+ if (Arrays.asList(ugi.getGroupNames()).contains(adminsGroupString)) {
+ expectedOwner = adminsGroupString;
+ }
+ }
+ assertEquals(expectedOwner, owner);
assertNotNull(stat.getGroup());
assertTrue(!stat.getGroup().isEmpty());
assertEquals("Stat mode field should indicate a regular file",
@@ -136,10 +144,6 @@ public class TestNativeIO {
@Test (timeout = 30000)
public void testFstatClosedFd() throws Exception {
- if (Path.WINDOWS) {
- return;
- }
-
FileOutputStream fos = new FileOutputStream(
new File(TEST_DIR, "testfstat2"));
fos.close();