You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@twill.apache.org by ch...@apache.org on 2017/08/04 17:23:10 UTC
twill git commit: (TWILL-237) Twill is using hdfs HAUtil api that is
nont-compatible with hadoop 2.8
Repository: twill
Updated Branches:
refs/heads/master f866b50d7 -> dbbc2a349
(TWILL-237) Twill is using hdfs HAUtil api that is nont-compatible with hadoop 2.8
+ Use Java's MethodHandle (dynamic lang support) rather than Method (reflection) in FileContextLocationUtil
+ Expose static API rather than the handle itself
This closes #55 on Github
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/twill/commit/dbbc2a34
Tree: http://git-wip-us.apache.org/repos/asf/twill/tree/dbbc2a34
Diff: http://git-wip-us.apache.org/repos/asf/twill/diff/dbbc2a34
Branch: refs/heads/master
Commit: dbbc2a349636cbc2f0c5991f5ad5281d97c0cbb0
Parents: f866b50
Author: Sudheesh Katkam <su...@dremio.com>
Authored: Fri Jun 16 15:42:18 2017 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Fri Aug 4 10:22:59 2017 -0700
----------------------------------------------------------------------
.../twill/filesystem/FileContextLocation.java | 6 +-
.../filesystem/FileContextLocationUtil.java | 77 ++++++++++++++++++++
2 files changed, 81 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/twill/blob/dbbc2a34/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocation.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocation.java b/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocation.java
index 8ccd180..f407f01 100644
--- a/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocation.java
+++ b/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocation.java
@@ -19,6 +19,7 @@ package org.apache.twill.filesystem;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
+
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileContext;
@@ -28,7 +29,6 @@ import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.security.AccessControlException;
import java.io.FileNotFoundException;
@@ -41,6 +41,7 @@ import java.util.EnumSet;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
+
import javax.annotation.Nullable;
/**
@@ -162,7 +163,8 @@ final class FileContextLocation implements Location {
// append "port" to the path URI, while the DistributedFileSystem always use the cluster logical
// name, which doesn't allow having port in it.
URI uri = path.toUri();
- if (HAUtil.isLogicalUri(locationFactory.getConfiguration(), uri)) {
+
+ if (FileContextLocationUtil.useLogicalUri(locationFactory.getConfiguration(), uri)) {
try {
// Need to strip out the port if in HA
return new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(),
http://git-wip-us.apache.org/repos/asf/twill/blob/dbbc2a34/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocationUtil.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocationUtil.java b/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocationUtil.java
new file mode 100644
index 0000000..964cfdd
--- /dev/null
+++ b/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocationUtil.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.filesystem;
+
+import com.google.common.base.Throwables;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.HAUtil;
+
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
+import java.net.URI;
+
+/**
+ * Utility class.
+ */
+final class FileContextLocationUtil {
+
+ // To check whether logical URI is needed, hdfs.HAUtil class is used. But the class is meant for internal purposes,
+ // and in Hadoop 2.8, the method was renamed from "isLogicalUri" to "useLogicalUri". So resolve to the
+ // correct method.
+ private static final MethodHandle HA_UTIL_USE_LOGICAL_URI_HANDLE;
+
+ private static MethodHandle lookupInHAUtil(final String methodName)
+ throws NoSuchMethodException, IllegalAccessException {
+ return MethodHandles.publicLookup()
+ .findStatic(HAUtil.class, methodName,
+ MethodType.methodType(boolean.class, new Class[]{Configuration.class, URI.class}));
+ }
+
+ static {
+ MethodHandle handle;
+ try {
+ try {
+ // hadoop version < 2.8
+ handle = lookupInHAUtil("isLogicalUri");
+ } catch (NoSuchMethodException ignored) {
+ try {
+ // hadoop version = 2.8
+ handle = lookupInHAUtil("useLogicalUri");
+ } catch (NoSuchMethodException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+ } catch (IllegalAccessException e) {
+ throw Throwables.propagate(e);
+ }
+ HA_UTIL_USE_LOGICAL_URI_HANDLE = handle;
+ }
+
+ static boolean useLogicalUri(final Configuration configuration, final URI uri) {
+ try {
+ return (Boolean) HA_UTIL_USE_LOGICAL_URI_HANDLE.invoke(configuration, uri);
+ } catch (Throwable e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ private FileContextLocationUtil() {
+ }
+}