You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pb...@apache.org on 2017/09/22 11:00:03 UTC
oozie git commit: OOZIE-3054 Disable erasure coding for sharelib if
Oozie runs on Hadoop 3 (pbacsko)
Repository: oozie
Updated Branches:
refs/heads/master a5f9aa54a -> f730573b8
OOZIE-3054 Disable erasure coding for sharelib if Oozie runs on Hadoop 3 (pbacsko)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f730573b
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f730573b
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f730573b
Branch: refs/heads/master
Commit: f730573b83ccddc35158b4c1b66de9ae5147c977
Parents: a5f9aa5
Author: Peter Bacsko <pb...@cloudera.com>
Authored: Fri Sep 22 12:58:37 2017 +0200
Committer: Peter Bacsko <pb...@cloudera.com>
Committed: Fri Sep 22 12:58:37 2017 +0200
----------------------------------------------------------------------
release-log.txt | 1 +
.../apache/oozie/tools/ECPolicyDisabler.java | 114 +++++++++++++++++++
.../apache/oozie/tools/OozieSharelibCLI.java | 10 +-
3 files changed, 120 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/f730573b/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 803da2f..250a51e 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.0.0 release (trunk - unreleased)
+OOZIE-3054 Disable erasure coding for sharelib if Oozie runs on Hadoop 3 (pbacsko)
OOZIE-3048 Check El Functions for the coordinator action (satishsaley)
OOZIE-3058 nocleanup option is missing in oozie-coordinator-0.5.xsd (satishsaley)
OOZIE-2909 LauncherAM: rewrite UGI calls (gezapeti)
http://git-wip-us.apache.org/repos/asf/oozie/blob/f730573b/tools/src/main/java/org/apache/oozie/tools/ECPolicyDisabler.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/oozie/tools/ECPolicyDisabler.java b/tools/src/main/java/org/apache/oozie/tools/ECPolicyDisabler.java
new file mode 100644
index 0000000..2d7f1f8
--- /dev/null
+++ b/tools/src/main/java/org/apache/oozie/tools/ECPolicyDisabler.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.tools;
+
+import java.lang.reflect.Method;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+
+/**
+ * Utility class which can disable Erasure Coding for a given path.
+ *
+ * Note that Erasure Coding was introduced in Hadoop 3, so in order for this class
+ * to be compilable, reflection is used. Later, when we drop support for Hadoop 2.x,
+ * this can be rewritten.
+ */
+public final class ECPolicyDisabler {
+ private static final String GETREPLICATIONPOLICY_METHOD = "getReplicationPolicy";
+ private static final String ERASURECODING_POLICIES_CLASS = "org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies";
+ private static final String GETNAME_METHOD = "getName";
+ private static final String SETERASURECODINGPOLICY_METHOD = "setErasureCodingPolicy";
+ private static final String GETERASURECODINGPOLICY_METHOD = "getErasureCodingPolicy";
+
+ public static void tryDisableECPolicyForPath(FileSystem fs, Path path) {
+ if (fs instanceof DistributedFileSystem && supportsErasureCoding()) {
+ System.out.println("Found Hadoop that supports Erasure Coding. Trying to disable Erasure Coding for path: "+ path);
+ DistributedFileSystem dfs = (DistributedFileSystem) fs;
+ final Object replicationPolicy = getReplicationPolicy();
+ Method getErasureCodingPolicyMethod = getMethod(dfs, GETERASURECODINGPOLICY_METHOD);
+ final Object currentECPolicy = invokeMethod(getErasureCodingPolicyMethod, dfs, path);
+
+ if (currentECPolicy != replicationPolicy) {
+ Method setECPolicyMethod = getMethod(dfs, SETERASURECODINGPOLICY_METHOD);
+ Method policyGetNameMethod = getMethod(replicationPolicy, GETNAME_METHOD);
+
+ String name = (String) invokeMethod(policyGetNameMethod, replicationPolicy);
+
+ invokeMethod(setECPolicyMethod, dfs, path, name);
+ System.out.println("Done");
+ } else {
+ System.out.println("Current policy is already replication");
+ }
+ } else {
+ System.out.println("Found Hadoop that does not support Erasure Coding. Not taking any action.");
+ }
+ }
+
+ private static boolean supportsErasureCoding() {
+ try {
+ getECPoliciesClass();
+ return true;
+ } catch (ClassNotFoundException e) {
+ return false;
+ }
+ }
+
+ private static Object getReplicationPolicy() {
+ try {
+ Class<?> c = getECPoliciesClass();
+ Method m = c.getMethod(GETREPLICATIONPOLICY_METHOD);
+ return m.invoke(null);
+ } catch (Exception e) {
+ System.err.println("Error accessing method with reflection");
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static Class<?> getECPoliciesClass() throws ClassNotFoundException {
+ return Class.forName(ERASURECODING_POLICIES_CLASS);
+ }
+
+ private static Method getMethod(Object object, String methodName) {
+ Method[] methods = object.getClass().getMethods();
+ Method method = null;
+ for (Method m : methods) {
+ if (m.getName().equals(methodName)) {
+ method = m;
+ break;
+ }
+ }
+
+ if (method == null) {
+ throw new RuntimeException("Method " + methodName + "() not found");
+ }
+
+ return method;
+ }
+
+ private static Object invokeMethod(Method m, Object instance, Object... args) {
+ try {
+ return m.invoke(instance, args);
+ } catch (Exception e) {
+ System.err.println("Error invoking method with reflection");
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/f730573b/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java b/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java
index a844aa0..c565d9d 100644
--- a/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java
+++ b/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java
@@ -160,6 +160,11 @@ public class OozieSharelibCLI {
WorkflowAppService lwas = services.get(WorkflowAppService.class);
HadoopAccessorService has = services.get(HadoopAccessorService.class);
Path dstPath = lwas.getSystemLibPath();
+ URI uri = new Path(hdfsUri).toUri();
+ Configuration fsConf = has.createConfiguration(uri.getAuthority());
+ FileSystem fs = FileSystem.get(uri, fsConf);
+
+ ECPolicyDisabler.tryDisableECPolicyForPath(fs, dstPath);
if (sharelibAction.equals(CREATE_CMD) || sharelibAction.equals(UPGRADE_CMD)){
dstPath= new Path(dstPath.toString() + Path.SEPARATOR + SHARE_LIB_PREFIX + getTimestampDirectory() );
@@ -167,11 +172,6 @@ public class OozieSharelibCLI {
System.out.println("the destination path for sharelib is: " + dstPath);
- URI uri = new Path(hdfsUri).toUri();
- Configuration fsConf = has.createConfiguration(uri.getAuthority());
- FileSystem fs = FileSystem.get(uri, fsConf);
-
-
if (!srcFile.exists()){
throw new IOException(srcPath + " cannot be found");
}