You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by km...@apache.org on 2019/09/12 06:38:15 UTC
[oozie] branch master updated: OOZIE-3542 Handle better old Hdfs
implementations in ECPolicyDisabler (zsombor dionusos via kmarton)
This is an automated email from the ASF dual-hosted git repository.
kmarton pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/oozie.git
The following commit(s) were added to refs/heads/master by this push:
new e1c865e OOZIE-3542 Handle better old Hdfs implementations in ECPolicyDisabler (zsombor dionusos via kmarton)
e1c865e is described below
commit e1c865e1d898443cbc918d7deaa622ce0556e3e1
Author: kmarton <km...@apache.org>
AuthorDate: Thu Sep 12 08:37:55 2019 +0200
OOZIE-3542 Handle better old Hdfs implementations in ECPolicyDisabler (zsombor dionusos via kmarton)
---
release-log.txt | 1 +
.../org/apache/oozie/tools/ECPolicyDisabler.java | 53 +++++++++-
.../hdfs/protocol/SystemErasureCodingPolicies.java | 39 +++++++
.../apache/oozie/tools/TestECPolicyDisabler.java | 112 +++++++++++++++++++++
4 files changed, 201 insertions(+), 4 deletions(-)
diff --git a/release-log.txt b/release-log.txt
index 8aef769..550debf 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.2.0 release (trunk - unreleased)
+OOZIE-3542 Handle better old Hdfs implementations in ECPolicyDisabler (zsombor dionusos via kmarton)
OOZIE-3540 Use StringBuilder instead of StringBuffer if concurrent access is not required (zsombor via asalamon74)
OOZIE-3539 amend Support http proxy/basic authentication in the command line client (zsombor via asalamon74)
OOZIE-3539 Support http proxy/basic authentication in the command line client (zsombor via asalamon74)
diff --git a/tools/src/main/java/org/apache/oozie/tools/ECPolicyDisabler.java b/tools/src/main/java/org/apache/oozie/tools/ECPolicyDisabler.java
index 2d7f1f8..6d51342 100644
--- a/tools/src/main/java/org/apache/oozie/tools/ECPolicyDisabler.java
+++ b/tools/src/main/java/org/apache/oozie/tools/ECPolicyDisabler.java
@@ -18,11 +18,14 @@
package org.apache.oozie.tools;
+import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcErrorCodeProto;
/**
* Utility class which can disable Erasure Coding for a given path.
@@ -38,7 +41,28 @@ public final class ECPolicyDisabler {
private static final String SETERASURECODINGPOLICY_METHOD = "setErasureCodingPolicy";
private static final String GETERASURECODINGPOLICY_METHOD = "getErasureCodingPolicy";
+ enum Result {
+ DONE, NO_SUCH_METHOD, ALREADY_SET, NOT_SUPPORTED;
+ }
+
public static void tryDisableECPolicyForPath(FileSystem fs, Path path) {
+ switch (check(fs, path)) {
+ case DONE:
+ System.out.println("Done");
+ break;
+ case ALREADY_SET:
+ System.out.println("Current policy is already replication");
+ break;
+ case NOT_SUPPORTED:
+ System.out.println("Found Hadoop that does not support Erasure Coding. Not taking any action.");
+ break;
+ case NO_SUCH_METHOD:
+ System.out.println("HDFS Namenode doesn't support Erasure Coding.");
+ break;
+ }
+ }
+
+ static Result check(FileSystem fs, Path path) {
if (fs instanceof DistributedFileSystem && supportsErasureCoding()) {
System.out.println("Found Hadoop that supports Erasure Coding. Trying to disable Erasure Coding for path: "+ path);
DistributedFileSystem dfs = (DistributedFileSystem) fs;
@@ -52,16 +76,34 @@ public final class ECPolicyDisabler {
String name = (String) invokeMethod(policyGetNameMethod, replicationPolicy);
- invokeMethod(setECPolicyMethod, dfs, path, name);
- System.out.println("Done");
+ try {
+ invokeMethod(setECPolicyMethod, dfs, path, name);
+ } catch (RuntimeException e) {
+ RpcErrorCodeProto errorCode = unwrapRemote(e);
+ if (errorCode == RpcErrorCodeProto.ERROR_NO_SUCH_METHOD) {
+ return Result.NO_SUCH_METHOD;
+ }
+ throw e;
+ }
+ return Result.DONE;
} else {
- System.out.println("Current policy is already replication");
+ return Result.ALREADY_SET;
}
} else {
- System.out.println("Found Hadoop that does not support Erasure Coding. Not taking any action.");
+ return Result.NOT_SUPPORTED;
}
}
+ private static RpcErrorCodeProto unwrapRemote(Throwable e) {
+ if (e instanceof InvocationTargetException) {
+ return unwrapRemote(e.getCause());
+ }
+ if (e instanceof RuntimeException) {
+ return unwrapRemote(e.getCause());
+ }
+ return (e instanceof RemoteException) ? ((RemoteException) e).getErrorCode() : null;
+ }
+
private static boolean supportsErasureCoding() {
try {
getECPoliciesClass();
@@ -106,6 +148,9 @@ public final class ECPolicyDisabler {
private static Object invokeMethod(Method m, Object instance, Object... args) {
try {
return m.invoke(instance, args);
+ } catch (RuntimeException e) {
+ System.err.println("Error invoking method with reflection");
+ return e;
} catch (Exception e) {
System.err.println("Error invoking method with reflection");
throw new RuntimeException(e);
diff --git a/tools/src/test/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java b/tools/src/test/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java
new file mode 100644
index 0000000..ca6b65a
--- /dev/null
+++ b/tools/src/test/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.protocol;
+
+
+/**
+ * Mock class for testing, called from ECPolicyDisabler with reflection.
+ *
+ */
+public class SystemErasureCodingPolicies {
+ public enum ReplicationPolicy {
+ DEFAULT, OTHER;
+
+ public String getName() {
+ return name();
+ }
+ }
+
+ public static ReplicationPolicy getReplicationPolicy() {
+ return ReplicationPolicy.DEFAULT;
+ }
+
+}
diff --git a/tools/src/test/java/org/apache/oozie/tools/TestECPolicyDisabler.java b/tools/src/test/java/org/apache/oozie/tools/TestECPolicyDisabler.java
new file mode 100644
index 0000000..b085eb2
--- /dev/null
+++ b/tools/src/test/java/org/apache/oozie/tools/TestECPolicyDisabler.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.tools;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
+import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies.ReplicationPolicy;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcErrorCodeProto;
+import org.apache.oozie.tools.ECPolicyDisabler.Result;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+
+/**
+ * Test for the Erasure Coding disabler code.
+ */
+public class TestECPolicyDisabler {
+
+ static abstract class MockDistributedFileSystem extends DistributedFileSystem {
+ public abstract SystemErasureCodingPolicies.ReplicationPolicy getErasureCodingPolicy(Path path);
+ public abstract void setErasureCodingPolicy(Path path, String policy);
+ }
+
+ @Test
+ public void testNotSupported() {
+ FileSystem fs = mock(FileSystem.class);
+ ECPolicyDisabler.Result result = ECPolicyDisabler.check(fs, null);
+ Assert.assertEquals("result is expected", Result.NOT_SUPPORTED, result);
+ }
+
+ @Test
+ public void testOkNotChanged() {
+ MockDistributedFileSystem fs = mock(MockDistributedFileSystem.class);
+ when(fs.getErasureCodingPolicy(any())).thenReturn(ReplicationPolicy.DEFAULT);
+ ECPolicyDisabler.Result result = ECPolicyDisabler.check(fs, null);
+ assertEquals("result is expected", Result.ALREADY_SET, result);
+ verify(fs).getErasureCodingPolicy(any());
+ verifyNoMoreInteractions(fs);
+ }
+
+ @Test
+ public void testOkChanged() {
+ MockDistributedFileSystem fs = mock(MockDistributedFileSystem.class);
+ when(fs.getErasureCodingPolicy(any())).thenReturn(ReplicationPolicy.OTHER);
+ ECPolicyDisabler.Result result = ECPolicyDisabler.check(fs, null);
+ assertEquals("result is expected", Result.DONE, result);
+ verify(fs).getErasureCodingPolicy(any());
+ verify(fs).setErasureCodingPolicy(any(), eq("DEFAULT"));
+ verifyNoMoreInteractions(fs);
+ }
+
+ @Test
+ public void testServerNotSupports() {
+ MockDistributedFileSystem fs = mock(MockDistributedFileSystem.class);
+ when(fs.getErasureCodingPolicy(any())).thenReturn(ReplicationPolicy.OTHER);
+ Mockito.doThrow(createNoSuchMethodException()).when(fs).setErasureCodingPolicy(any(), any());
+ ECPolicyDisabler.Result result = ECPolicyDisabler.check(fs, null);
+ assertEquals("result is expected", Result.NO_SUCH_METHOD, result);
+ verify(fs).getErasureCodingPolicy(any());
+ verify(fs).setErasureCodingPolicy(any(), eq("DEFAULT"));
+ verifyNoMoreInteractions(fs);
+ }
+
+ @Test
+ public void testOtherRuntimeExceptionThrown() {
+ MockDistributedFileSystem fs = mock(MockDistributedFileSystem.class);
+ when(fs.getErasureCodingPolicy(any())).thenReturn(ReplicationPolicy.OTHER);
+ Mockito.doThrow(new RuntimeException("mock io exception")).when(fs).setErasureCodingPolicy(any(), any());
+ try {
+ ECPolicyDisabler.check(fs, null);
+ Assert.fail("exception expected");
+ } catch (RuntimeException e) {
+ assertNotNull("runtime exception got", e);
+ }
+ verify(fs).getErasureCodingPolicy(any());
+ verify(fs).setErasureCodingPolicy(any(), eq("DEFAULT"));
+ verifyNoMoreInteractions(fs);
+ }
+
+ private RuntimeException createNoSuchMethodException() {
+ return new RuntimeException(new RemoteException("test", "error", RpcErrorCodeProto.ERROR_NO_SUCH_METHOD));
+ }
+}