You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2023/09/15 11:09:08 UTC
[hbase] branch branch-2.5 updated: HBASE-28061 HBaseTestingUtility failed to start MiniHbaseCluster in case of Hadoop3.3.1 (#5401)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push:
new 62942a94746 HBASE-28061 HBaseTestingUtility failed to start MiniHbaseCluster in case of Hadoop3.3.1 (#5401)
62942a94746 is described below
commit 62942a94746066581f588817480a535e5c2315f7
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Fri Sep 15 18:56:19 2023 +0800
HBASE-28061 HBaseTestingUtility failed to start MiniHbaseCluster in case of Hadoop3.3.1 (#5401)
Co-authored-by: Butao Zhang <bu...@163.com>
Signed-off-by: Xin Sun <dd...@gmail.com>
(cherry picked from commit ff2c10c1c254eafb2440f3f3f57968dfafc62d2f)
---
.../io/asyncfs/FanOutOneBlockAsyncDFSOutput.java | 3 +-
.../FanOutOneBlockAsyncDFSOutputHelper.java | 5 +-
.../hadoop/hbase/util/LocatedBlockHelper.java | 57 ++++++++++++++++++++++
.../org/apache/hadoop/hbase/fs/HFileSystem.java | 4 +-
.../java/org/apache/hadoop/hbase/util/FSUtils.java | 3 +-
.../apache/hadoop/hbase/fs/TestBlockReorder.java | 6 ++-
.../hbase/fs/TestBlockReorderBlockLocation.java | 29 ++++++-----
.../hbase/fs/TestBlockReorderMultiBlocks.java | 21 ++++----
8 files changed, 99 insertions(+), 29 deletions(-)
diff --git a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutput.java b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutput.java
index 5febcc8daa1..55a2f6c86ae 100644
--- a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutput.java
+++ b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutput.java
@@ -22,6 +22,7 @@ import static org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHel
import static org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile;
import static org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.endFileLease;
import static org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.getStatus;
+import static org.apache.hadoop.hbase.util.LocatedBlockHelper.getLocatedBlockLocations;
import static org.apache.hadoop.hbase.util.NettyFutureUtils.consume;
import static org.apache.hadoop.hbase.util.NettyFutureUtils.safeWrite;
import static org.apache.hadoop.hbase.util.NettyFutureUtils.safeWriteAndFlush;
@@ -364,7 +365,7 @@ public class FanOutOneBlockAsyncDFSOutput implements AsyncFSOutput {
this.clientName = clientName;
this.src = src;
this.block = locatedBlock.getBlock();
- this.locations = locatedBlock.getLocations();
+ this.locations = getLocatedBlockLocations(locatedBlock);
this.encryptor = encryptor;
this.datanodeInfoMap = datanodeInfoMap;
this.summer = summer;
diff --git a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java
index 04b13701451..b7bfd715bd0 100644
--- a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java
+++ b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.io.asyncfs;
import static org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createEncryptor;
import static org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.trySaslNegotiate;
+import static org.apache.hadoop.hbase.util.LocatedBlockHelper.getLocatedBlockLocations;
import static org.apache.hadoop.hbase.util.NettyFutureUtils.addListener;
import static org.apache.hadoop.hbase.util.NettyFutureUtils.safeClose;
import static org.apache.hadoop.hbase.util.NettyFutureUtils.safeWriteAndFlush;
@@ -441,7 +442,7 @@ public final class FanOutOneBlockAsyncDFSOutputHelper {
BlockConstructionStage stage, DataChecksum summer, EventLoopGroup eventLoopGroup,
Class<? extends Channel> channelClass) {
StorageType[] storageTypes = locatedBlock.getStorageTypes();
- DatanodeInfo[] datanodeInfos = locatedBlock.getLocations();
+ DatanodeInfo[] datanodeInfos = getLocatedBlockLocations(locatedBlock);
boolean connectToDnViaHostname =
conf.getBoolean(DFS_CLIENT_USE_DN_HOSTNAME, DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT);
int timeoutMs = conf.getInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY, READ_TIMEOUT);
@@ -555,7 +556,7 @@ public final class FanOutOneBlockAsyncDFSOutputHelper {
futureList = connectToDataNodes(conf, client, clientName, locatedBlock, 0L, 0L,
PIPELINE_SETUP_CREATE, summer, eventLoopGroup, channelClass);
for (int i = 0, n = futureList.size(); i < n; i++) {
- DatanodeInfo datanodeInfo = locatedBlock.getLocations()[i];
+ DatanodeInfo datanodeInfo = getLocatedBlockLocations(locatedBlock)[i];
try {
datanodes.put(futureList.get(i).syncUninterruptibly().getNow(), datanodeInfo);
} catch (Exception e) {
diff --git a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/util/LocatedBlockHelper.java b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/util/LocatedBlockHelper.java
new file mode 100644
index 00000000000..932bce2b161
--- /dev/null
+++ b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/util/LocatedBlockHelper.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * hadoop 3.3.1 changed the return value of this method from {@code DatanodeInfo[]} to
+ * {@code DatanodeInfoWithStorage[]}, which causes the JVM can not locate the method if we are
+ * compiled with hadoop 3.2 and then link with hadoop 3.3+, so here we need to use reflection to
+ * make it work for both hadoop versions, otherwise we need to publish more artifacts for different
+ * hadoop versions...
+ */
+@InterfaceAudience.Private
+public final class LocatedBlockHelper {
+
+ private static final Method GET_LOCATED_BLOCK_LOCATIONS_METHOD;
+
+ static {
+ try {
+ GET_LOCATED_BLOCK_LOCATIONS_METHOD = LocatedBlock.class.getMethod("getLocations");
+ } catch (Exception e) {
+ throw new Error("Can not initialize access to HDFS LocatedBlock.getLocations method", e);
+ }
+ }
+
+ private LocatedBlockHelper() {
+ }
+
+ public static DatanodeInfo[] getLocatedBlockLocations(LocatedBlock block) {
+ try {
+ // DatanodeInfoWithStorage[] can be casted to DatanodeInfo[] directly
+ return (DatanodeInfo[]) GET_LOCATED_BLOCK_LOCATIONS_METHOD.invoke(block);
+ } catch (IllegalAccessException | InvocationTargetException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
index ac90d2c686c..f893e6d73c4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.fs;
+import static org.apache.hadoop.hbase.util.LocatedBlockHelper.getLocatedBlockLocations;
+
import edu.umd.cs.findbugs.annotations.Nullable;
import java.io.Closeable;
import java.io.IOException;
@@ -425,7 +427,7 @@ public class HFileSystem extends FilterFileSystem {
// Just check for all blocks
for (LocatedBlock lb : lbs.getLocatedBlocks()) {
- DatanodeInfo[] dnis = lb.getLocations();
+ DatanodeInfo[] dnis = getLocatedBlockLocations(lb);
if (dnis != null && dnis.length > 1) {
boolean found = false;
for (int i = 0; i < dnis.length - 1 && !found; i++) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 728a46b51a4..4b553de4174 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.util;
+import static org.apache.hadoop.hbase.util.LocatedBlockHelper.getLocatedBlockLocations;
import static org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction.SAFEMODE_GET;
import edu.umd.cs.findbugs.annotations.CheckForNull;
@@ -682,7 +683,7 @@ public final class FSUtils {
}
private static String[] getHostsForLocations(LocatedBlock block) {
- DatanodeInfo[] locations = block.getLocations();
+ DatanodeInfo[] locations = getLocatedBlockLocations(block);
String[] hosts = new String[locations.length];
for (int i = 0; i < hosts.length; i++) {
hosts[i] = locations[i].getHostName();
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java
index ba772769c48..e1331ac3cba 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.fs;
+import static org.apache.hadoop.hbase.util.LocatedBlockHelper.getLocatedBlockLocations;
+
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.BindException;
@@ -160,8 +162,8 @@ public class TestBlockReorder {
@Override
public void reorderBlocks(Configuration c, LocatedBlocks lbs, String src) {
for (LocatedBlock lb : lbs.getLocatedBlocks()) {
- if (lb.getLocations().length > 1) {
- DatanodeInfo[] infos = lb.getLocations();
+ if (getLocatedBlockLocations(lb).length > 1) {
+ DatanodeInfo[] infos = getLocatedBlockLocations(lb);
if (infos[0].getHostName().equals(lookup)) {
LOG.info("HFileSystem bad host, inverting");
DatanodeInfo tmp = infos[0];
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorderBlockLocation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorderBlockLocation.java
index ea9a80a3409..27f9e37e4d8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorderBlockLocation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorderBlockLocation.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.fs;
+import static org.apache.hadoop.hbase.util.LocatedBlockHelper.getLocatedBlockLocations;
+
import java.lang.reflect.Field;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -117,21 +119,22 @@ public class TestBlockReorderBlockLocation {
for (int i = 0; i < 10; i++) {
// The interceptor is not set in this test, so we get the raw list at this point
- LocatedBlocks l;
+ LocatedBlocks lbs;
final long max = EnvironmentEdgeManager.currentTime() + 10000;
do {
- l = getNamenode(dfs.getClient()).getBlockLocations(fileName, 0, 1);
- Assert.assertNotNull(l.getLocatedBlocks());
- Assert.assertEquals(1, l.getLocatedBlocks().size());
- Assert.assertTrue("Expecting " + repCount + " , got " + l.get(0).getLocations().length,
+ lbs = getNamenode(dfs.getClient()).getBlockLocations(fileName, 0, 1);
+ Assert.assertNotNull(lbs.getLocatedBlocks());
+ Assert.assertEquals(1, lbs.getLocatedBlocks().size());
+ Assert.assertTrue(
+ "Expecting " + repCount + " , got " + getLocatedBlockLocations(lbs.get(0)).length,
EnvironmentEdgeManager.currentTime() < max);
- } while (l.get(0).getLocations().length != repCount);
+ } while (getLocatedBlockLocations(lbs.get(0)).length != repCount);
// Should be filtered, the name is different => The order won't change
- Object originalList[] = l.getLocatedBlocks().toArray();
+ Object[] originalList = lbs.getLocatedBlocks().toArray();
HFileSystem.ReorderWALBlocks lrb = new HFileSystem.ReorderWALBlocks();
- lrb.reorderBlocks(conf, l, fileName);
- Assert.assertArrayEquals(originalList, l.getLocatedBlocks().toArray());
+ lrb.reorderBlocks(conf, lbs, fileName);
+ Assert.assertArrayEquals(originalList, lbs.getLocatedBlocks().toArray());
// Should be reordered, as we pretend to be a file name with a compliant stuff
Assert.assertNotNull(conf.get(HConstants.HBASE_DIR));
@@ -144,12 +147,12 @@ public class TestBlockReorderBlockLocation {
AbstractFSWALProvider.getServerNameFromWALDirectoryName(dfs.getConf(), pseudoLogFile));
// And check we're doing the right reorder.
- lrb.reorderBlocks(conf, l, pseudoLogFile);
- Assert.assertEquals(host1, l.get(0).getLocations()[2].getHostName());
+ lrb.reorderBlocks(conf, lbs, pseudoLogFile);
+ Assert.assertEquals(host1, getLocatedBlockLocations(lbs.get(0))[2].getHostName());
// Check again, it should remain the same.
- lrb.reorderBlocks(conf, l, pseudoLogFile);
- Assert.assertEquals(host1, l.get(0).getLocations()[2].getHostName());
+ lrb.reorderBlocks(conf, lbs, pseudoLogFile);
+ Assert.assertEquals(host1, getLocatedBlockLocations(lbs.get(0))[2].getHostName());
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorderMultiBlocks.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorderMultiBlocks.java
index 73872b7084b..a551911e3dd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorderMultiBlocks.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorderMultiBlocks.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.fs;
+import static org.apache.hadoop.hbase.util.LocatedBlockHelper.getLocatedBlockLocations;
+
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.Field;
@@ -247,25 +249,26 @@ public class TestBlockReorderMultiBlocks {
throws Exception {
// Multiple times as the order is random
for (int i = 0; i < 10; i++) {
- LocatedBlocks l;
+ LocatedBlocks lbs;
// The NN gets the block list asynchronously, so we may need multiple tries to get the list
final long max = EnvironmentEdgeManager.currentTime() + 10000;
boolean done;
do {
Assert.assertTrue("Can't get enouth replica", EnvironmentEdgeManager.currentTime() < max);
- l = getNamenode(dfs.getClient()).getBlockLocations(src, 0, 1);
- Assert.assertNotNull("Can't get block locations for " + src, l);
- Assert.assertNotNull(l.getLocatedBlocks());
- Assert.assertTrue(l.getLocatedBlocks().size() > 0);
+ lbs = getNamenode(dfs.getClient()).getBlockLocations(src, 0, 1);
+ Assert.assertNotNull("Can't get block locations for " + src, lbs);
+ Assert.assertNotNull(lbs.getLocatedBlocks());
+ Assert.assertTrue(lbs.getLocatedBlocks().size() > 0);
done = true;
- for (int y = 0; y < l.getLocatedBlocks().size() && done; y++) {
- done = (l.get(y).getLocations().length == repCount);
+ for (int y = 0; y < lbs.getLocatedBlocks().size() && done; y++) {
+ done = getLocatedBlockLocations(lbs.get(y)).length == repCount;
}
} while (!done);
- for (int y = 0; y < l.getLocatedBlocks().size() && done; y++) {
- Assert.assertEquals(localhost, l.get(y).getLocations()[repCount - 1].getHostName());
+ for (int y = 0; y < lbs.getLocatedBlocks().size() && done; y++) {
+ Assert.assertEquals(localhost,
+ getLocatedBlockLocations(lbs.get(y))[repCount - 1].getHostName());
}
}
}