You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2024/09/06 09:02:56 UTC
(hbase) branch branch-3 updated (52eae6934d4 -> 8ffd93d9096)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a change to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
from 52eae6934d4 HBASE-28778 NPE may occur when opening master-status or table.jsp or procedure.jsp while Master is initializing (#6152)
new a387e560a26 HBASE-28580 Revert the deprecation for methods in WALObserver (#6205)
new 8ffd93d9096 HBASE-28775 Change the output of DatanodeInfo in the log to the hostname of the datanode (#6148)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../FanOutOneBlockAsyncDFSOutputHelper.java | 19 ++++++++++++--
.../hadoop/hbase/coprocessor/WALObserver.java | 30 +++++++++++++---------
.../hbase/regionserver/wal/AbstractFSWAL.java | 5 +++-
3 files changed, 39 insertions(+), 15 deletions(-)
(hbase) 02/02: HBASE-28775 Change the output of DatanodeInfo in the log to the hostname of the datanode (#6148)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 8ffd93d9096485826274e4b5f7524c09ecf46098
Author: WangXin <14...@qq.com>
AuthorDate: Fri Sep 6 16:22:27 2024 +0800
HBASE-28775 Change the output of DatanodeInfo in the log to the hostname of the datanode (#6148)
Co-authored-by: wangxin <wa...@gmail.com>
Signed-off-by: Duo Zhang <zh...@apache.org>
Signed-off-by: Nihal Jain <ni...@apache.org>
Reviewed-by: Vineet Kumar Maheshwari <vi...@gmail.com>
Reviewed-by: guluo <lu...@qq.com>
(cherry picked from commit 241bbaf0718d6215775cd4996e9ca08ba12eb086)
---
.../asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java | 19 +++++++++++++++++--
.../hadoop/hbase/regionserver/wal/AbstractFSWAL.java | 5 ++++-
2 files changed, 21 insertions(+), 3 deletions(-)
diff --git a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java
index 7a4f624e6e0..879dfda77ec 100644
--- a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java
+++ b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java
@@ -35,6 +35,7 @@ import java.io.InterruptedIOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.IdentityHashMap;
@@ -42,6 +43,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
import org.apache.hadoop.crypto.Encryptor;
@@ -473,8 +475,10 @@ public final class FanOutOneBlockAsyncDFSOutputHelper {
Set<DatanodeInfo> toExcludeNodes =
new HashSet<>(excludeDatanodeManager.getExcludeDNs().keySet());
for (int retry = 0;; retry++) {
- LOG.debug("When create output stream for {}, exclude list is {}, retry={}", src,
- toExcludeNodes, retry);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("When create output stream for {}, exclude list is {}, retry={}", src,
+ getDataNodeInfo(toExcludeNodes), retry);
+ }
HdfsFileStatus stat;
try {
stat = FILE_CREATOR.create(namenode, src,
@@ -620,4 +624,15 @@ public final class FanOutOneBlockAsyncDFSOutputHelper {
} catch (InterruptedException e) {
}
}
+
+ public static String getDataNodeInfo(Collection<DatanodeInfo> datanodeInfos) {
+ if (datanodeInfos.isEmpty()) {
+ return "[]";
+ }
+ return datanodeInfos.stream()
+ .map(datanodeInfo -> new StringBuilder().append("(").append(datanodeInfo.getHostName())
+ .append("/").append(datanodeInfo.getInfoAddr()).append(":")
+ .append(datanodeInfo.getInfoPort()).append(")").toString())
+ .collect(Collectors.joining(",", "[", "]"));
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
index bba9bd534e9..77c296b096c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
@@ -71,6 +71,7 @@ import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Supplier;
+import java.util.stream.Collectors;
import org.apache.commons.lang3.mutable.MutableLong;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -85,6 +86,7 @@ import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.client.ConnectionUtils;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.exceptions.TimeoutIOException;
+import org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper;
import org.apache.hadoop.hbase.io.util.MemorySizeUtil;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.ipc.ServerCall;
@@ -1105,7 +1107,8 @@ public abstract class AbstractFSWAL<W extends WriterBase> implements WAL {
tellListenersAboutPostLogRoll(oldPath, newPath);
if (LOG.isDebugEnabled()) {
LOG.debug("Create new " + implClassName + " writer with pipeline: "
- + Arrays.toString(getPipeline()));
+ + FanOutOneBlockAsyncDFSOutputHelper
+ .getDataNodeInfo(Arrays.stream(getPipeline()).collect(Collectors.toList())));
}
// We got a new writer, so reset the slow sync count
lastTimeCheckSlowSync = EnvironmentEdgeManager.currentTime();
(hbase) 01/02: HBASE-28580 Revert the deprecation for methods in WALObserver (#6205)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit a387e560a26576e5a0b97cbe867d2fc28879667c
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Fri Sep 6 16:55:17 2024 +0800
HBASE-28580 Revert the deprecation for methods in WALObserver (#6205)
Signed-off-by: Nick Dimiduk <nd...@apache.org>
(cherry picked from commit 35c7fc0bab4371fd5e726838ee093b933503cb39)
---
.../hadoop/hbase/coprocessor/WALObserver.java | 30 +++++++++++++---------
1 file changed, 18 insertions(+), 12 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java
index bc57dbc735a..768cc3955e5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java
@@ -56,25 +56,31 @@ import org.apache.yetus.audience.InterfaceStability;
@InterfaceStability.Evolving
public interface WALObserver {
/**
- * Called before a {@link WALEdit} is writen to WAL. Do not amend the WALKey. It is
- * InterfaceAudience.Private. Changing the WALKey will cause damage.
- * @deprecated Since hbase-2.0.0. To be replaced with an alternative that does not expose
- * InterfaceAudience classes such as WALKey and WALEdit. Will be removed in
- * hbase-3.0.0.
+ * Called before a {@link WALEdit} is writen to WAL.
+ * <p>
+ * The method is marked as deprecated in 2.0.0, but later we abstracted the WALKey interface for
+ * coprocessors, now it is OK to expose this to coprocessor users, so we revert the deprecation.
+ * But you still need to be careful while changing {@link WALEdit}, as when reaching here, if you
+ * add some cells to WALEdit, it will only be written to WAL but no in memstore, but when
+ * replaying you will get these cells and there are CP hooks to intercept these cells.
+ * <p>
+ * See HBASE-28580.
*/
- @Deprecated
default void preWALWrite(ObserverContext<? extends WALCoprocessorEnvironment> ctx,
RegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {
}
/**
- * Called after a {@link WALEdit} is writen to WAL. Do not amend the WALKey. It is
- * InterfaceAudience.Private. Changing the WALKey will cause damage.
- * @deprecated Since hbase-2.0.0. To be replaced with an alternative that does not expose
- * InterfaceAudience classes such as WALKey and WALEdit. Will be removed in
- * hbase-3.0.0.
+ * Called after a {@link WALEdit} is writen to WAL.
+ * <p>
+ * The method is marked as deprecated in 2.0.0, but later we abstracted the WALKey interface for
+ * coprocessors, now it is OK to expose this to coprocessor users, so we revert the deprecation.
+ * But you still need to be careful while changing {@link WALEdit}, as when reaching here, if you
+ * add some cells to WALEdit, it will only be written to WAL but no in memstore, but when
+ * replaying you will get these cells and there are CP hooks to intercept these cells.
+ * <p>
+ * See HBASE-28580.
*/
- @Deprecated
default void postWALWrite(ObserverContext<? extends WALCoprocessorEnvironment> ctx,
RegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {
}