You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2018/07/04 02:30:00 UTC
hbase git commit: HBASE-20839 Fallback to FSHLog if we can not
instantiated AsyncFSWAL when user does not specify AsyncFSWAL explicitly
Repository: hbase
Updated Branches:
refs/heads/master 0f2378418 -> 4366720bd
HBASE-20839 Fallback to FSHLog if we can not instantiated AsyncFSWAL when user does not specify AsyncFSWAL explicitly
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4366720b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4366720b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4366720b
Branch: refs/heads/master
Commit: 4366720bd186276322ce8b7167a790608c801d6a
Parents: 0f23784
Author: zhangduo <zh...@apache.org>
Authored: Tue Jul 3 16:50:22 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Wed Jul 4 10:29:24 2018 +0800
----------------------------------------------------------------------
.../hadoop/hbase/regionserver/wal/AsyncFSWAL.java | 2 +-
.../hadoop/hbase/wal/AsyncFSWALProvider.java | 17 +++++++++++++++++
.../org/apache/hadoop/hbase/wal/WALFactory.java | 14 +++++++++++++-
3 files changed, 31 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/4366720b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
index 7f3e30b..b0c2549 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/hbase/blob/4366720b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AsyncFSWALProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AsyncFSWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AsyncFSWALProvider.java
index 56edb75..062b368 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AsyncFSWALProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AsyncFSWALProvider.java
@@ -21,6 +21,9 @@ import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput;
+import org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper;
+import org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper;
import org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL;
import org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter;
import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
@@ -115,4 +118,18 @@ public class AsyncFSWALProvider extends AbstractFSWALProvider<AsyncFSWAL> {
throw new IOException("cannot get log writer", e);
}
}
+
+ /**
+ * Test whether we can load the helper classes for async dfs output.
+ */
+ public static boolean load() {
+ try {
+ Class.forName(FanOutOneBlockAsyncDFSOutput.class.getName());
+ Class.forName(FanOutOneBlockAsyncDFSOutputHelper.class.getName());
+ Class.forName(FanOutOneBlockAsyncDFSOutputSaslHelper.class.getName());
+ return true;
+ } catch (Throwable e) {
+ return false;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/4366720b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
index 4f6a898..8eb1690 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
@@ -123,7 +123,19 @@ public class WALFactory {
@VisibleForTesting
public Class<? extends WALProvider> getProviderClass(String key, String defaultValue) {
try {
- return Providers.valueOf(conf.get(key, defaultValue)).clazz;
+ Providers provider = Providers.valueOf(conf.get(key, defaultValue));
+ if (provider != Providers.defaultProvider) {
+ // User gives a wal provider explicitly, just use that one
+ return provider.clazz;
+ }
+ // AsyncFSWAL has better performance in most cases, and also uses less resources, we will try
+ // to use it if possible. But it deeply hacks into the internal of DFSClient so will be easily
+ // broken when upgrading hadoop. If it is broken, then we fall back to use FSHLog.
+ if (AsyncFSWALProvider.load()) {
+ return AsyncFSWALProvider.class;
+ } else {
+ return FSHLogProvider.class;
+ }
} catch (IllegalArgumentException exception) {
// Fall back to them specifying a class name
// Note that the passed default class shouldn't actually be used, since the above only fails