You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by md...@apache.org on 2017/12/14 18:19:53 UTC
hbase git commit: HBASE-19289 Add flag to disable stream capability
enforcement
Repository: hbase
Updated Branches:
refs/heads/master d5aefbd2c -> 2c9ef8a47
HBASE-19289 Add flag to disable stream capability enforcement
Signed-off-by: Josh Elser <el...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2c9ef8a4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2c9ef8a4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2c9ef8a4
Branch: refs/heads/master
Commit: 2c9ef8a471148ece655b881cc490b6b685d634f4
Parents: d5aefbd
Author: Mike Drob <md...@apache.org>
Authored: Tue Dec 5 14:25:37 2017 -0600
Committer: Mike Drob <md...@apache.org>
Committed: Thu Dec 14 12:19:22 2017 -0600
----------------------------------------------------------------------
.../java/org/apache/hadoop/hbase/util/CommonFSUtils.java | 3 +++
.../hbase/procedure2/store/wal/WALProcedureStore.java | 6 ++++--
hbase-procedure/src/test/resources/hbase-site.xml | 9 +++++++++
.../apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java | 5 +++--
.../hadoop/hbase/regionserver/wal/ProtobufLogWriter.java | 3 ++-
hbase-server/src/test/resources/hbase-site.xml | 9 +++++++++
6 files changed, 30 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2c9ef8a4/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
index 38ae476..2a83a5d 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
@@ -62,6 +62,9 @@ public abstract class CommonFSUtils {
/** Parameter name for HBase WAL directory */
public static final String HBASE_WAL_DIR = "hbase.wal.dir";
+ /** Parameter to disable stream capability enforcement checks */
+ public static final String UNSAFE_STREAM_CAPABILITY_ENFORCE = "hbase.unsafe.stream.capability.enforce";
+
/** Full access permissions (starting point for a umask) */
public static final String FULL_RWX_PERMISSIONS = "777";
http://git-wip-us.apache.org/repos/asf/hbase/blob/2c9ef8a4/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
index f49833c..84cda65 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
@@ -131,6 +131,7 @@ public class WALProcedureStore extends ProcedureStoreBase {
private final FileSystem fs;
private final Path walDir;
private final Path walArchiveDir;
+ private final boolean enforceStreamCapability;
private final AtomicReference<Throwable> syncException = new AtomicReference<>();
private final AtomicBoolean loading = new AtomicBoolean(true);
@@ -205,6 +206,7 @@ public class WALProcedureStore extends ProcedureStoreBase {
this.walDir = walDir;
this.walArchiveDir = walArchiveDir;
this.fs = walDir.getFileSystem(conf);
+ this.enforceStreamCapability = conf.getBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true);
// Create the log directory for the procedure store
if (!fs.exists(walDir)) {
@@ -1028,8 +1030,8 @@ public class WALProcedureStore extends ProcedureStoreBase {
// ensure that we can provide the level of data safety we're configured
// to provide.
final String durability = useHsync ? "hsync" : "hflush";
- if (!(CommonFSUtils.hasCapability(newStream, durability))) {
- throw new IllegalStateException("The procedure WAL relies on the ability to " + durability +
+ if (enforceStreamCapability && !(CommonFSUtils.hasCapability(newStream, durability))) {
+ throw new IllegalStateException("The procedure WAL relies on the ability to " + durability +
" for proper operation during component failures, but the underlying filesystem does " +
"not support doing so. Please check the config value of '" + USE_HSYNC_CONF_KEY +
"' to set the desired level of robustness and ensure the config value of '" +
http://git-wip-us.apache.org/repos/asf/hbase/blob/2c9ef8a4/hbase-procedure/src/test/resources/hbase-site.xml
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/resources/hbase-site.xml b/hbase-procedure/src/test/resources/hbase-site.xml
index d350127..114ee8a 100644
--- a/hbase-procedure/src/test/resources/hbase-site.xml
+++ b/hbase-procedure/src/test/resources/hbase-site.xml
@@ -32,4 +32,13 @@
procedure to have an owner
</description>
</property>
+ <property>
+ <name>hbase.unsafe.stream.capability.enforce</name>
+ <value>false</value>
+ <description>
+ Controls whether HBase will check for stream capabilities (hflush/hsync).
+ Disable this if you intend to run on LocalFileSystem.
+ WARNING: Doing so may expose you to additional risk of data loss!
+ </description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2c9ef8a4/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java
index 6a7e4fa..583759b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java
@@ -73,8 +73,9 @@ public final class AsyncFSOutputHelper {
// After we create the stream but before we attempt to use it at all
// ensure that we can provide the level of data safety we're configured
// to provide.
- if (!(CommonFSUtils.hasCapability(fsOut, "hflush") &&
- CommonFSUtils.hasCapability(fsOut, "hsync"))) {
+ if (fs.getConf().getBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true) &&
+ !(CommonFSUtils.hasCapability(fsOut, "hflush") &&
+ CommonFSUtils.hasCapability(fsOut, "hsync"))) {
throw new CommonFSUtils.StreamLacksCapabilityException("hflush and hsync");
}
final ExecutorService flushExecutor =
http://git-wip-us.apache.org/repos/asf/hbase/blob/2c9ef8a4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
index 64acfba..7a135c9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
@@ -93,7 +93,8 @@ public class ProtobufLogWriter extends AbstractProtobufLogWriter
this.output = fs.createNonRecursive(path, overwritable, bufferSize, replication, blockSize,
null);
// TODO Be sure to add a check for hsync if this branch includes HBASE-19024
- if (!(CommonFSUtils.hasCapability(output, "hflush"))) {
+ if (fs.getConf().getBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true) &&
+ !(CommonFSUtils.hasCapability(output, "hflush"))) {
throw new StreamLacksCapabilityException("hflush");
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2c9ef8a4/hbase-server/src/test/resources/hbase-site.xml
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/resources/hbase-site.xml b/hbase-server/src/test/resources/hbase-site.xml
index 64a1964..dbdf776 100644
--- a/hbase-server/src/test/resources/hbase-site.xml
+++ b/hbase-server/src/test/resources/hbase-site.xml
@@ -158,4 +158,13 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
+ <property>
+ <name>hbase.unsafe.stream.capability.enforce</name>
+ <value>false</value>
+ <description>
+ Controls whether HBase will check for stream capabilities (hflush/hsync).
+ Disable this if you intend to run on LocalFileSystem.
+ WARNING: Doing so may expose you to additional risk of data loss!
+ </description>
+ </property>
</configuration>