You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ha...@apache.org on 2022/03/16 12:30:24 UTC
[hbase] branch branch-2 updated (d3629bb -> fdc7c49)
This is an automated email from the ASF dual-hosted git repository.
haxiaolin pushed a change to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git.
from d3629bb HBASE-26837 Set SFT config when creating TableDescriptor in TestCloneSnapshotProcedure (#4226)
new db5e954 HBASE-26680 Close and do not write trailer for the broken WAL writer (#4174)
new 50757ef Hbase 26275 update error message when executing deleteall with ROWPREFIXFILTER in meta table (#4205)
new fdc7c49 HBASE-26720 ExportSnapshot should validate the source snapshot before copying files (#4216)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hadoop/hbase/snapshot/ExportSnapshot.java | 12 ++++
.../wal/AbstractProtobufLogWriter.java | 68 +++++++++++++---------
.../regionserver/wal/AsyncProtobufLogWriter.java | 12 +++-
.../hbase/regionserver/wal/ProtobufLogWriter.java | 11 ++++
.../apache/hadoop/hbase/wal/FSHLogProvider.java | 7 ---
hbase-shell/src/main/ruby/hbase/table.rb | 6 +-
hbase-shell/src/test/ruby/hbase/table_test.rb | 7 +++
7 files changed, 85 insertions(+), 38 deletions(-)
[hbase] 03/03: HBASE-26720 ExportSnapshot should validate the source snapshot before copying files (#4216)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haxiaolin pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit fdc7c49c012e720ca4e36d2cc3841e185baa85d9
Author: xicm <36...@users.noreply.github.com>
AuthorDate: Wed Mar 16 20:03:28 2022 +0800
HBASE-26720 ExportSnapshot should validate the source snapshot before copying files (#4216)
Signed-off-by: Andrew Purtell <ap...@apache.org>
Signed-off-by: Xiaolin Ha <ha...@apache.org>
---
.../org/apache/hadoop/hbase/snapshot/ExportSnapshot.java | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
index 430228b..1eda801 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -142,6 +142,8 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
"Do not verify checksum, use name+length only.");
static final Option NO_TARGET_VERIFY = new Option(null, "no-target-verify", false,
"Do not verify the integrity of the exported snapshot.");
+ static final Option NO_SOURCE_VERIFY = new Option(null, "no-source-verify", false,
+ "Do not verify the source of the snapshot.");
static final Option OVERWRITE = new Option(null, "overwrite", false,
"Rewrite the snapshot manifest if already exists.");
static final Option CHUSER = new Option(null, "chuser", true,
@@ -912,6 +914,7 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
}
private boolean verifyTarget = true;
+ private boolean verifySource = true;
private boolean verifyChecksum = true;
private String snapshotName = null;
private String targetName = null;
@@ -943,6 +946,7 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
// And verifyChecksum and verifyTarget with values read from old args in processOldArgs(...).
verifyChecksum = !cmd.hasOption(Options.NO_CHECKSUM_VERIFY.getLongOpt());
verifyTarget = !cmd.hasOption(Options.NO_TARGET_VERIFY.getLongOpt());
+ verifySource = !cmd.hasOption(Options.NO_SOURCE_VERIFY.getLongOpt());
}
/**
@@ -993,6 +997,13 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
LOG.debug("outputFs={}, outputRoot={}, skipTmp={}, initialOutputSnapshotDir={}",
outputFs, outputRoot.toString(), skipTmp, initialOutputSnapshotDir);
+ // Verify snapshot source before copying files
+ if (verifySource) {
+ LOG.info("Verify snapshot source, inputFs={}, inputRoot={}, snapshotDir={}.",
+ inputFs.getUri(), inputRoot, snapshotDir);
+ verifySnapshot(srcConf, inputFs, inputRoot, snapshotDir);
+ }
+
// Find the necessary directory which need to change owner and group
Path needSetOwnerDir = SnapshotDescriptionUtils.getSnapshotRootDir(outputRoot);
if (outputFs.exists(needSetOwnerDir)) {
@@ -1143,6 +1154,7 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
addOption(Options.TARGET_NAME);
addOption(Options.NO_CHECKSUM_VERIFY);
addOption(Options.NO_TARGET_VERIFY);
+ addOption(Options.NO_SOURCE_VERIFY);
addOption(Options.OVERWRITE);
addOption(Options.CHUSER);
addOption(Options.CHGROUP);
[hbase] 01/03: HBASE-26680 Close and do not write trailer for the broken WAL writer (#4174)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haxiaolin pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit db5e9540688b02d50dfacc702bbea81bc9aa8d16
Author: Xiaolin Ha <ha...@apache.org>
AuthorDate: Wed Mar 16 19:58:45 2022 +0800
HBASE-26680 Close and do not write trailer for the broken WAL writer (#4174)
Signed-off-by: Duo Zhang <zh...@apache.org>
---
.../wal/AbstractProtobufLogWriter.java | 68 +++++++++++++---------
.../regionserver/wal/AsyncProtobufLogWriter.java | 12 +++-
.../hbase/regionserver/wal/ProtobufLogWriter.java | 11 ++++
.../apache/hadoop/hbase/wal/FSHLogProvider.java | 7 ---
4 files changed, 61 insertions(+), 37 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractProtobufLogWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractProtobufLogWriter.java
index 42335a5..a56a31a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractProtobufLogWriter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractProtobufLogWriter.java
@@ -166,38 +166,43 @@ public abstract class AbstractProtobufLogWriter {
public void init(FileSystem fs, Path path, Configuration conf, boolean overwritable,
long blocksize, StreamSlowMonitor monitor) throws IOException,
StreamLacksCapabilityException {
- this.conf = conf;
- boolean doCompress = initializeCompressionContext(conf, path);
- this.trailerWarnSize = conf.getInt(WAL_TRAILER_WARN_SIZE, DEFAULT_WAL_TRAILER_WARN_SIZE);
- int bufferSize = CommonFSUtils.getDefaultBufferSize(fs);
- short replication = (short) conf.getInt("hbase.regionserver.hlog.replication",
- CommonFSUtils.getDefaultReplication(fs, path));
-
- initOutput(fs, path, overwritable, bufferSize, replication, blocksize, monitor);
-
- boolean doTagCompress = doCompress &&
- conf.getBoolean(CompressionContext.ENABLE_WAL_TAGS_COMPRESSION, true);
- boolean doValueCompress = doCompress &&
- conf.getBoolean(CompressionContext.ENABLE_WAL_VALUE_COMPRESSION, false);
- WALHeader.Builder headerBuilder = WALHeader.newBuilder()
- .setHasCompression(doCompress)
- .setHasTagCompression(doTagCompress)
- .setHasValueCompression(doValueCompress);
- if (doValueCompress) {
- headerBuilder.setValueCompressionAlgorithm(
- CompressionContext.getValueCompressionAlgorithm(conf).ordinal());
- }
- length.set(writeMagicAndWALHeader(ProtobufLogReader.PB_WAL_MAGIC,
- buildWALHeader(conf, headerBuilder)));
+ try {
+ this.conf = conf;
+ boolean doCompress = initializeCompressionContext(conf, path);
+ this.trailerWarnSize = conf.getInt(WAL_TRAILER_WARN_SIZE, DEFAULT_WAL_TRAILER_WARN_SIZE);
+ int bufferSize = CommonFSUtils.getDefaultBufferSize(fs);
+ short replication = (short) conf.getInt("hbase.regionserver.hlog.replication",
+ CommonFSUtils.getDefaultReplication(fs, path));
+
+ initOutput(fs, path, overwritable, bufferSize, replication, blocksize, monitor);
+
+ boolean doTagCompress =
+ doCompress && conf.getBoolean(CompressionContext.ENABLE_WAL_TAGS_COMPRESSION, true);
+ boolean doValueCompress =
+ doCompress && conf.getBoolean(CompressionContext.ENABLE_WAL_VALUE_COMPRESSION, false);
+ WALHeader.Builder headerBuilder =
+ WALHeader.newBuilder().setHasCompression(doCompress).setHasTagCompression(doTagCompress)
+ .setHasValueCompression(doValueCompress);
+ if (doValueCompress) {
+ headerBuilder.setValueCompressionAlgorithm(
+ CompressionContext.getValueCompressionAlgorithm(conf).ordinal());
+ }
+ length.set(writeMagicAndWALHeader(ProtobufLogReader.PB_WAL_MAGIC,
+ buildWALHeader(conf, headerBuilder)));
- initAfterHeader(doCompress);
+ initAfterHeader(doCompress);
- // instantiate trailer to default value.
- trailer = WALTrailer.newBuilder().build();
+ // instantiate trailer to default value.
+ trailer = WALTrailer.newBuilder().build();
- if (LOG.isTraceEnabled()) {
- LOG.trace("Initialized protobuf WAL={}, compression={}, tagCompression={}" +
- ", valueCompression={}", path, doCompress, doTagCompress, doValueCompress);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Initialized protobuf WAL={}, compression={}, tagCompression={}"
+ + ", valueCompression={}", path, doCompress, doTagCompress, doValueCompress);
+ }
+ } catch (Exception e) {
+ LOG.warn("Init output failed, path={}", path, e);
+ closeOutput();
+ throw e;
}
}
@@ -266,6 +271,11 @@ public abstract class AbstractProtobufLogWriter {
throws IOException, StreamLacksCapabilityException;
/**
+ * simply close the output, do not need to write trailer like the Writer.close
+ */
+ protected abstract void closeOutput();
+
+ /**
* return the file length after written.
*/
protected abstract long writeMagicAndWALHeader(byte[] magic, WALHeader header) throws IOException;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncProtobufLogWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncProtobufLogWriter.java
index bc2998b..fbd3882 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncProtobufLogWriter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncProtobufLogWriter.java
@@ -47,7 +47,6 @@ import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
import org.apache.hbase.thirdparty.io.netty.channel.Channel;
import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
-
import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALHeader;
import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALTrailer;
@@ -197,6 +196,17 @@ public class AsyncProtobufLogWriter extends AbstractProtobufLogWriter
this.asyncOutputWrapper = new OutputStreamWrapper(output);
}
+ @Override
+ protected void closeOutput() {
+ if (this.output != null) {
+ try {
+ this.output.close();
+ } catch (IOException e) {
+ LOG.warn("Close output failed", e);
+ }
+ }
+ }
+
private long writeWALMetadata(Consumer<CompletableFuture<Long>> action) throws IOException {
CompletableFuture<Long> future = new CompletableFuture<>();
action.accept(future);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
index 7ed267b..2df19fd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
@@ -119,6 +119,17 @@ public class ProtobufLogWriter extends AbstractProtobufLogWriter
}
@Override
+ protected void closeOutput() {
+ if (this.output != null) {
+ try {
+ this.output.close();
+ } catch (IOException e) {
+ LOG.warn("Close output failed", e);
+ }
+ }
+ }
+
+ @Override
protected long writeMagicAndWALHeader(byte[] magic, WALHeader header) throws IOException {
output.write(magic);
header.writeDelimitedTo(output);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/FSHLogProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/FSHLogProvider.java
index 0dedda9..58f072d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/FSHLogProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/FSHLogProvider.java
@@ -90,13 +90,6 @@ public class FSHLogProvider extends AbstractFSWALProvider<FSHLog> {
} else {
LOG.debug("Error instantiating log writer.", e);
}
- if (writer != null) {
- try{
- writer.close();
- } catch(IOException ee){
- LOG.error("cannot close log writer", ee);
- }
- }
throw new IOException("cannot get log writer", e);
}
}
[hbase] 02/03: Hbase 26275 update error message when executing deleteall with ROWPREFIXFILTER in meta table (#4205)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haxiaolin pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 50757ef97d31a7ca500af9524378996344e41205
Author: xicm <36...@users.noreply.github.com>
AuthorDate: Wed Mar 16 20:02:38 2022 +0800
Hbase 26275 update error message when executing deleteall with ROWPREFIXFILTER in meta table (#4205)
Signed-off-by: Xiaolin Ha <ha...@apache.org>
---
hbase-shell/src/main/ruby/hbase/table.rb | 6 +++++-
hbase-shell/src/test/ruby/hbase/table_test.rb | 7 +++++++
2 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/hbase-shell/src/main/ruby/hbase/table.rb b/hbase-shell/src/main/ruby/hbase/table.rb
index 4404eae..c93e75f 100644
--- a/hbase-shell/src/main/ruby/hbase/table.rb
+++ b/hbase-shell/src/main/ruby/hbase/table.rb
@@ -244,7 +244,11 @@ EOF
# delete operation doesn't need read permission. Retaining the read check for
# meta table as a part of HBASE-5837.
if is_meta_table?
- raise ArgumentError, 'Row Not Found' if _get_internal(row).nil?
+ if row.is_a?(Hash) and row.key?('ROWPREFIXFILTER')
+ raise ArgumentError, 'deleteall with ROWPREFIXFILTER in hbase:meta is not allowed.'
+ else
+ raise ArgumentError, 'Row Not Found' if _get_internal(row).nil?
+ end
end
if row.is_a?(Hash)
_deleterows_internal(row, column, timestamp, args, all_version)
diff --git a/hbase-shell/src/test/ruby/hbase/table_test.rb b/hbase-shell/src/test/ruby/hbase/table_test.rb
index ee6f0f5..cb274f8 100644
--- a/hbase-shell/src/test/ruby/hbase/table_test.rb
+++ b/hbase-shell/src/test/ruby/hbase/table_test.rb
@@ -194,6 +194,13 @@ module Hbase
assert_nil(res2)
end
+ define_test "deleteall with row prefix in hbase:meta should not be allowed." do
+ assert_raise(ArgumentError) do
+ @meta_table = table('hbase:meta')
+ @meta_table.deleteall({ROWPREFIXFILTER => "test_meta"})
+ end
+ end
+
define_test "append should work with value" do
@test_table.append("123", 'x:cnt2', '123')
assert_equal("123123", @test_table._append_internal("123", 'x:cnt2', '123'))