You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bh...@apache.org on 2020/07/21 01:50:58 UTC
[hbase] branch branch-2.2 updated: HBASE-24740 Enable journal
logging for HBase snapshot operation (#2104)
This is an automated email from the ASF dual-hosted git repository.
bharathv pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new 4e5ec22 HBASE-24740 Enable journal logging for HBase snapshot operation (#2104)
4e5ec22 is described below
commit 4e5ec220b887739f0a43656620604b82aac084b2
Author: sguggilam <sa...@gmail.com>
AuthorDate: Mon Jul 20 18:11:41 2020 -0700
HBASE-24740 Enable journal logging for HBase snapshot operation (#2104)
Signed-off-by: Bharath Vissapragada <bh...@apache.org>
(cherry picked from commit 430602aed82c09cfac6781dbd3dc5eba0dfef4af)
---
.../hbase/master/snapshot/SnapshotManager.java | 18 +++++++++++----
.../hbase/master/snapshot/TakeSnapshotHandler.java | 7 +++++-
.../hadoop/hbase/monitoring/MonitoredTaskImpl.java | 2 +-
.../hadoop/hbase/snapshot/SnapshotManifest.java | 26 ++++++++++++++++++----
4 files changed, 43 insertions(+), 10 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index 709796d..dbcb7bd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -651,16 +651,26 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
TableName snapshotTable = TableName.valueOf(snapshot.getTable());
if (master.getTableStateManager().isTableState(snapshotTable,
TableState.State.ENABLED)) {
- LOG.debug("Table enabled, starting distributed snapshot.");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Table enabled, starting distributed snapshots for {}",
+ ClientSnapshotDescriptionUtils.toString(snapshot));
+ }
snapshotEnabledTable(snapshot);
- LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Started snapshot: {}", ClientSnapshotDescriptionUtils.toString(snapshot));
+ }
}
// For disabled table, snapshot is created by the master
else if (master.getTableStateManager().isTableState(snapshotTable,
TableState.State.DISABLED)) {
- LOG.debug("Table is disabled, running snapshot entirely on master.");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Table is disabled, running snapshot entirely on master for {}",
+ ClientSnapshotDescriptionUtils.toString(snapshot));
+ }
snapshotDisabledTable(snapshot);
- LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Started snapshot: {}", ClientSnapshotDescriptionUtils.toString(snapshot));
+ }
} else {
LOG.error("Can't snapshot table '" + snapshot.getTable()
+ "', isn't open or closed, we don't know what to do!");
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
index 0d65264..95bc5cb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -125,7 +125,6 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
this.snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
this.workingDirFs = this.workingDir.getFileSystem(this.conf);
this.monitor = new ForeignExceptionDispatcher(snapshot.getName());
- this.snapshotManifest = SnapshotManifest.create(conf, rootFs, workingDir, snapshot, monitor);
this.tableLock = master.getLockManager().createMasterLock(
snapshotTable, LockType.EXCLUSIVE,
@@ -136,6 +135,9 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
// update the running tasks
this.status = TaskMonitor.get().createStatus(
"Taking " + snapshot.getType() + " snapshot on table: " + snapshotTable);
+ this.status.enableStatusJournal(true);
+ this.snapshotManifest =
+ SnapshotManifest.create(conf, rootFs, workingDir, snapshot, monitor, status);
}
private TableDescriptor loadTableDescriptor()
@@ -249,6 +251,9 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
} catch (IOException e) {
LOG.error("Couldn't delete snapshot working directory:" + workingDir);
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Table snapshot journal : \n" + status.prettyPrintJournal());
+ }
tableLockToRelease.release();
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java
index 03f570d..c95363b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java
@@ -259,7 +259,7 @@ class MonitoredTaskImpl implements MonitoredTask {
if (journal == null) {
journal = new ArrayList<StatusJournalEntry>();
}
- if (includeCurrentStatus) {
+ if (includeCurrentStatus && status != null) {
journal.add(new StatusJournalEntryImpl(status, statusTime));
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
index eaee728..e2a2c5b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
import org.apache.hadoop.hbase.mob.MobUtils;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.HStore;
@@ -84,6 +85,7 @@ public final class SnapshotManifest {
private final FileSystem rootFs;
private final FileSystem workingDirFs;
private int manifestSizeLimit;
+ private final MonitoredTask statusTask;
/**
*
@@ -97,12 +99,13 @@ public final class SnapshotManifest {
*/
private SnapshotManifest(final Configuration conf, final FileSystem rootFs,
final Path workingDir, final SnapshotDescription desc,
- final ForeignExceptionSnare monitor) throws IOException {
+ final ForeignExceptionSnare monitor, final MonitoredTask statusTask) throws IOException {
this.monitor = monitor;
this.desc = desc;
this.workingDir = workingDir;
this.conf = conf;
this.rootFs = rootFs;
+ this.statusTask = statusTask;
this.workingDirFs = this.workingDir.getFileSystem(this.conf);
this.manifestSizeLimit = conf.getInt(SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 64 * 1024 * 1024);
}
@@ -123,7 +126,14 @@ public final class SnapshotManifest {
public static SnapshotManifest create(final Configuration conf, final FileSystem fs,
final Path workingDir, final SnapshotDescription desc,
final ForeignExceptionSnare monitor) throws IOException {
- return new SnapshotManifest(conf, fs, workingDir, desc, monitor);
+ return create(conf, fs, workingDir, desc, monitor, null);
+
+ }
+
+ public static SnapshotManifest create(final Configuration conf, final FileSystem fs,
+ final Path workingDir, final SnapshotDescription desc, final ForeignExceptionSnare monitor,
+ final MonitoredTask statusTask) throws IOException {
+ return new SnapshotManifest(conf, fs, workingDir, desc, monitor, statusTask);
}
@@ -138,7 +148,7 @@ public final class SnapshotManifest {
*/
public static SnapshotManifest open(final Configuration conf, final FileSystem fs,
final Path workingDir, final SnapshotDescription desc) throws IOException {
- SnapshotManifest manifest = new SnapshotManifest(conf, fs, workingDir, desc, null);
+ SnapshotManifest manifest = new SnapshotManifest(conf, fs, workingDir, desc, null, null);
manifest.load();
return manifest;
}
@@ -455,6 +465,12 @@ public final class SnapshotManifest {
return this.regionManifests;
}
+ private void setStatusMsg(String msg) {
+ if (this.statusTask != null) {
+ statusTask.setStatus(msg);
+ }
+ }
+
/**
* Get all the Region Manifest from the snapshot.
* This is an helper to get a map with the region encoded name
@@ -478,7 +494,7 @@ public final class SnapshotManifest {
new FSTableDescriptors(conf, workingDirFs, rootDir)
.createTableDescriptorForTableDirectory(workingDir, htd, false);
} else {
- LOG.debug("Convert to Single Snapshot Manifest");
+ LOG.debug("Convert to Single Snapshot Manifest for {}", this.desc.getName());
convertToV2SingleManifest();
}
}
@@ -491,6 +507,7 @@ public final class SnapshotManifest {
// Try to load v1 and v2 regions
List<SnapshotRegionManifest> v1Regions, v2Regions;
ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
+ setStatusMsg("Loading Region manifests for " + this.desc.getName());
try {
v1Regions = SnapshotManifestV1.loadRegionManifests(conf, tpool, workingDirFs,
workingDir, desc);
@@ -514,6 +531,7 @@ public final class SnapshotManifest {
// Once the data-manifest is written, the snapshot can be considered complete.
// Currently snapshots are written in a "temporary" directory and later
// moved to the "complated" snapshot directory.
+ setStatusMsg("Writing data manifest for " + this.desc.getName());
SnapshotDataManifest dataManifest = dataManifestBuilder.build();
writeDataManifest(dataManifest);
this.regionManifests = dataManifest.getRegionManifestsList();