You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bh...@apache.org on 2020/07/21 01:56:55 UTC
[hbase] branch branch-2 updated: HBASE-24740 Enable journal logging
for HBase snapshot operation (#2104)
This is an automated email from the ASF dual-hosted git repository.
bharathv pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 19d0f36 HBASE-24740 Enable journal logging for HBase snapshot operation (#2104)
19d0f36 is described below
commit 19d0f364ed9b5fbc6fce5b86c3c41b9139952341
Author: sguggilam <sa...@gmail.com>
AuthorDate: Mon Jul 20 18:11:41 2020 -0700
HBASE-24740 Enable journal logging for HBase snapshot operation (#2104)
Signed-off-by: Bharath Vissapragada <bh...@apache.org>
(cherry picked from commit 430602aed82c09cfac6781dbd3dc5eba0dfef4af)
---
.../hbase/master/snapshot/SnapshotManager.java | 18 +++++++++++----
.../hbase/master/snapshot/TakeSnapshotHandler.java | 10 +++++----
.../hadoop/hbase/monitoring/MonitoredTaskImpl.java | 2 +-
.../hadoop/hbase/snapshot/SnapshotManifest.java | 26 ++++++++++++++++++----
4 files changed, 43 insertions(+), 13 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index 1e77182..f713718 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -654,16 +654,26 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
TableName snapshotTable = TableName.valueOf(snapshot.getTable());
if (master.getTableStateManager().isTableState(snapshotTable,
TableState.State.ENABLED)) {
- LOG.debug("Table enabled, starting distributed snapshot.");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Table enabled, starting distributed snapshots for {}",
+ ClientSnapshotDescriptionUtils.toString(snapshot));
+ }
snapshotEnabledTable(snapshot);
- LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Started snapshot: {}", ClientSnapshotDescriptionUtils.toString(snapshot));
+ }
}
// For disabled table, snapshot is created by the master
else if (master.getTableStateManager().isTableState(snapshotTable,
TableState.State.DISABLED)) {
- LOG.debug("Table is disabled, running snapshot entirely on master.");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Table is disabled, running snapshot entirely on master for {}",
+ ClientSnapshotDescriptionUtils.toString(snapshot));
+ }
snapshotDisabledTable(snapshot);
- LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Started snapshot: {}", ClientSnapshotDescriptionUtils.toString(snapshot));
+ }
} else {
LOG.error("Can't snapshot table '" + snapshot.getTable()
+ "', isn't open or closed, we don't know what to do!");
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
index e491467..7c9496b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -19,14 +19,12 @@ package org.apache.hadoop.hbase.master.snapshot;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.net.URI;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CancellationException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.ServerName;
@@ -47,7 +45,6 @@ import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.procedure2.LockType;
import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
import org.apache.hadoop.hbase.util.CommonFSUtils;
@@ -126,7 +123,6 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
this.snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
this.workingDirFs = this.workingDir.getFileSystem(this.conf);
this.monitor = new ForeignExceptionDispatcher(snapshot.getName());
- this.snapshotManifest = SnapshotManifest.create(conf, rootFs, workingDir, snapshot, monitor);
this.tableLock = master.getLockManager().createMasterLock(
snapshotTable, LockType.EXCLUSIVE,
@@ -137,6 +133,9 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
// update the running tasks
this.status = TaskMonitor.get().createStatus(
"Taking " + snapshot.getType() + " snapshot on table: " + snapshotTable);
+ this.status.enableStatusJournal(true);
+ this.snapshotManifest =
+ SnapshotManifest.create(conf, rootFs, workingDir, snapshot, monitor, status);
}
private TableDescriptor loadTableDescriptor()
@@ -256,6 +255,9 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
} catch (IOException e) {
LOG.error("Couldn't delete snapshot working directory:" + workingDir);
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Table snapshot journal : \n" + status.prettyPrintJournal());
+ }
tableLockToRelease.release();
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java
index 03f570d..c95363b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java
@@ -259,7 +259,7 @@ class MonitoredTaskImpl implements MonitoredTask {
if (journal == null) {
journal = new ArrayList<StatusJournalEntry>();
}
- if (includeCurrentStatus) {
+ if (includeCurrentStatus && status != null) {
journal.add(new StatusJournalEntryImpl(status, statusTime));
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
index bd0900b..02bac97 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
import org.apache.hadoop.hbase.mob.MobUtils;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.HStore;
@@ -85,6 +86,7 @@ public final class SnapshotManifest {
private final FileSystem rootFs;
private final FileSystem workingDirFs;
private int manifestSizeLimit;
+ private final MonitoredTask statusTask;
/**
*
@@ -98,12 +100,13 @@ public final class SnapshotManifest {
*/
private SnapshotManifest(final Configuration conf, final FileSystem rootFs,
final Path workingDir, final SnapshotDescription desc,
- final ForeignExceptionSnare monitor) throws IOException {
+ final ForeignExceptionSnare monitor, final MonitoredTask statusTask) throws IOException {
this.monitor = monitor;
this.desc = desc;
this.workingDir = workingDir;
this.conf = conf;
this.rootFs = rootFs;
+ this.statusTask = statusTask;
this.workingDirFs = this.workingDir.getFileSystem(this.conf);
this.manifestSizeLimit = conf.getInt(SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 64 * 1024 * 1024);
}
@@ -124,7 +127,14 @@ public final class SnapshotManifest {
public static SnapshotManifest create(final Configuration conf, final FileSystem fs,
final Path workingDir, final SnapshotDescription desc,
final ForeignExceptionSnare monitor) throws IOException {
- return new SnapshotManifest(conf, fs, workingDir, desc, monitor);
+ return create(conf, fs, workingDir, desc, monitor, null);
+
+ }
+
+ public static SnapshotManifest create(final Configuration conf, final FileSystem fs,
+ final Path workingDir, final SnapshotDescription desc, final ForeignExceptionSnare monitor,
+ final MonitoredTask statusTask) throws IOException {
+ return new SnapshotManifest(conf, fs, workingDir, desc, monitor, statusTask);
}
@@ -139,7 +149,7 @@ public final class SnapshotManifest {
*/
public static SnapshotManifest open(final Configuration conf, final FileSystem fs,
final Path workingDir, final SnapshotDescription desc) throws IOException {
- SnapshotManifest manifest = new SnapshotManifest(conf, fs, workingDir, desc, null);
+ SnapshotManifest manifest = new SnapshotManifest(conf, fs, workingDir, desc, null, null);
manifest.load();
return manifest;
}
@@ -456,6 +466,12 @@ public final class SnapshotManifest {
return this.regionManifests;
}
+ private void setStatusMsg(String msg) {
+ if (this.statusTask != null) {
+ statusTask.setStatus(msg);
+ }
+ }
+
/**
* Get all the Region Manifest from the snapshot.
* This is an helper to get a map with the region encoded name
@@ -478,7 +494,7 @@ public final class SnapshotManifest {
FSTableDescriptors.createTableDescriptorForTableDirectory(workingDirFs, workingDir, htd,
false);
} else {
- LOG.debug("Convert to Single Snapshot Manifest");
+ LOG.debug("Convert to Single Snapshot Manifest for {}", this.desc.getName());
convertToV2SingleManifest();
}
}
@@ -491,6 +507,7 @@ public final class SnapshotManifest {
// Try to load v1 and v2 regions
List<SnapshotRegionManifest> v1Regions, v2Regions;
ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
+ setStatusMsg("Loading Region manifests for " + this.desc.getName());
try {
v1Regions = SnapshotManifestV1.loadRegionManifests(conf, tpool, workingDirFs,
workingDir, desc);
@@ -514,6 +531,7 @@ public final class SnapshotManifest {
// Once the data-manifest is written, the snapshot can be considered complete.
// Currently snapshots are written in a "temporary" directory and later
// moved to the "complated" snapshot directory.
+ setStatusMsg("Writing data manifest for " + this.desc.getName());
SnapshotDataManifest dataManifest = dataManifestBuilder.build();
writeDataManifest(dataManifest);
this.regionManifests = dataManifest.getRegionManifestsList();