You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by rm...@apache.org on 2024/10/08 19:50:45 UTC
(hbase) 01/01: HBASE-28897 Force CF compatibility during incremental backup (#6340)
This is an automated email from the ASF dual-hosted git repository.
rmattingly pushed a commit to branch 28897-branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 8d768958d024a6ddc541912c027bd675d85a7bd4
Author: Hernan Romer <na...@gmail.com>
AuthorDate: Tue Oct 8 15:49:04 2024 -0400
HBASE-28897 Force CF compatibility during incremental backup (#6340)
Co-authored-by: Hernan Gelaf-Romer <hg...@hubspot.com>
Signed-off-by: Ray Mattingly <rm...@apache.org>
---
.../backup/impl/ColumnFamilyMismatchException.java | 66 +++++++++++++++
.../backup/impl/IncrementalTableBackupClient.java | 95 +++++++++++++++++++++-
.../hadoop/hbase/backup/TestIncrementalBackup.java | 36 ++++++--
3 files changed, 189 insertions(+), 8 deletions(-)
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/ColumnFamilyMismatchException.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/ColumnFamilyMismatchException.java
new file mode 100644
index 00000000000..87cc896e182
--- /dev/null
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/ColumnFamilyMismatchException.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup.impl;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Public
+public final class ColumnFamilyMismatchException extends BackupException {
+ private final List<TableName> mismatchedTables;
+
+ private ColumnFamilyMismatchException(String msg, List<TableName> mismatchedTables) {
+ super(msg);
+ this.mismatchedTables = mismatchedTables;
+ }
+
+ public static final class ColumnFamilyMismatchExceptionBuilder {
+ private final List<TableName> mismatchedTables = new ArrayList<>();
+ private final StringBuilder msg = new StringBuilder();
+
+ public ColumnFamilyMismatchExceptionBuilder addMismatchedTable(TableName tableName,
+ ColumnFamilyDescriptor[] currentCfs, ColumnFamilyDescriptor[] backupCfs) {
+ this.mismatchedTables.add(tableName);
+
+ String currentCfsParsed = StringUtils.join(currentCfs, ',');
+ String backupCfsParsed = StringUtils.join(backupCfs, ',');
+ msg.append("\nMismatch in column family descriptor for table: ").append(tableName)
+ .append("\n");
+ msg.append("Current families: ").append(currentCfsParsed).append("\n");
+ msg.append("Backup families: ").append(backupCfsParsed);
+
+ return this;
+ }
+
+ public ColumnFamilyMismatchException build() {
+ return new ColumnFamilyMismatchException(msg.toString(), mismatchedTables);
+ }
+ }
+
+ public List<TableName> getMismatchedTables() {
+ return mismatchedTables;
+ }
+
+ public static ColumnFamilyMismatchExceptionBuilder newBuilder() {
+ return new ColumnFamilyMismatchExceptionBuilder();
+ }
+}
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java
index bbb39cb3a03..88780fd2828 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -32,15 +33,20 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.backup.BackupCopyJob;
+import org.apache.hadoop.hbase.backup.BackupInfo;
import org.apache.hadoop.hbase.backup.BackupInfo.BackupPhase;
import org.apache.hadoop.hbase.backup.BackupRequest;
import org.apache.hadoop.hbase.backup.BackupRestoreFactory;
import org.apache.hadoop.hbase.backup.BackupType;
+import org.apache.hadoop.hbase.backup.HBackupFileSystem;
import org.apache.hadoop.hbase.backup.mapreduce.MapReduceBackupCopyJob;
import org.apache.hadoop.hbase.backup.util.BackupUtils;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.mapreduce.WALPlayer;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
@@ -51,6 +57,8 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
+
/**
* Incremental backup implementation. See the {@link #execute() execute} method.
*/
@@ -260,8 +268,11 @@ public class IncrementalTableBackupClient extends TableBackupClient {
}
@Override
- public void execute() throws IOException {
+ public void execute() throws IOException, ColumnFamilyMismatchException {
try {
+ Map<TableName, String> tablesToFullBackupIds = getFullBackupIds();
+ verifyCfCompatibility(backupInfo.getTables(), tablesToFullBackupIds);
+
// case PREPARE_INCREMENTAL:
beginBackup(backupManager, backupInfo);
backupInfo.setPhase(BackupPhase.PREPARE_INCREMENTAL);
@@ -434,4 +445,86 @@ public class IncrementalTableBackupClient extends TableBackupClient {
path = new Path(path, backupId);
return path;
}
+
+ private Map<TableName, String> getFullBackupIds() throws IOException {
+ // Ancestors are stored from newest to oldest, so we can iterate backwards
+ // in order to populate our backupId map with the most recent full backup
+ // for a given table
+ List<BackupManifest.BackupImage> images = getAncestors(backupInfo);
+ Map<TableName, String> results = new HashMap<>();
+ for (int i = images.size() - 1; i >= 0; i--) {
+ BackupManifest.BackupImage image = images.get(i);
+ if (image.getType() != BackupType.FULL) {
+ continue;
+ }
+
+ for (TableName tn : image.getTableNames()) {
+ results.put(tn, image.getBackupId());
+ }
+ }
+ return results;
+ }
+
+ /**
+ * Verifies that the current table descriptor CFs matches the descriptor CFs of the last full
+ * backup for the tables. This ensures CF compatibility across incremental backups. If a mismatch
+ * is detected, a full table backup should be taken, rather than an incremental one
+ */
+ private void verifyCfCompatibility(Set<TableName> tables,
+ Map<TableName, String> tablesToFullBackupId) throws IOException, ColumnFamilyMismatchException {
+ ColumnFamilyMismatchException.ColumnFamilyMismatchExceptionBuilder exBuilder =
+ ColumnFamilyMismatchException.newBuilder();
+ try (Admin admin = conn.getAdmin(); BackupAdminImpl backupAdmin = new BackupAdminImpl(conn)) {
+ for (TableName tn : tables) {
+ String backupId = tablesToFullBackupId.get(tn);
+ BackupInfo fullBackupInfo = backupAdmin.getBackupInfo(backupId);
+
+ ColumnFamilyDescriptor[] currentCfs = admin.getDescriptor(tn).getColumnFamilies();
+ String snapshotName = fullBackupInfo.getSnapshotName(tn);
+ Path root = HBackupFileSystem.getTableBackupPath(tn,
+ new Path(fullBackupInfo.getBackupRootDir()), fullBackupInfo.getBackupId());
+ Path manifestDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, root);
+
+ FileSystem fs;
+ try {
+ fs = FileSystem.get(new URI(fullBackupInfo.getBackupRootDir()), conf);
+ } catch (URISyntaxException e) {
+ throw new IOException("Unable to get fs", e);
+ }
+
+ SnapshotProtos.SnapshotDescription snapshotDescription =
+ SnapshotDescriptionUtils.readSnapshotInfo(fs, manifestDir);
+ SnapshotManifest manifest =
+ SnapshotManifest.open(conf, fs, manifestDir, snapshotDescription);
+
+ ColumnFamilyDescriptor[] backupCfs = manifest.getTableDescriptor().getColumnFamilies();
+ if (!areCfsCompatible(currentCfs, backupCfs)) {
+ exBuilder.addMismatchedTable(tn, currentCfs, backupCfs);
+ }
+ }
+ }
+
+ ColumnFamilyMismatchException ex = exBuilder.build();
+ if (!ex.getMismatchedTables().isEmpty()) {
+ throw ex;
+ }
+ }
+
+ private static boolean areCfsCompatible(ColumnFamilyDescriptor[] currentCfs,
+ ColumnFamilyDescriptor[] backupCfs) {
+ if (currentCfs.length != backupCfs.length) {
+ return false;
+ }
+
+ for (int i = 0; i < backupCfs.length; i++) {
+ String currentCf = currentCfs[i].getNameAsString();
+ String backupCf = backupCfs[i].getNameAsString();
+
+ if (!currentCf.equals(backupCf)) {
+ return false;
+ }
+ }
+
+ return true;
+ }
}
diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java
index 86966ddfd6e..b8beeaca143 100644
--- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java
+++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.hbase.backup;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
@@ -31,6 +33,7 @@ import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl;
import org.apache.hadoop.hbase.backup.impl.BackupManifest;
+import org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException;
import org.apache.hadoop.hbase.backup.util.BackupUtils;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
@@ -53,6 +56,7 @@ import org.junit.runners.Parameterized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
@@ -102,9 +106,7 @@ public class TestIncrementalBackup extends TestBackupBase {
insertIntoTable(conn, table1, mobName, 3, NB_ROWS_FAM3).close();
Admin admin = conn.getAdmin();
BackupAdminImpl client = new BackupAdminImpl(conn);
- BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR);
- String backupIdFull = client.backupTables(request);
- assertTrue(checkSucceeded(backupIdFull));
+ String backupIdFull = takeFullBackup(tables, client);
// #2 - insert some data to table
Table t1 = insertIntoTable(conn, table1, famName, 1, ADD_ROWS);
@@ -141,16 +143,14 @@ public class TestIncrementalBackup extends TestBackupBase {
// exception will be thrown.
LOG.debug("region is not splittable, because " + e);
}
- while (!admin.isTableAvailable(table1)) {
- Thread.sleep(100);
- }
+ TEST_UTIL.waitTableAvailable(table1);
long endSplitTime = EnvironmentEdgeManager.currentTime();
// split finished
LOG.debug("split finished in =" + (endSplitTime - startSplitTime));
// #3 - incremental backup for multiple tables
tables = Lists.newArrayList(table1, table2);
- request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
+ BackupRequest request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
String backupIdIncMultiple = client.backupTables(request);
assertTrue(checkSucceeded(backupIdIncMultiple));
BackupManifest manifest =
@@ -165,6 +165,13 @@ public class TestIncrementalBackup extends TestBackupBase {
.build();
TEST_UTIL.getAdmin().modifyTable(newTable1Desc);
+ // check that an incremental backup fails because the CFs don't match
+ final List<TableName> tablesCopy = tables;
+ IOException ex = assertThrows(IOException.class, () -> client
+ .backupTables(createBackupRequest(BackupType.INCREMENTAL, tablesCopy, BACKUP_ROOT_DIR)));
+ checkThrowsCFMismatch(ex, List.of(table1));
+ takeFullBackup(tables, client);
+
int NB_ROWS_FAM2 = 7;
Table t3 = insertIntoTable(conn, table1, fam2Name, 2, NB_ROWS_FAM2);
t3.close();
@@ -227,4 +234,19 @@ public class TestIncrementalBackup extends TestBackupBase {
admin.close();
}
}
+
+ private void checkThrowsCFMismatch(IOException ex, List<TableName> tables) {
+ Throwable cause = Throwables.getRootCause(ex);
+ assertEquals(cause.getClass(), ColumnFamilyMismatchException.class);
+ ColumnFamilyMismatchException e = (ColumnFamilyMismatchException) cause;
+ assertEquals(tables, e.getMismatchedTables());
+ }
+
+ private String takeFullBackup(List<TableName> tables, BackupAdminImpl backupAdmin)
+ throws IOException {
+ BackupRequest req = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR);
+ String backupId = backupAdmin.backupTables(req);
+ checkSucceeded(backupId);
+ return backupId;
+ }
}