You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by an...@apache.org on 2026/07/15 15:47:34 UTC
(hbase) 02/08: Delete hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java
This is an automated email from the ASF dual-hosted git repository.
anmolnar pushed a commit to branch HBASE-29164
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 7de873b583cf87a1c9dcc1498e1e9bb86e842a47
Author: Kevin Geiszler <ke...@gmail.com>
AuthorDate: Mon Jul 13 16:19:09 2026 -0400
Delete hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java
Change-Id: Ic99e588aa97765957f15458a39f863834fbfcd2b
---
.../hadoop/hbase/IntegrationTestBackupRestore.java | 468 ---------------------
1 file changed, 468 deletions(-)
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java
deleted file mode 100644
index ea55a5c61d7..00000000000
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java
+++ /dev/null
@@ -1,468 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase;
-
-import static org.apache.hadoop.hbase.IntegrationTestingUtility.createPreSplitLoadTestTable;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.backup.BackupAdmin;
-import org.apache.hadoop.hbase.backup.BackupInfo;
-import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
-import org.apache.hadoop.hbase.backup.BackupRequest;
-import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
-import org.apache.hadoop.hbase.backup.BackupType;
-import org.apache.hadoop.hbase.backup.RestoreRequest;
-import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl;
-import org.apache.hadoop.hbase.backup.impl.BackupManager;
-import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
-import org.apache.hadoop.hbase.chaos.actions.RestartRandomRsExceptMetaAction;
-import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey;
-import org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy;
-import org.apache.hadoop.hbase.chaos.policies.Policy;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.client.TableDescriptor;
-import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
-import org.apache.hadoop.hbase.testclassification.IntegrationTests;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.util.ToolRunner;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Tag;
-import org.junit.jupiter.api.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hbase.thirdparty.com.google.common.base.MoreObjects;
-import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
-import org.apache.hbase.thirdparty.com.google.common.util.concurrent.Uninterruptibles;
-import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
-
-/**
- * An integration test to detect regressions in HBASE-7912. Create a table with many regions, load
- * data, perform series backup/load operations, then restore and verify data
- * @see <a href="https://issues.apache.org/jira/browse/HBASE-7912">HBASE-7912</a>
- * @see <a href="https://issues.apache.org/jira/browse/HBASE-14123">HBASE-14123</a>
- */
-@Tag(IntegrationTests.TAG)
-public class IntegrationTestBackupRestore extends IntegrationTestBase {
- private static final String CLASS_NAME = IntegrationTestBackupRestore.class.getSimpleName();
- protected static final Logger LOG = LoggerFactory.getLogger(IntegrationTestBackupRestore.class);
- protected static final String NUMBER_OF_TABLES_KEY = "num_tables";
- protected static final String COLUMN_NAME = "f";
- protected static final String REGION_COUNT_KEY = "regions_per_rs";
- protected static final String REGIONSERVER_COUNT_KEY = "region_servers";
- protected static final String ROWS_PER_ITERATION_KEY = "rows_in_iteration";
- protected static final String NUM_ITERATIONS_KEY = "num_iterations";
- protected static final int DEFAULT_REGION_COUNT = 10;
- protected static final int DEFAULT_REGIONSERVER_COUNT = 5;
- protected static final int DEFAULT_NUMBER_OF_TABLES = 1;
- protected static final int DEFAULT_NUM_ITERATIONS = 10;
- protected static final int DEFAULT_ROWS_IN_ITERATION = 10000;
- protected static final String SLEEP_TIME_KEY = "sleeptime";
- // short default interval because tests don't run very long.
- protected static final long SLEEP_TIME_DEFAULT = 50000L;
-
- protected static int rowsInIteration;
- protected static int regionsCountPerServer;
- protected static int regionServerCount;
-
- protected static int numIterations;
- protected static int numTables;
- protected static TableName[] tableNames;
- protected long sleepTime;
- protected static Object lock = new Object();
-
- private static String BACKUP_ROOT_DIR = "backupIT";
-
- /*
- * This class is used to run the backup and restore thread(s). Throwing an exception in this
- * thread will not cause the test to fail, so the purpose of this class is to both kick off the
- * backup and restore and record any exceptions that occur so they can be thrown in the main
- * thread.
- */
- protected class BackupAndRestoreThread implements Runnable {
- private final TableName table;
- private Throwable throwable;
-
- public BackupAndRestoreThread(TableName table) {
- this.table = table;
- this.throwable = null;
- }
-
- public Throwable getThrowable() {
- return this.throwable;
- }
-
- @Override
- public void run() {
- try {
- runTestSingle(this.table);
- } catch (Throwable t) {
- LOG.error(
- "An error occurred in thread {} when performing a backup and restore with table {}: ",
- Thread.currentThread().getName(), this.table.getNameAsString(), t);
- this.throwable = t;
- }
- }
- }
-
- @Override
- @BeforeEach
- public void setUp() throws Exception {
- util = new IntegrationTestingUtility();
- Configuration conf = util.getConfiguration();
- regionsCountPerServer = conf.getInt(REGION_COUNT_KEY, DEFAULT_REGION_COUNT);
- regionServerCount = conf.getInt(REGIONSERVER_COUNT_KEY, DEFAULT_REGIONSERVER_COUNT);
- rowsInIteration = conf.getInt(ROWS_PER_ITERATION_KEY, DEFAULT_ROWS_IN_ITERATION);
- numIterations = conf.getInt(NUM_ITERATIONS_KEY, DEFAULT_NUM_ITERATIONS);
- numTables = conf.getInt(NUMBER_OF_TABLES_KEY, DEFAULT_NUMBER_OF_TABLES);
- sleepTime = conf.getLong(SLEEP_TIME_KEY, SLEEP_TIME_DEFAULT);
- enableBackup(conf);
- LOG.info("Initializing cluster with {} region servers.", regionServerCount);
- util.initializeCluster(regionServerCount);
- LOG.info("Cluster initialized and ready");
- }
-
- @AfterEach
- public void tearDown() throws IOException {
- LOG.info("Cleaning up after test.");
- if (util.isDistributedCluster()) {
- deleteTablesIfAny();
- LOG.info("Cleaning up after test. Deleted tables");
- cleanUpBackupDir();
- }
- LOG.info("Restoring cluster.");
- util.restoreCluster();
- LOG.info("Cluster restored.");
- }
-
- @Override
- public void setUpMonkey() throws Exception {
- Policy p =
- new PeriodicRandomActionPolicy(sleepTime, new RestartRandomRsExceptMetaAction(sleepTime));
- this.monkey = new PolicyBasedChaosMonkey(util, p);
- startMonkey();
- }
-
- private void deleteTablesIfAny() throws IOException {
- for (TableName table : tableNames) {
- util.deleteTableIfAny(table);
- }
- }
-
- private void createTables() throws Exception {
- tableNames = new TableName[numTables];
- for (int i = 0; i < numTables; i++) {
- tableNames[i] = TableName.valueOf(CLASS_NAME + ".table." + i);
- }
- for (TableName table : tableNames) {
- createTable(table);
- }
- }
-
- private void enableBackup(Configuration conf) {
- // Enable backup
- conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
- BackupManager.decorateMasterConfiguration(conf);
- BackupManager.decorateRegionServerConfiguration(conf);
- }
-
- private void cleanUpBackupDir() throws IOException {
- FileSystem fs = FileSystem.get(util.getConfiguration());
- fs.delete(new Path(BACKUP_ROOT_DIR), true);
- }
-
- @Test
- public void testBackupRestore() throws Exception {
- BACKUP_ROOT_DIR = util.getDataTestDirOnTestFS() + Path.SEPARATOR + BACKUP_ROOT_DIR;
- createTables();
- runTestMulti();
- }
-
- private void runTestMulti() throws Exception {
- LOG.info("IT backup & restore started");
- Thread[] workers = new Thread[numTables];
- BackupAndRestoreThread[] backupAndRestoreThreads = new BackupAndRestoreThread[numTables];
- for (int i = 0; i < numTables; i++) {
- final TableName table = tableNames[i];
- BackupAndRestoreThread backupAndRestoreThread = new BackupAndRestoreThread(table);
- backupAndRestoreThreads[i] = backupAndRestoreThread;
- workers[i] = new Thread(backupAndRestoreThread);
- workers[i].start();
- }
- // Wait for all workers to finish and check for errors
- Throwable error = null;
- Throwable threadThrowable;
- for (int i = 0; i < numTables; i++) {
- Uninterruptibles.joinUninterruptibly(workers[i]);
- threadThrowable = backupAndRestoreThreads[i].getThrowable();
- if (threadThrowable == null) {
- continue;
- }
- if (error == null) {
- error = threadThrowable;
- } else {
- error.addSuppressed(threadThrowable);
- }
- }
- // Throw any found errors after all threads have completed
- if (error != null) {
- throw new AssertionError("An error occurred in a backup and restore thread", error);
- }
- LOG.info("IT backup & restore finished");
- }
-
- private void createTable(TableName tableName) throws Exception {
- long startTime, endTime;
-
- TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
-
- TableDescriptor desc = builder.build();
- ColumnFamilyDescriptorBuilder cbuilder =
- ColumnFamilyDescriptorBuilder.newBuilder(COLUMN_NAME.getBytes(Charset.defaultCharset()));
- ColumnFamilyDescriptor[] columns = new ColumnFamilyDescriptor[] { cbuilder.build() };
- LOG.info("Creating table {} with {} splits.", tableName,
- regionsCountPerServer * regionServerCount);
- startTime = EnvironmentEdgeManager.currentTime();
- createPreSplitLoadTestTable(util.getConfiguration(), desc, columns, regionsCountPerServer);
- util.waitTableAvailable(tableName);
- endTime = EnvironmentEdgeManager.currentTime();
- LOG.info("Pre-split table created successfully in {}ms.", (endTime - startTime));
- }
-
- private void loadData(TableName table, int numRows) throws IOException {
- Connection conn = util.getConnection();
- // #0- insert some data to a table
- Table t1 = conn.getTable(table);
- util.loadRandomRows(t1, new byte[] { 'f' }, 100, numRows);
- // flush table
- conn.getAdmin().flush(TableName.valueOf(table.getName()));
- }
-
- private String backup(BackupRequest request, BackupAdmin client) throws IOException {
- return client.backupTables(request);
- }
-
- private void restore(RestoreRequest request, BackupAdmin client) throws IOException {
- client.restore(request);
- }
-
- private void merge(String[] backupIds, BackupAdmin client) throws IOException {
- client.mergeBackups(backupIds);
- }
-
- private void runTestSingle(TableName table) throws IOException {
-
- List<String> backupIds = new ArrayList<String>();
-
- try (Connection conn = util.getConnection(); Admin admin = conn.getAdmin();
- BackupAdmin client = new BackupAdminImpl(conn);) {
-
- // #0- insert some data to table 'table'
- loadData(table, rowsInIteration);
-
- // #1 - create full backup for table first
- LOG.info("create full backup image for {}", table);
- List<TableName> tables = Lists.newArrayList(table);
- BackupRequest.Builder builder = new BackupRequest.Builder();
- BackupRequest request = builder.withBackupType(BackupType.FULL).withTableList(tables)
- .withTargetRootDir(BACKUP_ROOT_DIR).build();
-
- String backupIdFull = backup(request, client);
- assertTrue(checkSucceeded(backupIdFull));
-
- backupIds.add(backupIdFull);
- // Now continue with incremental backups
- int count = 1;
- while (count++ < numIterations) {
-
- // Load data
- loadData(table, rowsInIteration);
- // Do incremental backup
- builder = new BackupRequest.Builder();
- request = builder.withBackupType(BackupType.INCREMENTAL).withTableList(tables)
- .withTargetRootDir(BACKUP_ROOT_DIR).build();
- String backupId = backup(request, client);
- assertTrue(checkSucceeded(backupId));
- backupIds.add(backupId);
-
- // Restore incremental backup for table, with overwrite for previous backup
- String previousBackupId = backupIds.get(backupIds.size() - 2);
- restoreVerifyTable(conn, client, table, previousBackupId, rowsInIteration * (count - 1));
- // Restore incremental backup for table, with overwrite for last backup
- restoreVerifyTable(conn, client, table, backupId, rowsInIteration * count);
- }
- // Now merge all incremental and restore
- String[] incBackupIds = allIncremental(backupIds);
- merge(incBackupIds, client);
- // Restore last one
- String backupId = incBackupIds[incBackupIds.length - 1];
- // restore incremental backup for table, with overwrite
- TableName[] tablesRestoreIncMultiple = new TableName[] { table };
- restore(createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, tablesRestoreIncMultiple, null,
- true), client);
- Table hTable = conn.getTable(table);
- assertEquals(util.countRows(hTable), rowsInIteration * numIterations);
- hTable.close();
- LOG.info("{} loop {} finished.", Thread.currentThread().getName(), (count - 1));
- }
- }
-
- private void restoreVerifyTable(Connection conn, BackupAdmin client, TableName table,
- String backupId, long expectedRows) throws IOException {
- TableName[] tablesRestoreIncMultiple = new TableName[] { table };
- restore(
- createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, tablesRestoreIncMultiple, null, true),
- client);
- Table hTable = conn.getTable(table);
- assertEquals(expectedRows, util.countRows(hTable));
- hTable.close();
- }
-
- private String[] allIncremental(List<String> backupIds) {
- int size = backupIds.size();
- backupIds = backupIds.subList(1, size);
- String[] arr = new String[size - 1];
- backupIds.toArray(arr);
- return arr;
- }
-
- /** Returns status of backup */
- protected boolean checkSucceeded(String backupId) throws IOException {
- BackupInfo status = getBackupInfo(backupId);
- if (status == null) {
- return false;
- }
- return status.getState() == BackupState.COMPLETE;
- }
-
- private BackupInfo getBackupInfo(String backupId) throws IOException {
- try (BackupSystemTable table = new BackupSystemTable(util.getConnection())) {
- return table.readBackupInfo(backupId);
- }
- }
-
- /**
- * Get restore request.
- * @param backupRootDir directory where backup is located
- * @param backupId backup ID
- * @param check check the backup
- * @param fromTables table names to restore from
- * @param toTables new table names to restore to
- * @param isOverwrite overwrite the table(s)
- * @return an instance of RestoreRequest
- */
- public RestoreRequest createRestoreRequest(String backupRootDir, String backupId, boolean check,
- TableName[] fromTables, TableName[] toTables, boolean isOverwrite) {
- RestoreRequest.Builder builder = new RestoreRequest.Builder();
- return builder.withBackupRootDir(backupRootDir).withBackupId(backupId).withCheck(check)
- .withFromTables(fromTables).withToTables(toTables).withOverwrite(isOverwrite).build();
- }
-
- @Override
- public void setUpCluster() throws Exception {
- util = getTestingUtil(getConf());
- enableBackup(getConf());
- LOG.debug("Initializing/checking cluster has {} servers", regionServerCount);
- util.initializeCluster(regionServerCount);
- LOG.debug("Done initializing/checking cluster");
- }
-
- /** Returns status of CLI execution */
- @Override
- public int runTestFromCommandLine() throws Exception {
- // Check if backup is enabled
- if (!BackupManager.isBackupEnabled(getConf())) {
- System.err.println(BackupRestoreConstants.ENABLE_BACKUP);
- return -1;
- }
- System.out.println(BackupRestoreConstants.VERIFY_BACKUP);
- testBackupRestore();
- return 0;
- }
-
- @Override
- public TableName getTablename() {
- // That is only valid when Monkey is CALM (no monkey)
- return null;
- }
-
- @Override
- protected Set<String> getColumnFamilies() {
- // That is only valid when Monkey is CALM (no monkey)
- return null;
- }
-
- @Override
- protected void addOptions() {
- addOptWithArg(REGIONSERVER_COUNT_KEY,
- "Total number of region servers. Default: '" + DEFAULT_REGIONSERVER_COUNT + "'");
- addOptWithArg(REGION_COUNT_KEY, "Total number of regions. Default: " + DEFAULT_REGION_COUNT);
- addOptWithArg(ROWS_PER_ITERATION_KEY,
- "Total number of data rows to be loaded during one iteration." + " Default: "
- + DEFAULT_ROWS_IN_ITERATION);
- addOptWithArg(NUM_ITERATIONS_KEY,
- "Total number iterations." + " Default: " + DEFAULT_NUM_ITERATIONS);
- addOptWithArg(NUMBER_OF_TABLES_KEY,
- "Total number of tables in the test." + " Default: " + DEFAULT_NUMBER_OF_TABLES);
- addOptWithArg(SLEEP_TIME_KEY, "Sleep time of chaos monkey in ms "
- + "to restart random region server. Default: " + SLEEP_TIME_DEFAULT);
- }
-
- @Override
- protected void processOptions(CommandLine cmd) {
- super.processOptions(cmd);
- regionsCountPerServer = Integer
- .parseInt(cmd.getOptionValue(REGION_COUNT_KEY, Integer.toString(DEFAULT_REGION_COUNT)));
- regionServerCount = Integer.parseInt(
- cmd.getOptionValue(REGIONSERVER_COUNT_KEY, Integer.toString(DEFAULT_REGIONSERVER_COUNT)));
- rowsInIteration = Integer.parseInt(
- cmd.getOptionValue(ROWS_PER_ITERATION_KEY, Integer.toString(DEFAULT_ROWS_IN_ITERATION)));
- numIterations = Integer
- .parseInt(cmd.getOptionValue(NUM_ITERATIONS_KEY, Integer.toString(DEFAULT_NUM_ITERATIONS)));
- numTables = Integer.parseInt(
- cmd.getOptionValue(NUMBER_OF_TABLES_KEY, Integer.toString(DEFAULT_NUMBER_OF_TABLES)));
- sleepTime =
- Long.parseLong(cmd.getOptionValue(SLEEP_TIME_KEY, Long.toString(SLEEP_TIME_DEFAULT)));
-
- LOG.info(MoreObjects.toStringHelper("Parsed Options")
- .add(REGION_COUNT_KEY, regionsCountPerServer).add(REGIONSERVER_COUNT_KEY, regionServerCount)
- .add(ROWS_PER_ITERATION_KEY, rowsInIteration).add(NUM_ITERATIONS_KEY, numIterations)
- .add(NUMBER_OF_TABLES_KEY, numTables).add(SLEEP_TIME_KEY, sleepTime).toString());
- }
-
- public static void main(String[] args) throws Exception {
- Configuration conf = HBaseConfiguration.create();
- IntegrationTestingUtility.setUseDistributedCluster(conf);
- int status = ToolRunner.run(conf, new IntegrationTestBackupRestore(), args);
- System.exit(status);
- }
-}