You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by an...@apache.org on 2026/03/12 14:36:40 UTC
(hbase) 01/01: HBASE-29081: Feature implementation
This is an automated email from the ASF dual-hosted git repository.
andor pushed a commit to branch HBASE-29081_rebased
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 6da4bfc0685524845d979cec1125d42a10ac6477
Author: Abhishek Kothalikar <99...@users.noreply.github.com>
AuthorDate: Tue Mar 11 20:03:25 2025 +0530
HBASE-29081: Feature implementation
(cherry picked from commit 7ab9d52801fc5be72b742582d1732a8f5e602d86)
---
.../backup/impl/IncrementalBackupManager.java | 3 +-
.../hbase/backup/master/BackupLogCleaner.java | 2 +-
.../master/SnapshotOfRegionAssignmentFromMeta.java | 6 +-
.../apache/hadoop/hbase/ActiveClusterSuffix.java | 98 +++++
.../java/org/apache/hadoop/hbase/client/Admin.java | 28 ++
.../hadoop/hbase/client/AdminOverAsyncAdmin.java | 20 +
.../org/apache/hadoop/hbase/client/AsyncAdmin.java | 20 +
.../hadoop/hbase/client/AsyncHBaseAdmin.java | 20 +
.../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 63 +++
.../org/apache/hadoop/hbase/client/RegionInfo.java | 2 +-
.../java/org/apache/hadoop/hbase/HConstants.java | 25 ++
.../java/org/apache/hadoop/hbase/TableName.java | 37 +-
.../master/MetricsMasterFileSystemSource.java | 3 +-
.../main/protobuf/server/ActiveClusterSuffix.proto | 33 ++
.../src/main/protobuf/server/master/Master.proto | 25 ++
.../protobuf/server/master/MasterProcedure.proto | 31 ++
.../org/apache/hadoop/hbase/MetaTableAccessor.java | 4 +-
.../org/apache/hadoop/hbase/TableDescriptors.java | 7 +
.../hadoop/hbase/coprocessor/CoprocessorHost.java | 35 ++
.../coprocessor/MasterCoprocessorEnvironment.java | 3 +
.../apache/hadoop/hbase/executor/EventType.java | 8 +-
.../apache/hadoop/hbase/executor/ExecutorType.java | 3 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 135 +++++-
.../hadoop/hbase/master/MasterCoprocessorHost.java | 5 +
.../hadoop/hbase/master/MasterFileSystem.java | 76 +++-
.../hadoop/hbase/master/MasterRpcServices.java | 31 ++
.../hbase/master/RegionPlacementMaintainer.java | 16 +-
.../hbase/master/assignment/AssignmentManager.java | 6 +-
.../assignment/MergeTableRegionsProcedure.java | 6 +-
.../hbase/master/assignment/RegionStateStore.java | 11 +-
.../assignment/SplitTableRegionProcedure.java | 6 +-
.../apache/hadoop/hbase/master/hbck/HbckChore.java | 3 +-
.../hbase/master/janitor/CatalogJanitor.java | 4 +-
.../hadoop/hbase/master/janitor/MetaFixer.java | 12 +-
.../hbase/master/janitor/ReportMakingVisitor.java | 6 +-
.../master/procedure/HBCKServerCrashProcedure.java | 13 +-
.../hbase/master/procedure/InitMetaProcedure.java | 2 +-
.../master/procedure/ModifyTableProcedure.java | 2 +-
.../procedure/RefreshHFilesRegionProcedure.java | 208 +++++++++
.../procedure/RefreshHFilesTableProcedure.java | 165 +++++++
.../master/procedure/RefreshMetaProcedure.java | 480 ++++++++++++++++++++
.../master/procedure/TableProcedureInterface.java | 3 +-
.../hadoop/hbase/master/procedure/TableQueue.java | 1 +
.../hbase/master/region/MasterRegionFactory.java | 28 +-
.../store/region/HFileProcedurePrettyPrinter.java | 2 +-
.../hadoop/hbase/regionserver/CompactSplit.java | 19 +
.../apache/hadoop/hbase/regionserver/HRegion.java | 25 ++
.../hadoop/hbase/regionserver/HRegionServer.java | 21 +
.../hbase/regionserver/RefreshHFilesCallable.java | 70 +++
.../storefiletracker/StoreFileTrackerBase.java | 19 +-
.../access/AbstractReadOnlyController.java | 105 +++++
.../access/BulkLoadReadOnlyController.java | 53 +++
.../access/EndpointReadOnlyController.java | 50 +++
.../security/access/MasterReadOnlyController.java | 433 ++++++++++++++++++
.../security/access/RegionReadOnlyController.java | 371 ++++++++++++++++
.../access/RegionServerReadOnlyController.java | 64 +++
.../hadoop/hbase/tool/BulkLoadHFilesTool.java | 12 +-
.../hadoop/hbase/util/ConfigurationUtil.java | 6 +
.../hbase/util/CoprocessorConfigurationUtil.java | 85 ++++
.../hadoop/hbase/util/FSTableDescriptors.java | 29 +-
.../java/org/apache/hadoop/hbase/util/FSUtils.java | 112 ++++-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 75 ++--
.../org/apache/hadoop/hbase/util/RegionMover.java | 9 +-
.../hadoop/hbase/TestMetaTableForReplica.java | 176 ++++++++
.../apache/hadoop/hbase/TestRefreshHFilesBase.java | 157 +++++++
.../hbase/client/TestRefreshHFilesFromClient.java | 139 ++++++
.../hadoop/hbase/master/TestMasterMetrics.java | 201 ++++++++-
.../hbase/master/assignment/TestHbckChore.java | 43 ++
.../procedure/TestRefreshHFilesProcedure.java | 116 +++++
...TestRefreshHFilesProcedureWithReadOnlyConf.java | 124 ++++++
.../master/procedure/TestRefreshMetaProcedure.java | 121 ++++++
.../TestRefreshMetaProcedureIntegration.java | 275 ++++++++++++
.../master/region/TestMasterRegionInitialize.java | 18 +
.../region/TestHFileProcedurePrettyPrinter.java | 2 +-
.../region/TestWALProcedurePrettyPrinter.java | 2 +-
.../regionserver/TestActiveClusterSuffix.java | 158 +++++++
.../regionserver/TestCompactSplitReadOnly.java | 87 ++++
.../DummyStoreFileTrackerForReadOnlyMode.java | 79 ++++
.../TestStoreFileTrackerBaseReadOnlyMode.java | 146 +++++++
.../hbase/rsgroup/VerifyingRSGroupAdmin.java | 19 +
.../access/TestCanStartHBaseInReadOnlyMode.java | 67 +++
.../security/access/TestReadOnlyController.java | 217 +++++++++
.../TestReadOnlyControllerBulkLoadObserver.java | 70 +++
.../TestReadOnlyControllerCoprocessorLoading.java | 258 +++++++++++
.../TestReadOnlyControllerEndpointObserver.java | 75 ++++
.../TestReadOnlyControllerMasterObserver.java | 393 +++++++++++++++++
.../TestReadOnlyControllerRegionObserver.java | 483 +++++++++++++++++++++
...TestReadOnlyControllerRegionServerObserver.java | 89 ++++
.../TestReadOnlyManageActiveClusterFile.java | 153 +++++++
.../util/TestCoprocessorConfigurationUtil.java | 200 +++++++++
.../hadoop/hbase/util/TestFSTableDescriptors.java | 33 ++
hbase-shell/src/main/ruby/hbase/admin.rb | 26 ++
hbase-shell/src/main/ruby/shell.rb | 2 +
.../src/main/ruby/shell/commands/refresh_hfiles.rb | 64 +++
.../src/main/ruby/shell/commands/refresh_meta.rb | 43 ++
.../hadoop/hbase/thrift2/client/ThriftAdmin.java | 20 +
.../hadoop/hbase/zookeeper/MetaTableLocator.java | 19 +-
.../org/apache/hadoop/hbase/zookeeper/ZKDump.java | 3 +-
98 files changed, 7180 insertions(+), 153 deletions(-)
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
index 2d7aeb646db..5f48bfc39ad 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.backup.util.BackupUtils;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
@@ -156,7 +157,7 @@ public class IncrementalBackupManager extends BackupManager {
LOG.debug("currentLogFile: " + log.getPath().toString());
if (AbstractFSWALProvider.isMetaFile(log.getPath())) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Skip hbase:meta log file: " + log.getPath().getName());
+ LOG.debug("Skip {} log file: {}", TableName.META_TABLE_NAME, log.getPath().getName());
}
continue;
}
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java
index 971e80e2f83..eff1151b035 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java
@@ -204,6 +204,6 @@ public class BackupLogCleaner extends BaseLogCleanerDelegate {
String fn = path.getName();
return fn.startsWith(WALProcedureStore.LOG_PREFIX)
|| fn.endsWith(MasterRegionFactory.ARCHIVED_WAL_SUFFIX)
- || path.toString().contains("/%s/".formatted(MasterRegionFactory.MASTER_STORE_DIR));
+ || path.toString().contains("/%s/".formatted(MasterRegionFactory.getMasterRegionDirName()));
}
}
diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/SnapshotOfRegionAssignmentFromMeta.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/SnapshotOfRegionAssignmentFromMeta.java
index 02c18c73bfb..74d49ec14c0 100644
--- a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/SnapshotOfRegionAssignmentFromMeta.java
+++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/SnapshotOfRegionAssignmentFromMeta.java
@@ -170,7 +170,8 @@ public class SnapshotOfRegionAssignmentFromMeta {
* Initialize the region assignment snapshot by scanning the hbase:meta table
*/
public void initialize() throws IOException {
- LOG.info("Start to scan the hbase:meta for the current region assignment " + "snappshot");
+ LOG.info("Start to scan {} for the current region assignment snapshot",
+ TableName.META_TABLE_NAME);
// Scan hbase:meta to pick up user regions
try (Table metaTable = connection.getTable(TableName.META_TABLE_NAME);
ResultScanner scanner = metaTable.getScanner(HConstants.CATALOG_FAMILY)) {
@@ -187,7 +188,8 @@ public class SnapshotOfRegionAssignmentFromMeta {
}
}
}
- LOG.info("Finished to scan the hbase:meta for the current region assignment" + "snapshot");
+ LOG.info("Finished scanning {} for the current region assignment snapshot",
+ TableName.META_TABLE_NAME);
}
private void addRegion(RegionInfo regionInfo) {
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ActiveClusterSuffix.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ActiveClusterSuffix.java
new file mode 100644
index 00000000000..68fd61a2253
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ActiveClusterSuffix.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ActiveClusterSuffixProtos;
+
+/**
+ * The suffix for this cluster. It is serialized to the filesystem and up into zookeeper. This is a
+ * container for the id. Also knows how to serialize and deserialize the cluster id.
+ */
+@InterfaceAudience.Private
+public class ActiveClusterSuffix {
+ private final String active_cluster_suffix;
+
+ /**
+ * New ActiveClusterSuffix.
+ */
+
+ public ActiveClusterSuffix(final String cs) {
+ this.active_cluster_suffix = cs;
+ }
+
+ public String getActiveClusterSuffix() {
+ return active_cluster_suffix;
+ }
+
+ /** Returns The active cluster suffix serialized using pb w/ pb magic prefix */
+ public byte[] toByteArray() {
+ return ProtobufUtil.prependPBMagic(convert().toByteArray());
+ }
+
+ /**
+ * Parse the serialized representation of the {@link ActiveClusterSuffix}
+ * @param bytes A pb serialized {@link ActiveClusterSuffix} instance with pb magic prefix
+ * @return An instance of {@link ActiveClusterSuffix} made from <code>bytes</code>
+ * @see #toByteArray()
+ */
+ public static ActiveClusterSuffix parseFrom(final byte[] bytes) throws DeserializationException {
+ if (ProtobufUtil.isPBMagicPrefix(bytes)) {
+ int pblen = ProtobufUtil.lengthOfPBMagic();
+ ActiveClusterSuffixProtos.ActiveClusterSuffix.Builder builder =
+ ActiveClusterSuffixProtos.ActiveClusterSuffix.newBuilder();
+ ActiveClusterSuffixProtos.ActiveClusterSuffix cs = null;
+ try {
+ ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
+ cs = builder.build();
+ } catch (IOException e) {
+ throw new DeserializationException(e);
+ }
+ return convert(cs);
+ } else {
+ // Presume it was written out this way, the old way.
+ return new ActiveClusterSuffix(Bytes.toString(bytes));
+ }
+ }
+
+ /** Returns A pb instance to represent this instance. */
+ public ActiveClusterSuffixProtos.ActiveClusterSuffix convert() {
+ ActiveClusterSuffixProtos.ActiveClusterSuffix.Builder builder =
+ ActiveClusterSuffixProtos.ActiveClusterSuffix.newBuilder();
+ return builder.setActiveClusterSuffix(this.active_cluster_suffix).build();
+ }
+
+ /** Returns A {@link ActiveClusterSuffix} made from the passed in <code>cs</code> */
+ public static ActiveClusterSuffix
+ convert(final ActiveClusterSuffixProtos.ActiveClusterSuffix cs) {
+ return new ActiveClusterSuffix(cs.getActiveClusterSuffix());
+ }
+
+ /**
+ * @see java.lang.Object#toString()
+ */
+ @Override
+ public String toString() {
+ return this.active_cluster_suffix;
+ }
+}
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index 65b3abcd413..5d00366a9fe 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -2705,4 +2705,32 @@ public interface Admin extends Abortable, Closeable {
@InterfaceAudience.Private
void restoreBackupSystemTable(String snapshotName) throws IOException;
+
+ /**
+ * Perform hbase:meta table refresh
+ */
+ Long refreshMeta() throws IOException;
+
+ /**
+ * Refresh HFiles for the table
+ * @param tableName table to refresh HFiles for
+ * @return ID of the procedure started for refreshing HFiles
+ * @throws IOException if a remote or network exception occurs
+ */
+ Long refreshHFiles(final TableName tableName) throws IOException;
+
+ /**
+ * Refresh HFiles for all the tables under given namespace
+ * @param namespace Namespace for which we should call refresh HFiles for all tables under it
+ * @return ID of the procedure started for refreshing HFiles
+ * @throws IOException if a remote or network exception occurs
+ */
+ Long refreshHFiles(final String namespace) throws IOException;
+
+ /**
+ * Refresh HFiles for all the tables
+ * @return ID of the procedure started for refreshing HFiles
+ * @throws IOException if a remote or network exception occurs
+ */
+ Long refreshHFiles() throws IOException;
}
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java
index 7117fd4fd33..6628b7b3f78 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java
@@ -1157,4 +1157,24 @@ class AdminOverAsyncAdmin implements Admin {
public void restoreBackupSystemTable(String snapshotName) throws IOException {
get(admin.restoreBackupSystemTable(snapshotName));
}
+
+ @Override
+ public Long refreshMeta() throws IOException {
+ return get(admin.refreshMeta());
+ }
+
+ @Override
+ public Long refreshHFiles(final TableName tableName) throws IOException {
+ return get(admin.refreshHFiles(tableName));
+ }
+
+ @Override
+ public Long refreshHFiles(final String namespace) throws IOException {
+ return get(admin.refreshHFiles(namespace));
+ }
+
+ @Override
+ public Long refreshHFiles() throws IOException {
+ return get(admin.refreshHFiles());
+ }
}
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
index 56211cedc49..e27d7136c4d 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
@@ -1892,4 +1892,24 @@ public interface AsyncAdmin {
@InterfaceAudience.Private
CompletableFuture<Void> restoreBackupSystemTable(String snapshotName);
+
+ /**
+ * Perform hbase:meta table refresh
+ */
+ CompletableFuture<Long> refreshMeta();
+
+ /**
+ * Refresh HFiles for the table
+ */
+ CompletableFuture<Long> refreshHFiles(final TableName tableName);
+
+ /**
+ * Refresh HFiles for all the tables under given namespace
+ */
+ CompletableFuture<Long> refreshHFiles(final String namespace);
+
+ /**
+ * Refresh HFiles for all the tables
+ */
+ CompletableFuture<Long> refreshHFiles();
}
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
index 8132b184809..f7923d6cad2 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
@@ -1025,4 +1025,24 @@ class AsyncHBaseAdmin implements AsyncAdmin {
public CompletableFuture<Void> restoreBackupSystemTable(String snapshotName) {
return wrap(rawAdmin.restoreBackupSystemTable(snapshotName));
}
+
+ @Override
+ public CompletableFuture<Long> refreshMeta() {
+ return wrap(rawAdmin.refreshMeta());
+ }
+
+ @Override
+ public CompletableFuture<Long> refreshHFiles(final TableName tableName) {
+ return wrap(rawAdmin.refreshHFiles(tableName));
+ }
+
+ @Override
+ public CompletableFuture<Long> refreshHFiles(final String namespace) {
+ return wrap(rawAdmin.refreshHFiles(namespace));
+ }
+
+ @Override
+ public CompletableFuture<Long> refreshHFiles() {
+ return wrap(rawAdmin.refreshHFiles());
+ }
}
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
index ea51d27b99a..58409251cef 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
@@ -263,6 +263,10 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.OfflineReg
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.OfflineRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RecommissionRegionServerRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RecommissionRegionServerResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RefreshHFilesRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RefreshHFilesResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RefreshMetaRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RefreshMetaResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ReopenTableRegionsRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ReopenTableRegionsResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RestoreSnapshotRequest;
@@ -4697,4 +4701,63 @@ class RawAsyncHBaseAdmin implements AsyncAdmin {
MasterProtos.RestoreBackupSystemTableResponse::getProcId,
new RestoreBackupSystemTableProcedureBiConsumer());
}
+
+ private CompletableFuture<Long> internalRefershHFiles(RefreshHFilesRequest request) {
+ return this.<Long> newMasterCaller()
+ .action((controller, stub) -> this.<RefreshHFilesRequest, RefreshHFilesResponse, Long> call(
+ controller, stub, request, MasterService.Interface::refreshHFiles,
+ RefreshHFilesResponse::getProcId))
+ .call();
+ }
+
+ @Override
+ public CompletableFuture<Long> refreshMeta() {
+ RefreshMetaRequest.Builder request = RefreshMetaRequest.newBuilder();
+ request.setNonceGroup(ng.getNonceGroup()).setNonce(ng.newNonce());
+ return this.<Long> newMasterCaller()
+ .action((controller, stub) -> this.<RefreshMetaRequest, RefreshMetaResponse, Long> call(
+ controller, stub, request.build(), MasterService.Interface::refreshMeta,
+ RefreshMetaResponse::getProcId))
+ .call();
+ }
+
+ @Override
+ public CompletableFuture<Long> refreshHFiles(final TableName tableName) {
+ if (tableName.isSystemTable()) {
+ LOG.warn("Refreshing HFiles for system table {} is not allowed", tableName.getNameAsString());
+ throw new IllegalArgumentException(
+ "Not allowed to refresh HFiles for system table '" + tableName.getNameAsString() + "'");
+ }
+ // Request builder
+ RefreshHFilesRequest.Builder request = RefreshHFilesRequest.newBuilder();
+ request.setTableName(ProtobufUtil.toProtoTableName(tableName));
+ request.setNonceGroup(ng.getNonceGroup()).setNonce(ng.newNonce());
+ return internalRefershHFiles(request.build());
+ }
+
+ @Override
+ public CompletableFuture<Long> refreshHFiles(final String namespace) {
+ if (
+ namespace.equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)
+ || namespace.equals(NamespaceDescriptor.BACKUP_NAMESPACE_NAME_STR)
+ ) {
+ LOG.warn("Refreshing HFiles for reserve namespace {} is not allowed", namespace);
+ throw new IllegalArgumentException(
+ "Not allowed to refresh HFiles for reserve namespace '" + namespace + "'");
+ }
+ // Request builder
+ RefreshHFilesRequest.Builder request = RefreshHFilesRequest.newBuilder();
+ request.setNamespace(namespace);
+ request.setNonceGroup(ng.getNonceGroup()).setNonce(ng.newNonce());
+ return internalRefershHFiles(request.build());
+ }
+
+ @Override
+ public CompletableFuture<Long> refreshHFiles() {
+ // Request builder
+ RefreshHFilesRequest.Builder request = RefreshHFilesRequest.newBuilder();
+ // Set nonce
+ request.setNonceGroup(ng.getNonceGroup()).setNonce(ng.newNonce());
+ return internalRefershHFiles(request.build());
+ }
}
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java
index 10c554e26f7..82b8711b776 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java
@@ -431,7 +431,7 @@ public interface RegionInfo extends Comparable<RegionInfo> {
*/
static String prettyPrint(final String encodedRegionName) {
if (encodedRegionName.equals("1028785192")) {
- return encodedRegionName + "/hbase:meta";
+ return encodedRegionName + "/" + TableName.META_TABLE_NAME;
}
return encodedRegionName;
}
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 9af711e7edf..4ed6d3dc9d5 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1665,6 +1665,31 @@ public final class HConstants {
*/
public final static boolean REJECT_DECOMMISSIONED_HOSTS_DEFAULT = false;
+ /**
+ * Adds a suffix to the meta table name: value=’test’ -> ‘hbase:meta_test’ Added in HBASE-XXXXX to
+ * support having multiple hbase:meta tables (with distinct names )to enable storage sharing by
+ * more than one clusters.
+ */
+ public final static String HBASE_META_TABLE_SUFFIX = "hbase.meta.table.suffix";
+
+ /**
+ * Default value of {@link #HBASE_META_TABLE_SUFFIX}
+ */
+ public final static String HBASE_META_TABLE_SUFFIX_DEFAULT_VALUE = "";
+
+ /**
+ * Should HBase only serve Read Requests
+ */
+ public final static String HBASE_GLOBAL_READONLY_ENABLED_KEY = "hbase.global.readonly.enabled";
+
+ /**
+ * Default value of {@link #HBASE_GLOBAL_READONLY_ENABLED_KEY}
+ */
+ public final static boolean HBASE_GLOBAL_READONLY_ENABLED_DEFAULT = false;
+
+ /** name of the file having active cluster suffix */
+ public static final String ACTIVE_CLUSTER_SUFFIX_FILE_NAME = "active.cluster.suffix.id";
+
private HConstants() {
// Can't be instantiated with this ctor.
}
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java
index b6d854c1378..73008c7ad5f 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java
@@ -17,16 +17,21 @@
*/
package org.apache.hadoop.hbase;
+import com.google.errorprone.annotations.RestrictedApi;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArraySet;
import org.apache.commons.lang3.ArrayUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hbase.thirdparty.com.google.common.base.Strings;
/**
* Immutable POJO class for representing a table name. Which is of the form: <table
@@ -44,6 +49,7 @@ import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
*/
@InterfaceAudience.Public
public final class TableName implements Comparable<TableName> {
+ private static final Logger LOG = LoggerFactory.getLogger(TableName.class);
/** See {@link #createTableNameIfNecessary(ByteBuffer, ByteBuffer)} */
private static final Set<TableName> tableCache = new CopyOnWriteArraySet<>();
@@ -65,9 +71,34 @@ public final class TableName implements Comparable<TableName> {
public static final String VALID_USER_TABLE_REGEX = "(?:(?:(?:" + VALID_NAMESPACE_REGEX + "\\"
+ NAMESPACE_DELIM + ")?)" + "(?:" + VALID_TABLE_QUALIFIER_REGEX + "))";
- /** The hbase:meta table's name. */
- public static final TableName META_TABLE_NAME =
- valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "meta");
+ /**
+ * The name of hbase meta table could either be hbase:meta_xxx or 'hbase:meta' otherwise. Config
+ * hbase.meta.table.suffix will govern the decision of adding suffix to the habase:meta
+ */
+ public static final TableName META_TABLE_NAME;
+ static {
+ Configuration conf = HBaseConfiguration.create();
+ META_TABLE_NAME = initializeHbaseMetaTableName(conf);
+ LOG.info("Meta table name: {}", META_TABLE_NAME);
+ }
+
+ /* Visible for testing only */
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*")
+ public static TableName getDefaultNameOfMetaForReplica() {
+ return valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "meta");
+ }
+
+ public static TableName initializeHbaseMetaTableName(Configuration conf) {
+ String suffix_val = conf.get(HConstants.HBASE_META_TABLE_SUFFIX,
+ HConstants.HBASE_META_TABLE_SUFFIX_DEFAULT_VALUE);
+ LOG.info("Meta table suffix value: {}", suffix_val);
+ if (Strings.isNullOrEmpty(suffix_val)) {
+ return valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "meta");
+ } else {
+ return valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "meta_" + suffix_val);
+ }
+ }
/**
* The Namespace table's name.
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFileSystemSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFileSystemSource.java
index 53ed8a25ed0..579171e1c3d 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFileSystemSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFileSystemSource.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.master;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.metrics.BaseSource;
import org.apache.yetus.audience.InterfaceAudience;
@@ -49,7 +50,7 @@ public interface MetricsMasterFileSystemSource extends BaseSource {
String SPLIT_SIZE_NAME = "hlogSplitSize";
String META_SPLIT_TIME_DESC = "Time it takes to finish splitMetaLog()";
- String META_SPLIT_SIZE_DESC = "Size of hbase:meta WAL files being split";
+ String META_SPLIT_SIZE_DESC = "Size of " + TableName.META_TABLE_NAME + " WAL files being split";
String SPLIT_TIME_DESC = "Time it takes to finish WAL.splitLog()";
String SPLIT_SIZE_DESC = "Size of WAL files being split";
diff --git a/hbase-protocol-shaded/src/main/protobuf/server/ActiveClusterSuffix.proto b/hbase-protocol-shaded/src/main/protobuf/server/ActiveClusterSuffix.proto
new file mode 100644
index 00000000000..89bc086212b
--- /dev/null
+++ b/hbase-protocol-shaded/src/main/protobuf/server/ActiveClusterSuffix.proto
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+syntax = "proto2";
+// This file contains protocol buffers that are shared throughout HBase
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.shaded.protobuf.generated";
+option java_outer_classname = "ActiveClusterSuffixProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+/**
+ * Content of the '/hbase/active_cluster_suffix.id' file to indicate the active cluster.
+ */
+message ActiveClusterSuffix {
+ // This is the active cluster suffix set by the user in the config, as a String
+ required string active_cluster_suffix = 1;
+}
diff --git a/hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto b/hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto
index f475d26060d..c774a93605a 100644
--- a/hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto
@@ -807,6 +807,17 @@ message ModifyColumnStoreFileTrackerResponse {
optional uint64 proc_id = 1;
}
+message RefreshHFilesRequest {
+ optional TableName table_name = 1;
+ optional string namespace = 2;
+ optional uint64 nonce_group = 3 [default = 0];
+ optional uint64 nonce = 4 [default = 0];
+}
+
+message RefreshHFilesResponse {
+ optional uint64 proc_id = 1;
+}
+
message FlushMasterStoreRequest {}
message FlushMasterStoreResponse {}
@@ -819,6 +830,14 @@ message RollAllWALWritersResponse {
optional uint64 proc_id = 1;
}
+message RefreshMetaRequest {
+ optional uint64 nonce_group = 1 [default = 0];
+ optional uint64 nonce = 2 [default = 0];
+}
+message RefreshMetaResponse {
+ optional uint64 proc_id = 1;
+}
+
service MasterService {
/** Used by the client to get the number of regions that have received the updated schema */
rpc GetSchemaAlterStatus(GetSchemaAlterStatusRequest)
@@ -1303,6 +1322,12 @@ service MasterService {
rpc rollAllWALWriters(RollAllWALWritersRequest)
returns(RollAllWALWritersResponse);
+
+ rpc RefreshMeta(RefreshMetaRequest)
+ returns(RefreshMetaResponse);
+
+ rpc RefreshHFiles(RefreshHFilesRequest)
+ returns(RefreshHFilesResponse);
}
// HBCK Service definitions.
diff --git a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
index 7e6c6c8e2fc..56086aed29e 100644
--- a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
@@ -864,3 +864,34 @@ message LogRollRemoteProcedureResult {
optional ServerName server_name = 1;
optional uint64 last_highest_wal_filenum = 2;
}
+
+enum RefreshMetaState {
+ REFRESH_META_INIT = 1;
+ REFRESH_META_SCAN_STORAGE = 2;
+ REFRESH_META_PREPARE = 3;
+ REFRESH_META_APPLY = 4;
+ REFRESH_META_FOLLOWUP = 5;
+ REFRESH_META_FINISH = 6;
+}
+
+message RefreshMetaStateData {
+}
+
+enum RefreshHFilesTableProcedureState {
+ REFRESH_HFILES_PREPARE = 1;
+ REFRESH_HFILES_REFRESH_REGION = 2;
+ REFRESH_HFILES_FINISH = 3;
+}
+
+message RefreshHFilesTableProcedureStateData {
+ optional TableName table_name = 1;
+ optional string namespace_name = 2;
+}
+
+message RefreshHFilesRegionProcedureStateData {
+ required RegionInfo region = 1;
+}
+
+message RefreshHFilesRegionParameter {
+ required RegionInfo region = 1;
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index 05b049e27db..92d6a18486a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -753,7 +753,7 @@ public final class MetaTableAccessor {
* @param connection connection we're using
* @param deletes Deletes to add to hbase:meta This list should support #remove.
*/
- private static void deleteFromMetaTable(final Connection connection, final List<Delete> deletes)
+ public static void deleteFromMetaTable(final Connection connection, final List<Delete> deletes)
throws IOException {
try (Table t = getMetaHTable(connection)) {
debugLogMutations(deletes);
@@ -859,7 +859,7 @@ public final class MetaTableAccessor {
private static void updateTableState(Connection connection, TableState state) throws IOException {
Put put = makePutFromTableState(state, EnvironmentEdgeManager.currentTime());
putToMetaTable(connection, put);
- LOG.info("Updated {} in hbase:meta", state);
+ LOG.info("Updated {} in {}", state, TableName.META_TABLE_NAME);
}
/**
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
index d22e46383d3..32594ffce48 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
@@ -78,4 +78,11 @@ public interface TableDescriptors extends Closeable {
/** Returns Instance of table descriptor or null if none found. */
TableDescriptor remove(TableName tablename) throws IOException;
+
+ /**
+ * Invalidates the table descriptor cache.
+ */
+ default void invalidateTableDescriptorCache() {
+ // do nothing by default
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index 3d5897c0a05..625ac1dc584 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -36,6 +36,8 @@ import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.conf.ConfigurationManager;
+import org.apache.hadoop.hbase.conf.ConfigurationObserver;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
@@ -130,6 +132,39 @@ public abstract class CoprocessorHost<C extends Coprocessor, E extends Coprocess
return returnValue;
}
+ /**
+ * Used to help make the relevant loaded coprocessors dynamically configurable by registering them
+ * to the {@link ConfigurationManager}. Coprocessors are considered "relevant" if they implement
+ * the {@link ConfigurationObserver} interface.
+ * @param configurationManager the ConfigurationManager the coprocessors get registered to
+ */
+ public void registerConfigurationObservers(ConfigurationManager configurationManager) {
+ Coprocessor foundCp;
+ Set<String> coprocessors = this.getCoprocessors();
+ for (String cp : coprocessors) {
+ foundCp = this.findCoprocessor(cp);
+ if (foundCp instanceof ConfigurationObserver) {
+ configurationManager.registerObserver((ConfigurationObserver) foundCp);
+ }
+ }
+ }
+
+ /**
+ * Deregisters relevant coprocessors from the {@link ConfigurationManager}. Coprocessors are
+ * considered "relevant" if they implement the {@link ConfigurationObserver} interface.
+ * @param configurationManager the ConfigurationManager the coprocessors get deregistered from
+ */
+ public void deregisterConfigurationObservers(ConfigurationManager configurationManager) {
+ Coprocessor foundCp;
+ Set<String> coprocessors = this.getCoprocessors();
+ for (String cp : coprocessors) {
+ foundCp = this.findCoprocessor(cp);
+ if (foundCp instanceof ConfigurationObserver) {
+ configurationManager.deregisterObserver((ConfigurationObserver) foundCp);
+ }
+ }
+ }
+
/**
* Load system coprocessors once only. Read the class names from configuration. Called by
* constructor.
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java
index 61c98346887..b6950f4c2e8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.metrics.MetricRegistry;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
@@ -88,4 +89,6 @@ public interface MasterCoprocessorEnvironment extends CoprocessorEnvironment<Mas
* @return A MetricRegistry for the coprocessor class to track and export metrics.
*/
MetricRegistry getMetricRegistryForMaster();
+
+ MasterServices getMasterServices();
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
index fee132b7a4d..90985923b72 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
@@ -309,7 +309,13 @@ public enum EventType {
* RS log roll.<br>
* RS_LOG_ROLL
*/
- RS_LOG_ROLL(91, ExecutorType.RS_LOG_ROLL);
+ RS_LOG_ROLL(91, ExecutorType.RS_LOG_ROLL),
+
+ /**
+ * RS refresh hfiles for a region.<br>
+ * RS_REFRESH_HFILES
+ */
+ RS_REFRESH_HFILES(92, ExecutorType.RS_REFRESH_HFILES);
private final int code;
private final ExecutorType executor;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
index 668cd701c0d..e2d357fbee6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
@@ -57,7 +57,8 @@ public enum ExecutorType {
RS_SNAPSHOT_OPERATIONS(36),
RS_FLUSH_OPERATIONS(37),
RS_RELOAD_QUOTAS_OPERATIONS(38),
- RS_LOG_ROLL(39);
+ RS_LOG_ROLL(39),
+ RS_REFRESH_HFILES(39);
ExecutorType(int value) {
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index c997f1c6e82..69b6f94d6d9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -84,6 +84,7 @@ import org.apache.hadoop.hbase.InvalidFamilyOperationException;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.NamespaceNotFoundException;
import org.apache.hadoop.hbase.PleaseHoldException;
import org.apache.hadoop.hbase.PleaseRestartMasterException;
import org.apache.hadoop.hbase.RegionMetrics;
@@ -170,6 +171,8 @@ import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure;
import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
import org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher;
+import org.apache.hadoop.hbase.master.procedure.RefreshHFilesTableProcedure;
+import org.apache.hadoop.hbase.master.procedure.RefreshMetaProcedure;
import org.apache.hadoop.hbase.master.procedure.ReloadQuotasProcedure;
import org.apache.hadoop.hbase.master.procedure.ReopenTableRegionsProcedure;
import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
@@ -248,10 +251,12 @@ import org.apache.hadoop.hbase.security.AccessDeniedException;
import org.apache.hadoop.hbase.security.SecurityConstants;
import org.apache.hadoop.hbase.security.Superusers;
import org.apache.hadoop.hbase.security.UserProvider;
+import org.apache.hadoop.hbase.security.access.AbstractReadOnlyController;
import org.apache.hadoop.hbase.trace.TraceUtil;
import org.apache.hadoop.hbase.util.Addressing;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.hbase.util.ConfigurationUtil;
import org.apache.hadoop.hbase.util.CoprocessorConfigurationUtil;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
@@ -611,6 +616,12 @@ public class HMaster extends HBaseServerBase<MasterRpcServices> implements Maste
private void registerConfigurationObservers() {
configurationManager.registerObserver(this.rpcServices);
configurationManager.registerObserver(this);
+ if (cpHost != null) {
+ cpHost.registerConfigurationObservers(configurationManager);
+ } else {
+ LOG.warn("Could not register HMaster coprocessors to the ConfigurationManager because "
+ + "MasterCoprocessorHost is null");
+ }
}
// Main run loop. Calls through to the regionserver run loop AFTER becoming active Master; will
@@ -619,7 +630,6 @@ public class HMaster extends HBaseServerBase<MasterRpcServices> implements Maste
public void run() {
try {
installShutdownHook();
- registerConfigurationObservers();
Threads.setDaemonThreadRunning(new Thread(TraceUtil.tracedRunnable(() -> {
try {
int infoPort = putUpJettyServer();
@@ -1080,6 +1090,11 @@ public class HMaster extends HBaseServerBase<MasterRpcServices> implements Maste
if (!maintenanceMode) {
startupTaskGroup.addTask("Initializing master coprocessors");
setQuotasObserver(conf);
+ CoprocessorConfigurationUtil.syncReadOnlyConfigurations(
+ ConfigurationUtil.isReadOnlyModeEnabled(conf), conf,
+ CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);
+ AbstractReadOnlyController.manageActiveClusterIdFile(
+ ConfigurationUtil.isReadOnlyModeEnabled(conf), this.getMasterFileSystem());
this.cpHost = new MasterCoprocessorHost(this, conf);
} else {
// start an in process region server for carrying system regions
@@ -1182,8 +1197,9 @@ public class HMaster extends HBaseServerBase<MasterRpcServices> implements Maste
int existingReplicasCount =
assignmentManager.getRegionStates().getRegionsOfTable(TableName.META_TABLE_NAME).size();
if (existingReplicasCount > metaDesc.getRegionReplication()) {
- LOG.info("Update replica count of hbase:meta from {}(in TableDescriptor)"
- + " to {}(existing ZNodes)", metaDesc.getRegionReplication(), existingReplicasCount);
+ LOG.info(
+ "Update replica count of {} from {}(in TableDescriptor)" + " to {}(existing ZNodes)",
+ TableName.META_TABLE_NAME, metaDesc.getRegionReplication(), existingReplicasCount);
metaDesc = TableDescriptorBuilder.newBuilder(metaDesc)
.setRegionReplication(existingReplicasCount).build();
tableDescriptors.update(metaDesc);
@@ -1192,8 +1208,9 @@ public class HMaster extends HBaseServerBase<MasterRpcServices> implements Maste
if (metaDesc.getRegionReplication() != replicasNumInConf) {
LOG.info(
"The {} config is {} while the replica count in TableDescriptor is {}"
- + " for hbase:meta, altering...",
- HConstants.META_REPLICAS_NUM, replicasNumInConf, metaDesc.getRegionReplication());
+ + " for {}, altering...",
+ HConstants.META_REPLICAS_NUM, replicasNumInConf, metaDesc.getRegionReplication(),
+ TableName.META_TABLE_NAME);
procedureExecutor.submitProcedure(new ModifyTableProcedure(
procedureExecutor.getEnvironment(), TableDescriptorBuilder.newBuilder(metaDesc)
.setRegionReplication(replicasNumInConf).build(),
@@ -3124,8 +3141,8 @@ public class HMaster extends HBaseServerBase<MasterRpcServices> implements Maste
if (isActiveMaster() && isInitialized() && assignmentManager != null) {
try {
Map<TableName, RegionStatesCount> tableRegionStatesCountMap = new HashMap<>();
- Map<String, TableDescriptor> tableDescriptorMap = getTableDescriptors().getAll();
- for (TableDescriptor tableDescriptor : tableDescriptorMap.values()) {
+ List<TableDescriptor> tableDescriptors = listTableDescriptors(null, null, null, true);
+ for (TableDescriptor tableDescriptor : tableDescriptors) {
TableName tableName = tableDescriptor.getTableName();
RegionStatesCount regionStatesCount =
assignmentManager.getRegionStatesCount(tableName);
@@ -3829,9 +3846,9 @@ public class HMaster extends HBaseServerBase<MasterRpcServices> implements Maste
}
/**
- * Return a list of table table descriptors after applying any provided filter parameters. Note
- * that the user-facing description of this filter logic is presented on the class-level javadoc
- * of {@link NormalizeTableFilterParams}.
+ * Return a list of table descriptors after applying any provided filter parameters. Note that the
+ * user-facing description of this filter logic is presented on the class-level javadoc of
+ * {@link NormalizeTableFilterParams}.
*/
private List<TableDescriptor> getTableDescriptors(final List<TableDescriptor> htds,
final String namespace, final String regex, final List<TableName> tableNameList,
@@ -3840,7 +3857,7 @@ public class HMaster extends HBaseServerBase<MasterRpcServices> implements Maste
// request for all TableDescriptors
Collection<TableDescriptor> allHtds;
if (namespace != null && namespace.length() > 0) {
- // Do a check on the namespace existence. Will fail if does not exist.
+ // Do a check on the namespace existence. Will fail if it does not exist.
this.clusterSchemaService.getNamespace(namespace);
allHtds = tableDescriptors.getByNamespace(namespace).values();
} else {
@@ -4478,6 +4495,11 @@ public class HMaster extends HBaseServerBase<MasterRpcServices> implements Maste
}
// append the quotas observer back to the master coprocessor key
setQuotasObserver(newConf);
+
+ boolean readOnlyMode = ConfigurationUtil.isReadOnlyModeEnabled(newConf);
+ CoprocessorConfigurationUtil.syncReadOnlyConfigurations(readOnlyMode, newConf,
+ CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);
+
// update region server coprocessor if the configuration has changed.
if (
CoprocessorConfigurationUtil.checkConfigurationChange(this.cpHost, newConf,
@@ -4485,6 +4507,10 @@ public class HMaster extends HBaseServerBase<MasterRpcServices> implements Maste
) {
LOG.info("Update the master coprocessor(s) because the configuration has changed");
this.cpHost = new MasterCoprocessorHost(this, newConf);
+ CoprocessorConfigurationUtil.syncReadOnlyConfigurations(readOnlyMode, this.conf,
+ CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);
+ AbstractReadOnlyController.manageActiveClusterIdFile(
+ ConfigurationUtil.isReadOnlyModeEnabled(newConf), this.getMasterFileSystem());
}
}
@@ -4636,4 +4662,91 @@ public class HMaster extends HBaseServerBase<MasterRpcServices> implements Maste
return mobFileCleanerChore;
}
+ public Long refreshMeta(long nonceGroup, long nonce) throws IOException {
+ return MasterProcedureUtil
+ .submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
+ @Override
+ protected void run() throws IOException {
+ LOG.info("Submitting RefreshMetaProcedure");
+ submitProcedure(new RefreshMetaProcedure(procedureExecutor.getEnvironment()));
+ }
+
+ @Override
+ protected String getDescription() {
+ return "RefreshMetaProcedure";
+ }
+ });
+ }
+
+ public Long refreshHfiles(final TableName tableName, final long nonceGroup, final long nonce)
+ throws IOException {
+ checkInitialized();
+
+ if (!tableDescriptors.exists(tableName)) {
+ LOG.info("RefreshHfilesProcedure failed because table {} does not exist",
+ tableName.getNameAsString());
+ throw new TableNotFoundException(tableName);
+ }
+
+ return MasterProcedureUtil
+ .submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
+ @Override
+ protected void run() throws IOException {
+ LOG.info("Submitting RefreshHfilesTableProcedure for table {}",
+ tableName.getNameAsString());
+ submitProcedure(
+ new RefreshHFilesTableProcedure(procedureExecutor.getEnvironment(), tableName));
+ }
+
+ @Override
+ protected String getDescription() {
+ return "RefreshHfilesProcedure for a table";
+ }
+ });
+ }
+
+ public Long refreshHfiles(final String namespace, final long nonceGroup, final long nonce)
+ throws IOException {
+ checkInitialized();
+
+ try {
+ this.clusterSchemaService.getNamespace(namespace);
+ } catch (IOException e) {
+ LOG.info("RefreshHfilesProcedure failed because namespace {} does not exist", namespace);
+ throw new NamespaceNotFoundException(namespace);
+ }
+
+ return MasterProcedureUtil
+ .submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
+ @Override
+ protected void run() throws IOException {
+ LOG.info("Submitting RefreshHfilesProcedure for namespace {}", namespace);
+ submitProcedure(
+ new RefreshHFilesTableProcedure(procedureExecutor.getEnvironment(), namespace));
+ }
+
+ @Override
+ protected String getDescription() {
+ return "RefreshHfilesProcedure for namespace";
+ }
+ });
+ }
+
+ public Long refreshHfiles(final long nonceGroup, final long nonce) throws IOException {
+ checkInitialized();
+
+ return MasterProcedureUtil
+ .submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
+ @Override
+ protected void run() throws IOException {
+ LOG.info("Submitting RefreshHfilesProcedure for all tables");
+ submitProcedure(new RefreshHFilesTableProcedure(procedureExecutor.getEnvironment()));
+ }
+
+ @Override
+ protected String getDescription() {
+ return "RefreshHfilesProcedure for all tables";
+ }
+ });
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
index ac183026eec..00b3e714d1d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
@@ -112,6 +112,11 @@ public class MasterCoprocessorHost
return metricRegistry;
}
+ @Override
+ public MasterServices getMasterServices() {
+ return services;
+ }
+
@Override
public void shutdown() {
super.shutdown();
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 5a43cd98feb..1734f03a273 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -19,12 +19,15 @@ package org.apache.hadoop.hbase.master;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.ActiveClusterSuffix;
import org.apache.hadoop.hbase.ClusterId;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.backup.HFileArchiver;
@@ -60,6 +63,8 @@ public class MasterFileSystem {
private final Configuration conf;
// Persisted unique cluster ID
private ClusterId clusterId;
+ // Persisted unique Active Cluster Suffix
+ private ActiveClusterSuffix activeClusterSuffix;
// Keep around for convenience.
private final FileSystem fs;
// Keep around for convenience.
@@ -158,6 +163,8 @@ public class MasterFileSystem {
if (isSecurityEnabled) {
fs.setPermission(new Path(rootdir, HConstants.VERSION_FILE_NAME), secureRootFilePerms);
fs.setPermission(new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME), secureRootFilePerms);
+ fs.setPermission(new Path(rootdir, HConstants.ACTIVE_CLUSTER_SUFFIX_FILE_NAME),
+ secureRootFilePerms);
}
FsPermission currentRootPerms = fs.getFileStatus(this.rootdir).getPermission();
if (
@@ -262,10 +269,14 @@ public class MasterFileSystem {
throw iae;
}
// Make sure cluster ID exists
- if (!FSUtils.checkClusterIdExists(fs, rd, threadWakeFrequency)) {
- FSUtils.setClusterId(fs, rd, new ClusterId(), threadWakeFrequency);
+ if (
+ !FSUtils.checkFileExistsInHbaseRootDir(fs, rootdir, HConstants.CLUSTER_ID_FILE_NAME,
+ threadWakeFrequency)
+ ) {
+ FSUtils.setClusterId(fs, rootdir, new ClusterId(), threadWakeFrequency);
}
- clusterId = FSUtils.getClusterId(fs, rd);
+ clusterId = FSUtils.getClusterId(fs, rootdir);
+ negotiateActiveClusterSuffixFile(threadWakeFrequency);
}
/**
@@ -382,4 +393,63 @@ public class MasterFileSystem {
public void logFileSystemState(Logger log) throws IOException {
CommonFSUtils.logFileSystemState(fs, rootdir, log);
}
+
+ private void negotiateActiveClusterSuffixFile(long wait) throws IOException {
+ if (!isReadOnlyModeEnabled(conf)) {
+ try {
+ // verify the contents against the config set
+ ActiveClusterSuffix acs = FSUtils.getActiveClusterSuffix(fs, rootdir);
+ LOG.debug("File Suffix {} : Configured suffix {} : Cluster ID : {}", acs,
+ getSuffixFromConfig(), getClusterId());
+ if (Objects.equals(acs.getActiveClusterSuffix(), getSuffixFromConfig())) {
+ this.activeClusterSuffix = acs;
+ } else {
+ // throw error
+ LOG.info("rootdir {} : Active Cluster File Suffix {} ", rootdir, acs);
+ throw new IOException("Cannot start master, because another cluster is running in active "
+ + "(read-write) mode on this storage location. Active Cluster Id: {} " + acs
+ + " This cluster Id: " + getClusterId());
+ }
+ LOG.info(
+ "This is the active cluster on this storage location, " + "File Suffix {} : Suffix {} : ",
+ acs, getActiveClusterSuffix());
+ } catch (FileNotFoundException fnfe) {
+ // this is the active cluster, create active cluster suffix file if it does not exist
+ FSUtils.setActiveClusterSuffix(fs, rootdir, computeAndSetSuffixFileDataToWrite(), wait);
+ }
+ } else {
+ // this is a replica cluster
+ LOG.info("Replica cluster is being started in Read Only Mode");
+ }
+ }
+
+ public ActiveClusterSuffix getActiveClusterSuffix() {
+ return activeClusterSuffix;
+ }
+
+ private boolean isReadOnlyModeEnabled(Configuration conf) {
+ return conf.getBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY,
+ HConstants.HBASE_GLOBAL_READONLY_ENABLED_DEFAULT);
+ }
+
+ private String getActiveClusterSuffixFromConfig(Configuration conf) {
+ return conf.get(HConstants.HBASE_META_TABLE_SUFFIX,
+ HConstants.HBASE_META_TABLE_SUFFIX_DEFAULT_VALUE);
+ }
+
+ public String getSuffixFromConfig() {
+ return getClusterId().toString() + ":" + getActiveClusterSuffixFromConfig(conf);
+ }
+
+ // Used only for testing
+ public byte[] getSuffixFileDataToCompare() {
+ String str = this.activeClusterSuffix.toString();
+ return str.getBytes(StandardCharsets.UTF_8);
+ }
+
+ public byte[] computeAndSetSuffixFileDataToWrite() {
+ String str = getClusterId().toString() + ":" + getActiveClusterSuffixFromConfig(conf);
+ this.activeClusterSuffix = new ActiveClusterSuffix(str);
+ return str.getBytes(StandardCharsets.UTF_8);
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 85336eed2fb..a020378d7e2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -3742,4 +3742,35 @@ public class MasterRpcServices extends HBaseRpcServicesBase<HMaster>
throw new ServiceException(ioe);
}
}
+
+ @Override
+ public MasterProtos.RefreshMetaResponse refreshMeta(RpcController controller,
+ MasterProtos.RefreshMetaRequest request) throws ServiceException {
+ try {
+ Long procId = server.refreshMeta(request.getNonceGroup(), request.getNonce());
+ return MasterProtos.RefreshMetaResponse.newBuilder().setProcId(procId).build();
+ } catch (IOException ioe) {
+ throw new ServiceException(ioe);
+ }
+ }
+
+ @Override
+ public MasterProtos.RefreshHFilesResponse refreshHFiles(RpcController controller,
+ MasterProtos.RefreshHFilesRequest request) throws ServiceException {
+ try {
+ Long procId;
+ if (request.hasTableName()) {
+ procId = server.refreshHfiles(ProtobufUtil.toTableName(request.getTableName()),
+ request.getNonceGroup(), request.getNonce());
+ } else if (request.hasNamespace()) {
+ procId =
+ server.refreshHfiles(request.getNamespace(), request.getNonceGroup(), request.getNonce());
+ } else {
+ procId = server.refreshHfiles(request.getNonceGroup(), request.getNonce());
+ }
+ return MasterProtos.RefreshHFilesResponse.newBuilder().setProcId(procId).build();
+ } catch (IOException ioe) {
+ throw new ServiceException(ioe);
+ }
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java
index 854c21da2bc..69dfb040831 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java
@@ -605,7 +605,7 @@ public class RegionPlacementMaintainer implements Closeable {
*/
public void updateAssignmentPlanToMeta(FavoredNodesPlan plan) throws IOException {
try {
- LOG.info("Start to update the hbase:meta with the new assignment plan");
+ LOG.info("Started updating {} with the new assignment plan", TableName.META_TABLE_NAME);
Map<String, List<ServerName>> assignmentMap = plan.getAssignmentMap();
Map<RegionInfo, List<ServerName>> planToUpdate = new HashMap<>(assignmentMap.size());
Map<String, RegionInfo> regionToRegionInfoMap =
@@ -615,10 +615,10 @@ public class RegionPlacementMaintainer implements Closeable {
}
FavoredNodeAssignmentHelper.updateMetaWithFavoredNodesInfo(planToUpdate, conf);
- LOG.info("Updated the hbase:meta with the new assignment plan");
+ LOG.info("Updated {} with the new assignment plan", TableName.META_TABLE_NAME);
} catch (Exception e) {
- LOG.error(
- "Failed to update hbase:meta with the new assignment" + "plan because " + e.getMessage());
+ LOG.error("Failed to update {} with the new assignment plan because {}",
+ TableName.META_TABLE_NAME, e.getMessage());
}
}
@@ -690,14 +690,14 @@ public class RegionPlacementMaintainer implements Closeable {
}
public void updateAssignmentPlan(FavoredNodesPlan plan) throws IOException {
- LOG.info("Start to update the new assignment plan for the hbase:meta table and"
- + " the region servers");
+ LOG.info("Started updating the new assignment plan for {} and the region servers",
+ TableName.META_TABLE_NAME);
// Update the new assignment plan to META
updateAssignmentPlanToMeta(plan);
// Update the new assignment plan to Region Servers
updateAssignmentPlanToRegionServers(plan);
- LOG.info("Finish to update the new assignment plan for the hbase:meta table and"
- + " the region servers");
+ LOG.info("Finished updating the new assignment plan for {} and the region servers",
+ TableName.META_TABLE_NAME);
}
/**
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
index 7bfa79e2148..fbe89c3eb32 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
@@ -354,7 +354,7 @@ public class AssignmentManager {
if (RegionReplicaUtil.isDefaultReplica(regionInfo.getReplicaId())) {
setMetaAssigned(regionInfo, state == State.OPEN);
}
- LOG.debug("Loaded hbase:meta {}", regionNode);
+ LOG.debug("Loaded {} {}", TableName.META_TABLE_NAME, regionNode);
}, result);
}
}
@@ -1962,8 +1962,8 @@ public class AssignmentManager {
boolean meta = isMetaRegion(hri);
boolean metaLoaded = isMetaLoaded();
if (!meta && !metaLoaded) {
- throw new PleaseHoldException(
- "Master not fully online; hbase:meta=" + meta + ", metaLoaded=" + metaLoaded);
+ throw new PleaseHoldException("Master not fully online; " + TableName.META_TABLE_NAME + "="
+ + meta + ", metaLoaded=" + metaLoaded);
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
index c370fed9d9c..5fdd2893cca 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
@@ -718,8 +718,10 @@ public class MergeTableRegionsProcedure
RegionInfo.parseRegionName(p.getRow());
}
} catch (IOException e) {
- LOG.error("Row key of mutation from coprocessor is not parsable as region name. "
- + "Mutations from coprocessor should only be for hbase:meta table.", e);
+ LOG.error(
+ "Row key of mutation from coprocessor is not parsable as region name. "
+ + "Mutations from coprocessor should only be for {} table.",
+ TableName.META_TABLE_NAME, e);
throw e;
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java
index 5987fc7537b..6dac4c37c1b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java
@@ -169,9 +169,10 @@ public class RegionStateStore {
final long openSeqNum = hrl.getSeqNum();
LOG.debug(
- "Load hbase:meta entry region={}, regionState={}, lastHost={}, "
+ "Load {} entry region={}, regionState={}, lastHost={}, "
+ "regionLocation={}, openSeqNum={}",
- regionInfo.getEncodedName(), state, lastHost, regionLocation, openSeqNum);
+ TableName.META_TABLE_NAME, regionInfo.getEncodedName(), state, lastHost, regionLocation,
+ openSeqNum);
visitor.visitRegionState(result, regionInfo, state, regionLocation, lastHost, openSeqNum);
}
}
@@ -190,8 +191,8 @@ public class RegionStateStore {
final Put put = new Put(CatalogFamilyFormat.getMetaKeyForRegion(regionInfo), time);
MetaTableAccessor.addRegionInfo(put, regionInfo);
final StringBuilder info =
- new StringBuilder("pid=").append(pid).append(" updating hbase:meta row=")
- .append(regionInfo.getEncodedName()).append(", regionState=").append(state);
+ new StringBuilder("pid=").append(pid).append(" updating ").append(TableName.META_TABLE_NAME)
+ .append(" row=").append(regionInfo.getEncodedName()).append(", regionState=").append(state);
if (openSeqNum >= 0) {
Preconditions.checkArgument(state == State.OPEN && regionLocation != null,
"Open region should be on a server");
@@ -694,7 +695,7 @@ public class RegionStateStore {
return State.valueOf(state);
} catch (IllegalArgumentException e) {
LOG.warn(
- "BAD value {} in hbase:meta info:state column for region {} , "
+ "BAD value {} in " + TableName.META_TABLE_NAME + " info:state column for region {} , "
+ "Consider using HBCK2 setRegionState ENCODED_REGION_NAME STATE",
state, regionInfo.getEncodedName());
return null;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
index 3d3d3d18de2..c9d158d3fb4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
@@ -903,8 +903,10 @@ public class SplitTableRegionProcedure
RegionInfo.parseRegionName(p.getRow());
}
} catch (IOException e) {
- LOG.error("pid=" + getProcId() + " row key of mutation from coprocessor not parsable as "
- + "region name." + "Mutations from coprocessor should only for hbase:meta table.");
+ LOG.error(
+ "pid={} row key of mutation from coprocessor not parsable as region name. "
+ + "Mutations from coprocessor should only be for {} table.",
+ getProcId(), TableName.META_TABLE_NAME);
throw e;
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/hbck/HbckChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/hbck/HbckChore.java
index 75df2da5a71..34dc1ea81e2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/hbck/HbckChore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/hbck/HbckChore.java
@@ -261,7 +261,8 @@ public class HbckChore extends ScheduledChore {
FileSystem fs = master.getMasterFileSystem().getFileSystem();
int numRegions = 0;
- List<Path> tableDirs = FSUtils.getTableDirs(fs, rootDir);
+ List<Path> tableDirs =
+ FSUtils.getTableDirs(fs, rootDir).stream().filter(FSUtils::isLocalMetaTable).toList();
for (Path tableDir : tableDirs) {
List<Path> regionDirs = FSUtils.getRegionDirs(fs, tableDir);
for (Path regionDir : regionDirs) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java
index 0d3ddb43abd..14cf61ef970 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java
@@ -105,7 +105,7 @@ public class CatalogJanitor extends ScheduledChore {
scan();
}
} catch (IOException e) {
- LOG.warn("Failed initial janitorial scan of hbase:meta table", e);
+ LOG.warn("Failed initial janitorial scan of {} table", TableName.META_TABLE_NAME, e);
return false;
}
return true;
@@ -145,7 +145,7 @@ public class CatalogJanitor extends ScheduledChore {
+ this.services.getServerManager().isClusterShutdown());
}
} catch (IOException e) {
- LOG.warn("Failed janitorial scan of hbase:meta table", e);
+ LOG.warn("Failed janitorial scan of {} table", TableName.META_TABLE_NAME, e);
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/MetaFixer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/MetaFixer.java
index 1244d5bf352..3068a581f6f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/MetaFixer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/MetaFixer.java
@@ -203,19 +203,19 @@ public class MetaFixer {
.flatMap(List::stream).collect(Collectors.toList());
final List<IOException> createMetaEntriesFailures = addMetaEntriesResults.stream()
.filter(Either::hasRight).map(Either::getRight).collect(Collectors.toList());
- LOG.debug("Added {}/{} entries to hbase:meta", createMetaEntriesSuccesses.size(),
- newRegionInfos.size());
+ LOG.debug("Added {}/{} entries to {}", createMetaEntriesSuccesses.size(), newRegionInfos.size(),
+ TableName.META_TABLE_NAME);
if (!createMetaEntriesFailures.isEmpty()) {
LOG.warn(
- "Failed to create entries in hbase:meta for {}/{} RegionInfo descriptors. First"
+ "Failed to create entries in {} for {}/{} RegionInfo descriptors. First"
+ " failure message included; full list of failures with accompanying stack traces is"
+ " available at log level DEBUG. message={}",
- createMetaEntriesFailures.size(), addMetaEntriesResults.size(),
+ TableName.META_TABLE_NAME, createMetaEntriesFailures.size(), addMetaEntriesResults.size(),
createMetaEntriesFailures.get(0).getMessage());
if (LOG.isDebugEnabled()) {
- createMetaEntriesFailures
- .forEach(ioe -> LOG.debug("Attempt to fix region hole in hbase:meta failed.", ioe));
+ createMetaEntriesFailures.forEach(ioe -> LOG
+ .debug("Attempt to fix region hole in {} failed.", TableName.META_TABLE_NAME, ioe));
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/ReportMakingVisitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/ReportMakingVisitor.java
index c712f1cba67..b194bca221c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/ReportMakingVisitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/ReportMakingVisitor.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.RegionLocations;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.client.Result;
@@ -137,8 +138,9 @@ class ReportMakingVisitor implements ClientMetaTableAccessor.CloseableVisitor {
if (!Bytes.equals(metaTableRow.getRow(), ri.getRegionName())) {
LOG.warn(
"INCONSISTENCY: Row name is not equal to serialized info:regioninfo content; "
- + "row={} {}; See if RegionInfo is referenced in another hbase:meta row? Delete?",
- Bytes.toStringBinary(metaTableRow.getRow()), ri.getRegionNameAsString());
+ + "row={} {}; See if RegionInfo is referenced in another {} row? Delete?",
+ Bytes.toStringBinary(metaTableRow.getRow()), ri.getRegionNameAsString(),
+ TableName.META_TABLE_NAME);
return null;
}
// Skip split parent region
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/HBCKServerCrashProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/HBCKServerCrashProcedure.java
index 43d69361c2d..e14579ae668 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/HBCKServerCrashProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/HBCKServerCrashProcedure.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.RegionLocations;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.Result;
@@ -102,14 +103,14 @@ public class HBCKServerCrashProcedure extends ServerCrashProcedure {
MetaTableAccessor.scanMetaForTableRegions(env.getMasterServices().getConnection(), visitor,
null);
} catch (IOException ioe) {
- LOG.warn("Failed scan of hbase:meta for 'Unknown Servers'", ioe);
+ LOG.warn("Failed scan of {} for 'Unknown Servers'", TableName.META_TABLE_NAME, ioe);
return ris;
}
// create the server state node too
env.getAssignmentManager().getRegionStates().createServer(getServerName());
- LOG.info("Found {} mentions of {} in hbase:meta of OPEN/OPENING Regions: {}",
- visitor.getReassigns().size(), getServerName(), visitor.getReassigns().stream()
- .map(RegionInfo::getEncodedName).collect(Collectors.joining(",")));
+ LOG.info("Found {} mentions of {} in {} of OPEN/OPENING Regions: {}",
+ visitor.getReassigns().size(), getServerName(), TableName.META_TABLE_NAME, visitor
+ .getReassigns().stream().map(RegionInfo::getEncodedName).collect(Collectors.joining(",")));
return visitor.getReassigns();
}
@@ -150,8 +151,8 @@ public class HBCKServerCrashProcedure extends ServerCrashProcedure {
RegionState rs = new RegionState(hrl.getRegion(), state, hrl.getServerName());
if (rs.isClosing()) {
// Move region to CLOSED in hbase:meta.
- LOG.info("Moving {} from CLOSING to CLOSED in hbase:meta",
- hrl.getRegion().getRegionNameAsString());
+ LOG.info("Moving {} from CLOSING to CLOSED in {}",
+ hrl.getRegion().getRegionNameAsString(), TableName.META_TABLE_NAME);
try {
MetaTableAccessor.updateRegionState(this.connection, hrl.getRegion(),
RegionState.State.CLOSED);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/InitMetaProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/InitMetaProcedure.java
index 2d54eaf6c58..0bc4ddee899 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/InitMetaProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/InitMetaProcedure.java
@@ -77,7 +77,7 @@ public class InitMetaProcedure extends AbstractStateMachineTableProcedure<InitMe
private static TableDescriptor writeFsLayout(Path rootDir, MasterProcedureEnv env)
throws IOException {
- LOG.info("BOOTSTRAP: creating hbase:meta region");
+ LOG.info("BOOTSTRAP: creating {} region", TableName.META_TABLE_NAME);
FileSystem fs = rootDir.getFileSystem(env.getMasterConfiguration());
Path tableDir = CommonFSUtils.getTableDir(rootDir, TableName.META_TABLE_NAME);
if (fs.exists(tableDir) && !deleteMetaTableDirectoryIfPartial(fs, tableDir)) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java
index 3450f305910..b40ad310293 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java
@@ -110,7 +110,7 @@ public class ModifyTableProcedure extends AbstractStateMachineTableProcedure<Mod
for (byte[] family : UNDELETABLE_META_COLUMNFAMILIES) {
if (!cfs.contains(family)) {
throw new HBaseIOException(
- "Delete of hbase:meta column family " + Bytes.toString(family));
+ "Delete of " + TableName.META_TABLE_NAME + " column family " + Bytes.toString(family));
}
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RefreshHFilesRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RefreshHFilesRegionProcedure.java
new file mode 100644
index 00000000000..6f6e0fb46f5
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RefreshHFilesRegionProcedure.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.procedure;
+
+import java.io.IOException;
+import java.util.Optional;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.master.RegionState;
+import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
+import org.apache.hadoop.hbase.master.assignment.RegionStates;
+import org.apache.hadoop.hbase.procedure2.FailedRemoteDispatchException;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureEvent;
+import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
+import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
+import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
+import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
+import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher;
+import org.apache.hadoop.hbase.procedure2.RemoteProcedureException;
+import org.apache.hadoop.hbase.regionserver.RefreshHFilesCallable;
+import org.apache.hadoop.hbase.util.RetryCounter;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
+
+/**
+ * A master-side procedure that handles refreshing HFiles (store files) for a specific region in
+ * HBase. It performs remote procedure dispatch to the RegionServer hosting the region and manages
+ * retries, suspensions, and timeouts as needed. This procedure ensures safe execution by verifying
+ * the region state, handling remote operation results, and applying retry mechanisms in case of
+ * failures. It gives the call to {@link RefreshHFilesCallable} which gets executed on region
+ * server.
+ */
+
+@InterfaceAudience.Private
+public class RefreshHFilesRegionProcedure extends Procedure<MasterProcedureEnv>
+ implements TableProcedureInterface,
+ RemoteProcedureDispatcher.RemoteProcedure<MasterProcedureEnv, ServerName> {
+ private static final Logger LOG = LoggerFactory.getLogger(RefreshHFilesRegionProcedure.class);
+ private RegionInfo region;
+ private ProcedureEvent<?> event;
+ private boolean dispatched;
+ private boolean succ;
+ private RetryCounter retryCounter;
+
+ public RefreshHFilesRegionProcedure() {
+ }
+
+ public RefreshHFilesRegionProcedure(RegionInfo region) {
+ this.region = region;
+ }
+
+ @Override
+ protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ MasterProcedureProtos.RefreshHFilesRegionProcedureStateData data =
+ serializer.deserialize(MasterProcedureProtos.RefreshHFilesRegionProcedureStateData.class);
+ this.region = ProtobufUtil.toRegionInfo(data.getRegion());
+ }
+
+ @Override
+ protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ MasterProcedureProtos.RefreshHFilesRegionProcedureStateData.Builder builder =
+ MasterProcedureProtos.RefreshHFilesRegionProcedureStateData.newBuilder();
+ builder.setRegion(ProtobufUtil.toRegionInfo(region));
+ serializer.serialize(builder.build());
+ }
+
+ @Override
+ protected boolean abort(MasterProcedureEnv env) {
+ return false;
+ }
+
+ @Override
+ protected void rollback(MasterProcedureEnv env) throws IOException, InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ private void setTimeoutForSuspend(MasterProcedureEnv env, String reason) {
+ if (retryCounter == null) {
+ retryCounter = ProcedureUtil.createRetryCounter(env.getMasterConfiguration());
+ }
+ long backoff = retryCounter.getBackoffTimeAndIncrementAttempts();
+ LOG.warn("{} can not run currently because {}, wait {} ms to retry", this, reason, backoff);
+ setTimeout(Math.toIntExact(backoff));
+ setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT);
+ skipPersistence();
+ }
+
+ @Override
+ protected Procedure<MasterProcedureEnv>[] execute(MasterProcedureEnv env)
+ throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {
+ if (dispatched) {
+ if (succ) {
+ return null;
+ }
+ dispatched = false;
+ }
+
+ RegionStates regionStates = env.getAssignmentManager().getRegionStates();
+ RegionStateNode regionNode = regionStates.getRegionStateNode(region);
+
+ if (regionNode.getProcedure() != null) {
+ setTimeoutForSuspend(env, String.format("region %s has a TRSP attached %s",
+ region.getRegionNameAsString(), regionNode.getProcedure()));
+ throw new ProcedureSuspendedException();
+ }
+
+ if (!regionNode.isInState(RegionState.State.OPEN)) {
+ LOG.warn("State of region {} is not OPEN. Skip {} ...", region, this);
+ setTimeoutForSuspend(env, String.format("region state of %s is %s",
+ region.getRegionNameAsString(), regionNode.getState()));
+ throw new ProcedureSuspendedException();
+ }
+
+ ServerName targetServer = regionNode.getRegionLocation();
+ if (targetServer == null) {
+ setTimeoutForSuspend(env,
+ String.format("target server of region %s is null", region.getRegionNameAsString()));
+ throw new ProcedureSuspendedException();
+ }
+
+ try {
+ env.getRemoteDispatcher().addOperationToNode(targetServer, this);
+ dispatched = true;
+ event = new ProcedureEvent<>(this);
+ event.suspendIfNotReady(this);
+ throw new ProcedureSuspendedException();
+ } catch (FailedRemoteDispatchException e) {
+ setTimeoutForSuspend(env, "Failed send request to " + targetServer);
+ throw new ProcedureSuspendedException();
+ }
+ }
+
+ @Override
+ public TableOperationType getTableOperationType() {
+ return TableOperationType.REFRESH_HFILES;
+ }
+
+ @Override
+ public TableName getTableName() {
+ return region.getTable();
+ }
+
+ @Override
+ public void remoteOperationFailed(MasterProcedureEnv env, RemoteProcedureException error) {
+ complete(env, error);
+ }
+
+ @Override
+ public void remoteOperationCompleted(MasterProcedureEnv env, byte[] remoteResultData) {
+ complete(env, null);
+ }
+
+ @Override
+ public void remoteCallFailed(MasterProcedureEnv env, ServerName serverName, IOException e) {
+ complete(env, e);
+ }
+
+ private void complete(MasterProcedureEnv env, Throwable error) {
+ if (isFinished()) {
+ LOG.info("This procedure {} is already finished. Skip the rest of the processes",
+ this.getProcId());
+ return;
+ }
+ if (event == null) {
+ LOG.warn("procedure event for {} is null, maybe the procedure is created when recovery",
+ getProcId());
+ return;
+ }
+ if (error == null) {
+ succ = true;
+ }
+ event.wake(env.getProcedureScheduler());
+ event = null;
+ }
+
+ @Override
+ public Optional<RemoteProcedureDispatcher.RemoteOperation> remoteCallBuild(MasterProcedureEnv env,
+ ServerName serverName) {
+ MasterProcedureProtos.RefreshHFilesRegionParameter.Builder builder =
+ MasterProcedureProtos.RefreshHFilesRegionParameter.newBuilder();
+ builder.setRegion(ProtobufUtil.toRegionInfo(region));
+ return Optional
+ .of(new RSProcedureDispatcher.ServerOperation(this, getProcId(), RefreshHFilesCallable.class,
+ builder.build().toByteArray(), env.getMasterServices().getMasterActiveTime()));
+ }
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RefreshHFilesTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RefreshHFilesTableProcedure.java
new file mode 100644
index 00000000000..cb225d23b0d
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RefreshHFilesTableProcedure.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.procedure;
+
+import java.io.IOException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RefreshHFilesTableProcedureState;
+
+@InterfaceAudience.Private
+public class RefreshHFilesTableProcedure
+ extends AbstractStateMachineTableProcedure<RefreshHFilesTableProcedureState> {
+ private static final Logger LOG = LoggerFactory.getLogger(RefreshHFilesTableProcedure.class);
+
+ private TableName tableName;
+ private String namespaceName;
+
+ public RefreshHFilesTableProcedure() {
+ super();
+ }
+
+ public RefreshHFilesTableProcedure(MasterProcedureEnv env) {
+ super(env);
+ }
+
+ public RefreshHFilesTableProcedure(MasterProcedureEnv env, TableName tableName) {
+ super(env);
+ this.tableName = tableName;
+ }
+
+ public RefreshHFilesTableProcedure(MasterProcedureEnv env, String namespaceName) {
+ super(env);
+ this.namespaceName = namespaceName;
+ }
+
+ @Override
+ public TableOperationType getTableOperationType() {
+ return TableOperationType.REFRESH_HFILES;
+ }
+
+ @Override
+ protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ super.serializeStateData(serializer);
+ MasterProcedureProtos.RefreshHFilesTableProcedureStateData.Builder builder =
+ MasterProcedureProtos.RefreshHFilesTableProcedureStateData.newBuilder();
+ if (tableName != null && namespaceName == null) {
+ builder.setTableName(ProtobufUtil.toProtoTableName(tableName));
+ } else if (tableName == null && namespaceName != null) {
+ builder.setNamespaceName(namespaceName);
+ }
+ serializer.serialize(builder.build());
+ }
+
+ @Override
+ protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ super.deserializeStateData(serializer);
+ MasterProcedureProtos.RefreshHFilesTableProcedureStateData data =
+ serializer.deserialize(MasterProcedureProtos.RefreshHFilesTableProcedureStateData.class);
+ if (data.hasTableName() && !data.hasNamespaceName()) {
+ this.tableName = ProtobufUtil.toTableName(data.getTableName());
+ } else if (!data.hasTableName() && data.hasNamespaceName()) {
+ this.namespaceName = data.getNamespaceName();
+ }
+ }
+
+ @Override
+ public TableName getTableName() {
+ if (tableName != null && namespaceName == null) {
+ return tableName;
+ }
+ return DUMMY_NAMESPACE_TABLE_NAME;
+ }
+
+ @Override
+ protected RefreshHFilesTableProcedureState getInitialState() {
+ return RefreshHFilesTableProcedureState.REFRESH_HFILES_PREPARE;
+ }
+
+ @Override
+ protected int getStateId(RefreshHFilesTableProcedureState state) {
+ return state.getNumber();
+ }
+
+ @Override
+ protected RefreshHFilesTableProcedureState getState(int stateId) {
+ return RefreshHFilesTableProcedureState.forNumber(stateId);
+ }
+
+ @Override
+ protected void rollbackState(MasterProcedureEnv env, RefreshHFilesTableProcedureState state)
+ throws IOException, InterruptedException {
+ // Refresh HFiles is idempotent operation hence rollback is not needed
+ LOG.trace("Rollback not implemented for RefreshHFilesTableProcedure state: {}", state);
+ }
+
+ @Override
+ protected Flow executeFromState(MasterProcedureEnv env, RefreshHFilesTableProcedureState state) {
+ LOG.info("Executing RefreshHFilesTableProcedureState state: {}", state);
+
+ try {
+ return switch (state) {
+ case REFRESH_HFILES_PREPARE -> prepare(env);
+ case REFRESH_HFILES_REFRESH_REGION -> refreshHFiles(env);
+ case REFRESH_HFILES_FINISH -> finish();
+ default -> throw new UnsupportedOperationException("Unhandled state: " + state);
+ };
+ } catch (Exception ex) {
+ LOG.error("Error in RefreshHFilesTableProcedure state {}", state, ex);
+ setFailure("RefreshHFilesTableProcedure", ex);
+ return Flow.NO_MORE_STATE;
+ }
+ }
+
+ private Flow prepare(final MasterProcedureEnv env) {
+ setNextState(RefreshHFilesTableProcedureState.REFRESH_HFILES_REFRESH_REGION);
+ return Flow.HAS_MORE_STATE;
+ }
+
+ private void refreshHFilesForTable(final MasterProcedureEnv env, TableName tableName) {
+ addChildProcedure(env.getAssignmentManager().getTableRegions(tableName, true).stream()
+ .map(RefreshHFilesRegionProcedure::new).toArray(RefreshHFilesRegionProcedure[]::new));
+ }
+
+ private Flow refreshHFiles(final MasterProcedureEnv env) throws IOException {
+ if (tableName != null && namespaceName == null) {
+ refreshHFilesForTable(env, tableName);
+ } else if (tableName == null && namespaceName != null) {
+ env.getMasterServices().listTableNamesByNamespace(namespaceName)
+ .forEach(table -> refreshHFilesForTable(env, table));
+ } else {
+ env.getMasterServices().getTableDescriptors().getAll().values().stream()
+ .map(TableDescriptor::getTableName).filter(table -> !table.isSystemTable())
+ .forEach(table -> refreshHFilesForTable(env, table));
+ }
+
+ setNextState(RefreshHFilesTableProcedureState.REFRESH_HFILES_FINISH);
+ return Flow.HAS_MORE_STATE;
+ }
+
+ private Flow finish() {
+ return Flow.NO_MORE_STATE;
+ }
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RefreshMetaProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RefreshMetaProcedure.java
new file mode 100644
index 00000000000..4efd5769902
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RefreshMetaProcedure.java
@@ -0,0 +1,480 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.procedure;
+
+import static org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState.WAITING_TIMEOUT;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MetaTableAccessor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.TableState;
+import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
+import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
+import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.RetryCounter;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RefreshMetaState;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RefreshMetaStateData;
+
+@InterfaceAudience.Private
+public class RefreshMetaProcedure extends AbstractStateMachineTableProcedure<RefreshMetaState> {
+ private static final Logger LOG = LoggerFactory.getLogger(RefreshMetaProcedure.class);
+ private static final String HIDDEN_DIR_PATTERN = "^[._-].*";
+
+ private List<RegionInfo> currentRegions;
+ private List<RegionInfo> latestRegions;
+ private List<Mutation> pendingMutations;
+ private RetryCounter retryCounter;
+ private static final int MUTATION_BATCH_SIZE = 100;
+ private List<RegionInfo> newlyAddedRegions;
+ private List<TableName> deletedTables;
+
+ public RefreshMetaProcedure() {
+ super();
+ }
+
+ public RefreshMetaProcedure(MasterProcedureEnv env) {
+ super(env);
+ }
+
+ @Override
+ public TableName getTableName() {
+ return TableName.META_TABLE_NAME;
+ }
+
+ @Override
+ public TableOperationType getTableOperationType() {
+ return TableOperationType.EDIT;
+ }
+
+ @Override
+ protected Flow executeFromState(MasterProcedureEnv env, RefreshMetaState refreshMetaState) {
+ LOG.info("Executing RefreshMetaProcedure state: {}", refreshMetaState);
+
+ try {
+ return switch (refreshMetaState) {
+ case REFRESH_META_INIT -> executeInit(env);
+ case REFRESH_META_SCAN_STORAGE -> executeScanStorage(env);
+ case REFRESH_META_PREPARE -> executePrepare();
+ case REFRESH_META_APPLY -> executeApply(env);
+ case REFRESH_META_FOLLOWUP -> executeFollowup(env);
+ case REFRESH_META_FINISH -> executeFinish(env);
+ default -> throw new UnsupportedOperationException("Unhandled state: " + refreshMetaState);
+ };
+ } catch (Exception ex) {
+ LOG.error("Error in RefreshMetaProcedure state {}", refreshMetaState, ex);
+ setFailure("RefreshMetaProcedure", ex);
+ return Flow.NO_MORE_STATE;
+ }
+ }
+
+ private Flow executeInit(MasterProcedureEnv env) throws IOException {
+ LOG.trace("Getting current regions from {} table", TableName.META_TABLE_NAME);
+ try {
+ currentRegions = getCurrentRegions(env.getMasterServices().getConnection());
+ LOG.info("Found {} current regions in meta table", currentRegions.size());
+ setNextState(RefreshMetaState.REFRESH_META_SCAN_STORAGE);
+ return Flow.HAS_MORE_STATE;
+ } catch (IOException ioe) {
+ LOG.error("Failed to get current regions from meta table", ioe);
+ throw ioe;
+ }
+ }
+
+ private Flow executeScanStorage(MasterProcedureEnv env) throws IOException {
+ try {
+ latestRegions = scanBackingStorage(env.getMasterServices().getConnection());
+ LOG.info("Found {} regions in backing storage", latestRegions.size());
+ setNextState(RefreshMetaState.REFRESH_META_PREPARE);
+ return Flow.HAS_MORE_STATE;
+ } catch (IOException ioe) {
+ LOG.error("Failed to scan backing storage", ioe);
+ throw ioe;
+ }
+ }
+
+ private Flow executePrepare() throws IOException {
+ if (currentRegions == null || latestRegions == null) {
+ LOG.error(
+ "Can not execute update on null lists. " + "Meta Table Regions - {}, Storage Regions - {}",
+ currentRegions, latestRegions);
+ throw new IOException(
+ (currentRegions == null ? "current regions" : "latest regions") + " list is null");
+ }
+ LOG.info("Comparing regions. Current regions: {}, Latest regions: {}", currentRegions.size(),
+ latestRegions.size());
+
+ this.newlyAddedRegions = new ArrayList<>();
+ this.deletedTables = new ArrayList<>();
+
+ pendingMutations = prepareMutations(
+ currentRegions.stream()
+ .collect(Collectors.toMap(RegionInfo::getEncodedName, Function.identity())),
+ latestRegions.stream()
+ .collect(Collectors.toMap(RegionInfo::getEncodedName, Function.identity())));
+
+ if (pendingMutations.isEmpty()) {
+ LOG.info("RefreshMetaProcedure completed, No update needed.");
+ setNextState(RefreshMetaState.REFRESH_META_FINISH);
+ } else {
+ LOG.info("Prepared {} region mutations and {} tables for cleanup.", pendingMutations.size(),
+ deletedTables.size());
+ setNextState(RefreshMetaState.REFRESH_META_APPLY);
+ }
+ return Flow.HAS_MORE_STATE;
+ }
+
+ private Flow executeApply(MasterProcedureEnv env) throws ProcedureSuspendedException {
+ try {
+ if (pendingMutations != null && !pendingMutations.isEmpty()) {
+ applyMutations(env.getMasterServices().getConnection(), pendingMutations);
+ LOG.debug("RefreshMetaProcedure applied {} mutations to meta table",
+ pendingMutations.size());
+ }
+ } catch (IOException ioe) {
+ if (retryCounter == null) {
+ retryCounter = ProcedureUtil.createRetryCounter(env.getMasterConfiguration());
+ }
+ long backoff = retryCounter.getBackoffTimeAndIncrementAttempts();
+ LOG.warn("Failed to apply mutations to meta table, suspending for {} ms", backoff, ioe);
+ setTimeout(Math.toIntExact(backoff));
+ setState(WAITING_TIMEOUT);
+ skipPersistence();
+ throw new ProcedureSuspendedException();
+ }
+
+ if (
+ (this.newlyAddedRegions != null && !this.newlyAddedRegions.isEmpty())
+ || (this.deletedTables != null && !this.deletedTables.isEmpty())
+ ) {
+ setNextState(RefreshMetaState.REFRESH_META_FOLLOWUP);
+ } else {
+ LOG.info("RefreshMetaProcedure completed. No follow-up actions were required.");
+ setNextState(RefreshMetaState.REFRESH_META_FINISH);
+ }
+ return Flow.HAS_MORE_STATE;
+ }
+
+ private Flow executeFollowup(MasterProcedureEnv env) throws IOException {
+
+ LOG.info("Submitting assignment for new regions: {}", this.newlyAddedRegions);
+ addChildProcedure(env.getAssignmentManager().createAssignProcedures(newlyAddedRegions));
+
+ for (TableName tableName : this.deletedTables) {
+ LOG.debug("Submitting deletion for empty table {}", tableName);
+ env.getMasterServices().getAssignmentManager().deleteTable(tableName);
+ env.getMasterServices().getTableStateManager().setDeletedTable(tableName);
+ env.getMasterServices().getTableDescriptors().remove(tableName);
+ }
+ setNextState(RefreshMetaState.REFRESH_META_FINISH);
+ return Flow.HAS_MORE_STATE;
+ }
+
+ private Flow executeFinish(MasterProcedureEnv env) {
+ invalidateTableDescriptorCache(env);
+ LOG.info("RefreshMetaProcedure completed successfully. All follow-up actions finished.");
+ currentRegions = null;
+ latestRegions = null;
+ pendingMutations = null;
+ deletedTables = null;
+ newlyAddedRegions = null;
+ return Flow.NO_MORE_STATE;
+ }
+
+ private void invalidateTableDescriptorCache(MasterProcedureEnv env) {
+ LOG.debug("Invalidating the table descriptor cache to ensure new tables are discovered");
+ env.getMasterServices().getTableDescriptors().invalidateTableDescriptorCache();
+ }
+
+ /**
+ * Prepares mutations by comparing the current regions in hbase:meta with the latest regions from
+ * backing storage. Also populates newlyAddedRegions and deletedTables lists for follow-up
+ * actions.
+ * @param currentMap Current regions from hbase:meta
+ * @param latestMap Latest regions from backing storage
+ * @return List of mutations to apply to the meta table
+ * @throws IOException If there is an error creating mutations
+ */
+ private List<Mutation> prepareMutations(Map<String, RegionInfo> currentMap,
+ Map<String, RegionInfo> latestMap) throws IOException {
+ List<Mutation> mutations = new ArrayList<>();
+
+ for (String regionId : Stream.concat(currentMap.keySet().stream(), latestMap.keySet().stream())
+ .collect(Collectors.toSet())) {
+ RegionInfo currentRegion = currentMap.get(regionId);
+ RegionInfo latestRegion = latestMap.get(regionId);
+
+ if (latestRegion != null) {
+ if (currentRegion == null || hasBoundaryChanged(currentRegion, latestRegion)) {
+ mutations.add(MetaTableAccessor.makePutFromRegionInfo(latestRegion));
+ newlyAddedRegions.add(latestRegion);
+ }
+ } else {
+ mutations.add(MetaTableAccessor.makeDeleteFromRegionInfo(currentRegion,
+ EnvironmentEdgeManager.currentTime()));
+ }
+ }
+
+ if (!currentMap.isEmpty() || !latestMap.isEmpty()) {
+ Set<TableName> currentTables =
+ currentMap.values().stream().map(RegionInfo::getTable).collect(Collectors.toSet());
+ Set<TableName> latestTables =
+ latestMap.values().stream().map(RegionInfo::getTable).collect(Collectors.toSet());
+
+ Set<TableName> tablesToDeleteState = new HashSet<>(currentTables);
+ tablesToDeleteState.removeAll(latestTables);
+ if (!tablesToDeleteState.isEmpty()) {
+ LOG.warn(
+ "The following tables have no regions on storage and WILL BE REMOVED from the meta: {}",
+ tablesToDeleteState);
+ this.deletedTables.addAll(tablesToDeleteState);
+ }
+
+ Set<TableName> tablesToRestoreState = new HashSet<>(latestTables);
+ tablesToRestoreState.removeAll(currentTables);
+ if (!tablesToRestoreState.isEmpty()) {
+ LOG.info("Adding missing table:state entry for recovered tables: {}", tablesToRestoreState);
+ for (TableName tableName : tablesToRestoreState) {
+ TableState tableState = new TableState(tableName, TableState.State.ENABLED);
+ mutations.add(MetaTableAccessor.makePutFromTableState(tableState,
+ EnvironmentEdgeManager.currentTime()));
+ }
+ }
+ }
+ return mutations;
+ }
+
+ private void applyMutations(Connection connection, List<Mutation> mutations) throws IOException {
+ List<List<Mutation>> chunks = Lists.partition(mutations, MUTATION_BATCH_SIZE);
+
+ for (int i = 0; i < chunks.size(); i++) {
+ List<Mutation> chunk = chunks.get(i);
+
+ List<Put> puts =
+ chunk.stream().filter(m -> m instanceof Put).map(m -> (Put) m).collect(Collectors.toList());
+
+ List<Delete> deletes = chunk.stream().filter(m -> m instanceof Delete).map(m -> (Delete) m)
+ .collect(Collectors.toList());
+
+ if (!puts.isEmpty()) {
+ MetaTableAccessor.putsToMetaTable(connection, puts);
+ }
+ if (!deletes.isEmpty()) {
+ MetaTableAccessor.deleteFromMetaTable(connection, deletes);
+ }
+ LOG.debug("Successfully processed batch {}/{}", i + 1, chunks.size());
+ }
+ }
+
+ boolean hasBoundaryChanged(RegionInfo region1, RegionInfo region2) {
+ return !Arrays.equals(region1.getStartKey(), region2.getStartKey())
+ || !Arrays.equals(region1.getEndKey(), region2.getEndKey());
+ }
+
+ /**
+ * Scans the backing storage for all regions and returns a list of RegionInfo objects. This method
+ * scans the filesystem for region directories and reads their .regioninfo files.
+ * @param connection The HBase connection to use.
+ * @return List of RegionInfo objects found in the backing storage.
+ * @throws IOException If there is an error accessing the filesystem or reading region info files.
+ */
+ List<RegionInfo> scanBackingStorage(Connection connection) throws IOException {
+ List<RegionInfo> regions = new ArrayList<>();
+ Configuration conf = connection.getConfiguration();
+ FileSystem fs = FileSystem.get(conf);
+ Path rootDir = CommonFSUtils.getRootDir(conf);
+ Path dataDir = new Path(rootDir, HConstants.BASE_NAMESPACE_DIR);
+
+ LOG.info("Scanning backing storage under: {}", dataDir);
+
+ if (!fs.exists(dataDir)) {
+ LOG.warn("Data directory does not exist: {}", dataDir);
+ return regions;
+ }
+
+ FileStatus[] namespaceDirs =
+ fs.listStatus(dataDir, path -> !path.getName().matches(HIDDEN_DIR_PATTERN));
+ LOG.debug("Found {} namespace directories in data dir", Arrays.stream(namespaceDirs).toList());
+
+ for (FileStatus nsDir : namespaceDirs) {
+ String namespaceName = nsDir.getPath().getName();
+ if (NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR.equals(namespaceName)) {
+ LOG.info("Skipping system namespace {}", namespaceName);
+ continue;
+ }
+ try {
+ List<RegionInfo> namespaceRegions = scanTablesInNamespace(fs, nsDir.getPath());
+ regions.addAll(namespaceRegions);
+ LOG.debug("Found {} regions in namespace {}", namespaceRegions.size(),
+ nsDir.getPath().getName());
+ } catch (IOException e) {
+ LOG.error("Failed to scan namespace directory: {}", nsDir.getPath(), e);
+ }
+ }
+ LOG.info("Scanned backing storage and found {} regions", regions.size());
+ return regions;
+ }
+
+ private List<RegionInfo> scanTablesInNamespace(FileSystem fs, Path namespacePath)
+ throws IOException {
+ LOG.debug("Scanning namespace {}", namespacePath.getName());
+ List<Path> tableDirs = FSUtils.getLocalTableDirs(fs, namespacePath);
+
+ return tableDirs.parallelStream().flatMap(tableDir -> {
+ try {
+ List<RegionInfo> tableRegions = scanRegionsInTable(fs, FSUtils.getRegionDirs(fs, tableDir));
+ LOG.debug("Found {} regions in table {} in namespace {}", tableRegions.size(),
+ tableDir.getName(), namespacePath.getName());
+ return tableRegions.stream();
+ } catch (IOException e) {
+ LOG.warn("Failed to scan table directory: {} for namespace {}", tableDir,
+ namespacePath.getName(), e);
+ return Stream.empty();
+ }
+ }).toList();
+ }
+
+ private List<RegionInfo> scanRegionsInTable(FileSystem fs, List<Path> regionDirs)
+ throws IOException {
+ return regionDirs.stream().map(regionDir -> {
+ String encodedRegionName = regionDir.getName();
+ try {
+ Path regionInfoPath = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
+ if (fs.exists(regionInfoPath)) {
+ RegionInfo ri = readRegionInfo(fs, regionInfoPath);
+ if (ri != null && isValidRegionInfo(ri, encodedRegionName)) {
+ LOG.debug("Found region: {} -> {}", encodedRegionName, ri.getRegionNameAsString());
+ return ri;
+ } else {
+ LOG.warn("Invalid RegionInfo in file: {}", regionInfoPath);
+ }
+ } else {
+ LOG.debug("No .regioninfo file found in region directory: {}", regionDir);
+ }
+ } catch (Exception e) {
+ LOG.warn("Failed to read region info from directory: {}", encodedRegionName, e);
+ }
+ return null;
+ }).filter(Objects::nonNull).collect(Collectors.toList());
+ }
+
+ private boolean isValidRegionInfo(RegionInfo regionInfo, String expectedEncodedName) {
+ if (!expectedEncodedName.equals(regionInfo.getEncodedName())) {
+ LOG.warn("RegionInfo encoded name mismatch: directory={}, regioninfo={}", expectedEncodedName,
+ regionInfo.getEncodedName());
+ return false;
+ }
+ return true;
+ }
+
+ private RegionInfo readRegionInfo(FileSystem fs, Path regionInfoPath) {
+ try (FSDataInputStream inputStream = fs.open(regionInfoPath);
+ DataInputStream dataInputStream = new DataInputStream(inputStream)) {
+ return RegionInfo.parseFrom(dataInputStream);
+ } catch (Exception e) {
+ LOG.warn("Failed to parse .regioninfo file: {}", regionInfoPath, e);
+ return null;
+ }
+ }
+
+ /**
+ * Retrieves the current regions from the hbase:meta table.
+ * @param connection The HBase connection to use.
+ * @return List of RegionInfo objects representing the current regions in meta.
+ * @throws IOException If there is an error accessing the meta table.
+ */
+ List<RegionInfo> getCurrentRegions(Connection connection) throws IOException {
+ LOG.info("Getting all regions from meta table");
+ return MetaTableAccessor.getAllRegions(connection, true);
+ }
+
+ @Override
+ protected synchronized boolean setTimeoutFailure(MasterProcedureEnv env) {
+ setState(
+ org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState.RUNNABLE);
+ env.getProcedureScheduler().addFront(this);
+ return false;
+ }
+
+ @Override
+ protected void rollbackState(MasterProcedureEnv env, RefreshMetaState refreshMetaState)
+ throws IOException, InterruptedException {
+ // No specific rollback needed as it is generally safe to re-run the procedure.
+ LOG.trace("Rollback not implemented for RefreshMetaProcedure state: {}", refreshMetaState);
+ }
+
+ @Override
+ protected RefreshMetaState getState(int stateId) {
+ return RefreshMetaState.forNumber(stateId);
+ }
+
+ @Override
+ protected int getStateId(RefreshMetaState refreshMetaState) {
+ return refreshMetaState.getNumber();
+ }
+
+ @Override
+ protected RefreshMetaState getInitialState() {
+ return RefreshMetaState.REFRESH_META_INIT;
+ }
+
+ @Override
+ protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ // For now, we'll use a simple approach since we do not need to store any state data
+ RefreshMetaStateData.Builder builder = RefreshMetaStateData.newBuilder();
+ serializer.serialize(builder.build());
+ }
+
+ @Override
+ protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ // For now, we'll use a simple approach since we do not need to store any state data
+ serializer.deserialize(RefreshMetaStateData.class);
+ }
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java
index c5c7ec602ea..7014b7cf01f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java
@@ -51,7 +51,8 @@ public interface TableProcedureInterface {
REGION_GC,
MERGED_REGIONS_GC/* region operations */,
REGION_TRUNCATE,
- RESTORE_BACKUP_SYSTEM_TABLE
+ RESTORE_BACKUP_SYSTEM_TABLE,
+ REFRESH_HFILES,
}
/** Returns the name of the table the procedure is operating on */
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableQueue.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableQueue.java
index 7be4c4b1810..a58db4d5ed6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableQueue.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableQueue.java
@@ -73,6 +73,7 @@ class TableQueue extends Queue<TableName> {
case MERGED_REGIONS_GC:
case REGION_SNAPSHOT:
case REGION_TRUNCATE:
+ case REFRESH_HFILES:
return false;
default:
break;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFactory.java
index 878f8dc17a1..cec8cd4ce9e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFactory.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFactory.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.master.region;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
@@ -34,6 +35,8 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ReflectionUtils;
import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.hbase.thirdparty.com.google.common.base.Strings;
+
/**
* The factory class for creating a {@link MasterRegion}.
*/
@@ -54,7 +57,24 @@ public final class MasterRegionFactory {
public static final String USE_HSYNC_KEY = "hbase.master.store.region.wal.hsync";
- public static final String MASTER_STORE_DIR = "MasterData";
+ // Default master data dir
+ private static final String MASTER_REGION_DIR_NAME_DEFAULT = "MasterData";
+
+ private static final String MASTER_REGION_DIR_NAME;
+ static {
+ Configuration conf = HBaseConfiguration.create();
+ MASTER_REGION_DIR_NAME = initMasterRegionDirName(conf);
+ }
+
+ public static String initMasterRegionDirName(Configuration conf) {
+ String suffix = conf.get(HConstants.HBASE_META_TABLE_SUFFIX,
+ HConstants.HBASE_META_TABLE_SUFFIX_DEFAULT_VALUE);
+ if (Strings.isNullOrEmpty(suffix)) {
+ return MASTER_REGION_DIR_NAME_DEFAULT;
+ } else {
+ return MASTER_REGION_DIR_NAME_DEFAULT + "_" + suffix;
+ }
+ }
private static final String FLUSH_SIZE_KEY = "hbase.master.store.region.flush.size";
@@ -116,7 +136,7 @@ public final class MasterRegionFactory {
public static MasterRegion create(MasterServices server) throws IOException {
Configuration conf = server.getConfiguration();
MasterRegionParams params = new MasterRegionParams().server(server)
- .regionDirName(MASTER_STORE_DIR).tableDescriptor(withTrackerConfigs(conf));
+ .regionDirName(getMasterRegionDirName()).tableDescriptor(withTrackerConfigs(conf));
long flushSize = conf.getLong(FLUSH_SIZE_KEY, DEFAULT_FLUSH_SIZE);
long flushPerChanges = conf.getLong(FLUSH_PER_CHANGES_KEY, DEFAULT_FLUSH_PER_CHANGES);
long flushIntervalMs = conf.getLong(FLUSH_INTERVAL_MS_KEY, DEFAULT_FLUSH_INTERVAL_MS);
@@ -134,4 +154,8 @@ public final class MasterRegionFactory {
.archivedHFileSuffix(ARCHIVED_HFILE_SUFFIX);
return MasterRegion.create(params);
}
+
+ public static String getMasterRegionDirName() {
+ return MASTER_REGION_DIR_NAME;
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/HFileProcedurePrettyPrinter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/HFileProcedurePrettyPrinter.java
index c254201db9b..0752c814f38 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/HFileProcedurePrettyPrinter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/HFileProcedurePrettyPrinter.java
@@ -85,7 +85,7 @@ public class HFileProcedurePrettyPrinter extends AbstractHBaseTool {
private void addAllHFiles() throws IOException {
Path masterProcDir =
- new Path(CommonFSUtils.getRootDir(conf), MasterRegionFactory.MASTER_STORE_DIR);
+ new Path(CommonFSUtils.getRootDir(conf), MasterRegionFactory.getMasterRegionDirName());
Path tableDir = CommonFSUtils.getTableDir(masterProcDir, MasterRegionFactory.TABLE_NAME);
FileSystem fs = tableDir.getFileSystem(conf);
Path regionDir =
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplit.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplit.java
index 4ac1d7c6396..4aa7ab619d8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplit.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplit.java
@@ -36,6 +36,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.IntSupplier;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.conf.ConfigurationManager;
import org.apache.hadoop.hbase.conf.PropagatingConfigurationObserver;
@@ -344,6 +345,12 @@ public class CompactSplit implements CompactionRequester, PropagatingConfigurati
return;
}
+ // Should not allow compaction if cluster is in read-only mode
+ if (isReadOnlyEnabled()) {
+ LOG.info("Ignoring compaction request for " + region + ",because read-only mode is on.");
+ return;
+ }
+
if (
this.server.isStopped() || (region.getTableDescriptor() != null
&& !region.getTableDescriptor().isCompactionEnabled())
@@ -442,6 +449,13 @@ public class CompactSplit implements CompactionRequester, PropagatingConfigurati
LOG.info(String.format("User has disabled compactions"));
return Optional.empty();
}
+
+ // Should not allow compaction if cluster is in read-only mode
+ if (isReadOnlyEnabled()) {
+ LOG.info(String.format("Compaction request skipped as read-only mode is on"));
+ return Optional.empty();
+ }
+
Optional<CompactionContext> compaction = store.requestCompaction(priority, tracker, user);
if (!compaction.isPresent() && region.getRegionInfo() != null) {
String reason = "Not compacting " + region.getRegionInfo().getRegionNameAsString()
@@ -856,6 +870,11 @@ public class CompactSplit implements CompactionRequester, PropagatingConfigurati
return compactionsEnabled;
}
+ private boolean isReadOnlyEnabled() {
+ return conf.getBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY,
+ HConstants.HBASE_GLOBAL_READONLY_ENABLED_DEFAULT);
+ }
+
public void setCompactionsEnabled(boolean compactionsEnabled) {
this.compactionsEnabled = compactionsEnabled;
this.conf.setBoolean(HBASE_REGION_SERVER_ENABLE_COMPACTION, compactionsEnabled);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index a2e38e53279..13eb5979359 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -178,6 +178,7 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CancelableProgressable;
import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.hbase.util.ConfigurationUtil;
import org.apache.hadoop.hbase.util.CoprocessorConfigurationUtil;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -939,6 +940,11 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
: this.htableDescriptor.getDurability();
decorateRegionConfiguration(conf);
+
+ CoprocessorConfigurationUtil.syncReadOnlyConfigurations(
+ ConfigurationUtil.isReadOnlyModeEnabled(conf), this.conf,
+ CoprocessorHost.REGION_COPROCESSOR_CONF_KEY);
+
if (rsServices != null) {
this.rsAccounting = this.rsServices.getRegionServerAccounting();
// don't initialize coprocessors if not running within a regionserver
@@ -8982,6 +8988,11 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
@Override
public void onConfigurationChange(Configuration conf) {
this.storeHotnessProtector.update(conf);
+
+ boolean readOnlyMode = ConfigurationUtil.isReadOnlyModeEnabled(conf);
+ CoprocessorConfigurationUtil.syncReadOnlyConfigurations(readOnlyMode, conf,
+ CoprocessorHost.REGION_COPROCESSOR_CONF_KEY);
+
// update coprocessorHost if the configuration has changed.
if (
CoprocessorConfigurationUtil.checkConfigurationChange(this.coprocessorHost, conf,
@@ -8991,6 +9002,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
LOG.info("Update the system coprocessors because the configuration has changed");
decorateRegionConfiguration(conf);
this.coprocessorHost = new RegionCoprocessorHost(this, rsServices, conf);
+ CoprocessorConfigurationUtil.syncReadOnlyConfigurations(readOnlyMode, this.conf,
+ CoprocessorHost.REGION_COPROCESSOR_CONF_KEY);
}
}
@@ -9001,6 +9014,12 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
public void registerChildren(ConfigurationManager manager) {
configurationManager = manager;
stores.values().forEach(manager::registerObserver);
+ if (coprocessorHost != null) {
+ coprocessorHost.registerConfigurationObservers(manager);
+ } else {
+ LOG.warn("Could not register HRegion coprocessors to the ConfigurationManager because "
+ + "RegionCoprocessorHost is null");
+ }
}
/**
@@ -9009,6 +9028,12 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
@Override
public void deregisterChildren(ConfigurationManager manager) {
stores.values().forEach(configurationManager::deregisterObserver);
+ if (coprocessorHost != null) {
+ coprocessorHost.deregisterConfigurationObservers(manager);
+ } else {
+ LOG.warn("Could not deregister HRegion coprocessors from the ConfigurationManager because "
+ + "RegionCoprocessorHost is null");
+ }
}
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index eea82ca511e..657cb01f38d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -161,6 +161,7 @@ import org.apache.hadoop.hbase.security.UserProvider;
import org.apache.hadoop.hbase.trace.TraceUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CompressionTest;
+import org.apache.hadoop.hbase.util.ConfigurationUtil;
import org.apache.hadoop.hbase.util.CoprocessorConfigurationUtil;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -825,6 +826,10 @@ public class HRegionServer extends HBaseServerBase<RSRpcServices>
try {
if (!isStopped() && !isAborted()) {
installShutdownHook();
+
+ CoprocessorConfigurationUtil.syncReadOnlyConfigurations(
+ ConfigurationUtil.isReadOnlyModeEnabled(conf), conf,
+ CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY);
// Initialize the RegionServerCoprocessorHost now that our ephemeral
// node was created, in case any coprocessors want to use ZooKeeper
this.rsHost = new RegionServerCoprocessorHost(this, this.conf);
@@ -1972,6 +1977,10 @@ public class HRegionServer extends HBaseServerBase<RSRpcServices>
final int logRollThreads = conf.getInt("hbase.regionserver.executor.log.roll.threads", 1);
executorService.startExecutorService(executorService.new ExecutorConfig()
.setExecutorType(ExecutorType.RS_LOG_ROLL).setCorePoolSize(logRollThreads));
+ final int rsRefreshHFilesThreads =
+ conf.getInt("hbase.regionserver.executor.refresh.hfiles.threads", 3);
+ executorService.startExecutorService(executorService.new ExecutorConfig()
+ .setExecutorType(ExecutorType.RS_REFRESH_HFILES).setCorePoolSize(rsRefreshHFilesThreads));
Threads.setDaemonThreadRunning(this.walRoller, getName() + ".logRoller",
uncaughtExceptionHandler);
@@ -2149,6 +2158,12 @@ public class HRegionServer extends HBaseServerBase<RSRpcServices>
configurationManager.registerObserver(this.rpcServices);
configurationManager.registerObserver(this.prefetchExecutorNotifier);
configurationManager.registerObserver(this);
+ if (rsHost != null) {
+ rsHost.registerConfigurationObservers(configurationManager);
+ } else {
+ LOG.warn("Could not register HRegionServer coprocessors to the ConfigurationManager because "
+ + "RegionServerCoprocessorHost is null");
+ }
}
/*
@@ -3473,6 +3488,10 @@ public class HRegionServer extends HBaseServerBase<RSRpcServices>
LOG.warn("Failed to initialize SuperUsers on reloading of the configuration");
}
+ boolean readOnlyMode = ConfigurationUtil.isReadOnlyModeEnabled(newConf);
+ CoprocessorConfigurationUtil.syncReadOnlyConfigurations(readOnlyMode, newConf,
+ CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY);
+
// update region server coprocessor if the configuration has changed.
if (
CoprocessorConfigurationUtil.checkConfigurationChange(this.rsHost, newConf,
@@ -3480,6 +3499,8 @@ public class HRegionServer extends HBaseServerBase<RSRpcServices>
) {
LOG.info("Update region server coprocessors because the configuration has changed");
this.rsHost = new RegionServerCoprocessorHost(this, newConf);
+ CoprocessorConfigurationUtil.syncReadOnlyConfigurations(readOnlyMode, this.conf,
+ CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY);
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RefreshHFilesCallable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RefreshHFilesCallable.java
new file mode 100644
index 00000000000..153449b3c80
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RefreshHFilesCallable.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.executor.EventType;
+import org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
+
+/**
+ * This is a RegionServer-side callable used by the HBase Master Procedure framework to perform the
+ * actual HFiles refresh operation on a specific region. It is dispatched from the
+ * {@link org.apache.hadoop.hbase.master.procedure.RefreshHFilesRegionProcedure} to the RegionServer
+ * and executes the logic to refresh store files in each store of the region.
+ */
+
+@InterfaceAudience.Private
+public class RefreshHFilesCallable extends BaseRSProcedureCallable {
+ private static final Logger LOG = LoggerFactory.getLogger(RefreshHFilesCallable.class);
+
+ private RegionInfo regionInfo;
+
+ @Override
+ protected byte[] doCall() throws Exception {
+ HRegion region = rs.getRegion(regionInfo.getEncodedName());
+ LOG.debug("Starting refreshHfiles operation on region {}", region);
+
+ try {
+ for (Store store : region.getStores()) {
+ store.refreshStoreFiles();
+ }
+ } catch (IOException ioe) {
+ LOG.warn("Exception while trying to refresh store files: ", ioe);
+ }
+ return null;
+ }
+
+ @Override
+ protected void initParameter(byte[] parameter) throws Exception {
+ MasterProcedureProtos.RefreshHFilesRegionParameter param =
+ MasterProcedureProtos.RefreshHFilesRegionParameter.parseFrom(parameter);
+ this.regionInfo = ProtobufUtil.toRegionInfo(param.getRegion());
+ }
+
+ @Override
+ public EventType getEventType() {
+ return EventType.RS_REFRESH_HFILES;
+ }
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java
index 87eca7b93c9..638b4118c18 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.backup.HFileArchiver;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
@@ -85,12 +86,12 @@ abstract class StoreFileTrackerBase implements StoreFileTracker {
@Override
public final List<StoreFileInfo> load() throws IOException {
- return doLoadStoreFiles(!isPrimaryReplica);
+ return doLoadStoreFiles(!isPrimaryReplica || isReadOnlyEnabled());
}
@Override
public final void add(Collection<StoreFileInfo> newFiles) throws IOException {
- if (isPrimaryReplica) {
+ if (isPrimaryReplica && !isReadOnlyEnabled()) {
doAddNewStoreFiles(newFiles);
}
}
@@ -98,14 +99,14 @@ abstract class StoreFileTrackerBase implements StoreFileTracker {
@Override
public final void replace(Collection<StoreFileInfo> compactedFiles,
Collection<StoreFileInfo> newFiles) throws IOException {
- if (isPrimaryReplica) {
+ if (isPrimaryReplica && !isReadOnlyEnabled()) {
doAddCompactionResults(compactedFiles, newFiles);
}
}
@Override
public final void set(List<StoreFileInfo> files) throws IOException {
- if (isPrimaryReplica) {
+ if (isPrimaryReplica && !isReadOnlyEnabled()) {
doSetStoreFiles(files);
}
}
@@ -140,8 +141,9 @@ abstract class StoreFileTrackerBase implements StoreFileTracker {
@Override
public final StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException {
- if (!isPrimaryReplica) {
- throw new IllegalStateException("Should not call create writer on secondary replicas");
+ if (!isPrimaryReplica || isReadOnlyEnabled()) {
+ throw new IllegalStateException(
+ "Should not call create writer on secondary replicas or in read-only mode");
}
// creating new cache config for each new writer
final CacheConfig cacheConf = ctx.getCacheConf();
@@ -390,6 +392,11 @@ abstract class StoreFileTrackerBase implements StoreFileTracker {
storeFiles);
}
+ private boolean isReadOnlyEnabled() {
+ return conf.getBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY,
+ HConstants.HBASE_GLOBAL_READONLY_ENABLED_DEFAULT);
+ }
+
/**
* For primary replica, we will call load once when opening a region, and the implementation could
* choose to do some cleanup work. So here we use {@code readOnly} to indicate that whether you
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AbstractReadOnlyController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AbstractReadOnlyController.java
new file mode 100644
index 00000000000..adb8efd59ca
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AbstractReadOnlyController.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.access;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Coprocessor;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
+public abstract class AbstractReadOnlyController implements Coprocessor {
+ private MasterServices masterServices;
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractReadOnlyController.class);
+
+ protected void internalReadOnlyGuard() throws DoNotRetryIOException {
+ throw new DoNotRetryIOException("Operation not allowed in Read-Only Mode");
+ }
+
+ @Override
+ public void start(CoprocessorEnvironment env) throws IOException {
+ }
+
+ @Override
+ public void stop(CoprocessorEnvironment env) {
+ }
+
+ public static void manageActiveClusterIdFile(boolean readOnlyEnabled, MasterFileSystem mfs) {
+ FileSystem fs = mfs.getFileSystem();
+ Path rootDir = mfs.getRootDir();
+ Path activeClusterFile = new Path(rootDir, HConstants.ACTIVE_CLUSTER_SUFFIX_FILE_NAME);
+
+ try {
+ if (readOnlyEnabled) {
+ // ENABLING READ-ONLY (false -> true), delete the active cluster file.
+ LOG.debug("Global read-only mode is being ENABLED. Deleting active cluster file: {}",
+ activeClusterFile);
+ try (FSDataInputStream in = fs.open(activeClusterFile)) {
+ String actualClusterFileData = IOUtils.toString(in, StandardCharsets.UTF_8);
+ String expectedClusterFileData = mfs.getSuffixFromConfig();
+ if (actualClusterFileData.equals(expectedClusterFileData)) {
+ fs.delete(activeClusterFile, false);
+ LOG.info("Successfully deleted active cluster file: {}", activeClusterFile);
+ } else {
+ LOG.debug(
+ "Active cluster file data does not match expected data. "
+ + "Not deleting the file to avoid potential inconsistency. "
+ + "Actual data: {}, Expected data: {}",
+ new String(actualClusterFileData), new String(expectedClusterFileData));
+ }
+ } catch (FileNotFoundException e) {
+ LOG.debug("Active cluster file does not exist at: {}. No need to delete.",
+ activeClusterFile);
+ } catch (IOException e) {
+ LOG.error(
+ "Failed to delete active cluster file: {}. "
+ + "Read-only flag will be updated, but file system state is inconsistent.",
+ activeClusterFile, e);
+ }
+ } else {
+ // DISABLING READ-ONLY (true -> false), create the active cluster file id file
+ int wait = mfs.getConfiguration().getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
+ if (!fs.exists(activeClusterFile)) {
+ FSUtils.setActiveClusterSuffix(fs, rootDir, mfs.computeAndSetSuffixFileDataToWrite(),
+ wait);
+ } else {
+ LOG.debug("Active cluster file already exists at: {}. No need to create it again.",
+ activeClusterFile);
+ }
+ }
+ } catch (IOException e) {
+ // We still update the flag, but log that the operation failed.
+ LOG.error("Failed to perform file operation for read-only switch. "
+ + "Flag will be updated, but file system state may be inconsistent.", e);
+ }
+ }
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/BulkLoadReadOnlyController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/BulkLoadReadOnlyController.java
new file mode 100644
index 00000000000..d81515c98ed
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/BulkLoadReadOnlyController.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.access;
+
+import java.io.IOException;
+import java.util.Optional;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.coprocessor.BulkLoadObserver;
+import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.yetus.audience.InterfaceAudience;
+
+@CoreCoprocessor
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
+public class BulkLoadReadOnlyController extends AbstractReadOnlyController
+ implements BulkLoadObserver, RegionCoprocessor {
+
+ @Override
+ public Optional<BulkLoadObserver> getBulkLoadObserver() {
+ return Optional.of(this);
+ }
+
+ @Override
+ public void prePrepareBulkLoad(ObserverContext<RegionCoprocessorEnvironment> ctx)
+ throws IOException {
+ internalReadOnlyGuard();
+ BulkLoadObserver.super.prePrepareBulkLoad(ctx);
+ }
+
+ @Override
+ public void preCleanupBulkLoad(ObserverContext<RegionCoprocessorEnvironment> ctx)
+ throws IOException {
+ internalReadOnlyGuard();
+ BulkLoadObserver.super.preCleanupBulkLoad(ctx);
+ }
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/EndpointReadOnlyController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/EndpointReadOnlyController.java
new file mode 100644
index 00000000000..edecc839ce4
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/EndpointReadOnlyController.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.access;
+
+import java.io.IOException;
+import java.util.Optional;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.EndpointObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hbase.thirdparty.com.google.protobuf.Message;
+import org.apache.hbase.thirdparty.com.google.protobuf.Service;
+
+@CoreCoprocessor
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
+public class EndpointReadOnlyController extends AbstractReadOnlyController
+ implements EndpointObserver, RegionCoprocessor {
+
+ @Override
+ public Optional<EndpointObserver> getEndpointObserver() {
+ return Optional.of(this);
+ }
+
+ @Override
+ public Message preEndpointInvocation(ObserverContext<? extends RegionCoprocessorEnvironment> ctx,
+ Service service, String methodName, Message request) throws IOException {
+ internalReadOnlyGuard();
+ return EndpointObserver.super.preEndpointInvocation(ctx, service, methodName, request);
+ }
+
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/MasterReadOnlyController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/MasterReadOnlyController.java
new file mode 100644
index 00000000000..c2eb8833b08
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/MasterReadOnlyController.java
@@ -0,0 +1,433 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.access;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.BalanceRequest;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.MasterObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.net.Address;
+import org.apache.hadoop.hbase.quotas.GlobalQuotaSettings;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.SyncReplicationState;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@CoreCoprocessor
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
+public class MasterReadOnlyController extends AbstractReadOnlyController
+ implements MasterCoprocessor, MasterObserver {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MasterReadOnlyController.class);
+
+ private MasterServices masterServices;
+
+ @Override
+ public Optional<MasterObserver> getMasterObserver() {
+ return Optional.of(this);
+ }
+
+ @Override
+ public TableDescriptor preCreateTableRegionsInfos(
+ ObserverContext<MasterCoprocessorEnvironment> ctx, TableDescriptor desc) throws IOException {
+ internalReadOnlyGuard();
+ return MasterObserver.super.preCreateTableRegionsInfos(ctx, desc);
+ }
+
+ @Override
+ public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ TableDescriptor desc, RegionInfo[] regions) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preCreateTable(ctx, desc, regions);
+ }
+
+ @Override
+ public void preCreateTableAction(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ TableDescriptor desc, RegionInfo[] regions) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preCreateTableAction(ctx, desc, regions);
+ }
+
+ @Override
+ public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
+ throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preDeleteTable(ctx, tableName);
+ }
+
+ @Override
+ public void preDeleteTableAction(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ TableName tableName) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preDeleteTableAction(ctx, tableName);
+ }
+
+ @Override
+ public void preTruncateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ TableName tableName) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preTruncateTable(ctx, tableName);
+ }
+
+ @Override
+ public void preTruncateTableAction(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ TableName tableName) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preTruncateTableAction(ctx, tableName);
+ }
+
+ @Override
+ public TableDescriptor preModifyTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ TableName tableName, TableDescriptor currentDescriptor, TableDescriptor newDescriptor)
+ throws IOException {
+ internalReadOnlyGuard();
+ return MasterObserver.super.preModifyTable(ctx, tableName, currentDescriptor, newDescriptor);
+ }
+
+ @Override
+ public String preModifyTableStoreFileTracker(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ TableName tableName, String dstSFT) throws IOException {
+ internalReadOnlyGuard();
+ return MasterObserver.super.preModifyTableStoreFileTracker(ctx, tableName, dstSFT);
+ }
+
+ @Override
+ public String preModifyColumnFamilyStoreFileTracker(
+ ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName, byte[] family,
+ String dstSFT) throws IOException {
+ internalReadOnlyGuard();
+ return MasterObserver.super.preModifyColumnFamilyStoreFileTracker(ctx, tableName, family,
+ dstSFT);
+ }
+
+ @Override
+ public void preModifyTableAction(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ TableName tableName, TableDescriptor currentDescriptor, TableDescriptor newDescriptor)
+ throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preModifyTableAction(ctx, tableName, currentDescriptor, newDescriptor);
+ }
+
+ @Override
+ public void preSplitRegion(ObserverContext<MasterCoprocessorEnvironment> c, TableName tableName,
+ byte[] splitRow) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preSplitRegion(c, tableName, splitRow);
+ }
+
+ @Override
+ public void preSplitRegionAction(ObserverContext<MasterCoprocessorEnvironment> c,
+ TableName tableName, byte[] splitRow) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preSplitRegionAction(c, tableName, splitRow);
+ }
+
+ @Override
+ public void preSplitRegionBeforeMETAAction(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ byte[] splitKey, List<Mutation> metaEntries) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preSplitRegionBeforeMETAAction(ctx, splitKey, metaEntries);
+ }
+
+ @Override
+ public void preSplitRegionAfterMETAAction(ObserverContext<MasterCoprocessorEnvironment> ctx)
+ throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preSplitRegionAfterMETAAction(ctx);
+ }
+
+ @Override
+ public void preTruncateRegion(ObserverContext<MasterCoprocessorEnvironment> c,
+ RegionInfo regionInfo) {
+ try {
+ internalReadOnlyGuard();
+ } catch (IOException e) {
+ LOG.info("Region truncation of region {} not allowed in read-only mode",
+ regionInfo.getRegionNameAsString());
+ }
+ MasterObserver.super.preTruncateRegion(c, regionInfo);
+ }
+
+ @Override
+ public void preTruncateRegionAction(ObserverContext<MasterCoprocessorEnvironment> c,
+ RegionInfo regionInfo) {
+ try {
+ internalReadOnlyGuard();
+ } catch (IOException e) {
+ LOG.info("Region truncation of region {} not allowed in read-only mode",
+ regionInfo.getRegionNameAsString());
+ }
+ MasterObserver.super.preTruncateRegionAction(c, regionInfo);
+ }
+
+ @Override
+ public void preMergeRegionsAction(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final RegionInfo[] regionsToMerge) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preMergeRegionsAction(ctx, regionsToMerge);
+ }
+
+ @Override
+ public void preMergeRegionsCommitAction(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ RegionInfo[] regionsToMerge, List<Mutation> metaEntries) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preMergeRegionsCommitAction(ctx, regionsToMerge, metaEntries);
+ }
+
+ @Override
+ public void preSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ SnapshotDescription snapshot, TableDescriptor tableDescriptor) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preSnapshot(ctx, snapshot, tableDescriptor);
+ }
+
+ @Override
+ public void preCloneSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ SnapshotDescription snapshot, TableDescriptor tableDescriptor) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preCloneSnapshot(ctx, snapshot, tableDescriptor);
+ }
+
+ @Override
+ public void preRestoreSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ SnapshotDescription snapshot, TableDescriptor tableDescriptor) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preRestoreSnapshot(ctx, snapshot, tableDescriptor);
+ }
+
+ @Override
+ public void preDeleteSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ SnapshotDescription snapshot) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preDeleteSnapshot(ctx, snapshot);
+ }
+
+ @Override
+ public void preCreateNamespace(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ NamespaceDescriptor ns) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preCreateNamespace(ctx, ns);
+ }
+
+ @Override
+ public void preModifyNamespace(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ NamespaceDescriptor currentNsDescriptor, NamespaceDescriptor newNsDescriptor)
+ throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preModifyNamespace(ctx, currentNsDescriptor, newNsDescriptor);
+ }
+
+ @Override
+ public void preDeleteNamespace(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ String namespace) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preDeleteNamespace(ctx, namespace);
+ }
+
+ @Override
+ public void preMasterStoreFlush(ObserverContext<MasterCoprocessorEnvironment> ctx)
+ throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preMasterStoreFlush(ctx);
+ }
+
+ @Override
+ public void preSetUserQuota(ObserverContext<MasterCoprocessorEnvironment> ctx, String userName,
+ GlobalQuotaSettings quotas) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preSetUserQuota(ctx, userName, quotas);
+ }
+
+ @Override
+ public void preSetUserQuota(ObserverContext<MasterCoprocessorEnvironment> ctx, String userName,
+ TableName tableName, GlobalQuotaSettings quotas) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preSetUserQuota(ctx, userName, tableName, quotas);
+ }
+
+ @Override
+ public void preSetUserQuota(ObserverContext<MasterCoprocessorEnvironment> ctx, String userName,
+ String namespace, GlobalQuotaSettings quotas) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preSetUserQuota(ctx, userName, namespace, quotas);
+ }
+
+ @Override
+ public void preSetTableQuota(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ TableName tableName, GlobalQuotaSettings quotas) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preSetTableQuota(ctx, tableName, quotas);
+ }
+
+ @Override
+ public void preSetNamespaceQuota(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ String namespace, GlobalQuotaSettings quotas) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preSetNamespaceQuota(ctx, namespace, quotas);
+ }
+
+ @Override
+ public void preSetRegionServerQuota(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ String regionServer, GlobalQuotaSettings quotas) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preSetRegionServerQuota(ctx, regionServer, quotas);
+ }
+
+ @Override
+ public void preMergeRegions(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final RegionInfo[] regionsToMerge) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preMergeRegions(ctx, regionsToMerge);
+ }
+
+ @Override
+ public void preMoveServersAndTables(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preMoveServersAndTables(ctx, servers, tables, targetGroup);
+ }
+
+ @Override
+ public void preMoveServers(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ Set<Address> servers, String targetGroup) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preMoveServers(ctx, servers, targetGroup);
+ }
+
+ @Override
+ public void preMoveTables(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ Set<TableName> tables, String targetGroup) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preMoveTables(ctx, tables, targetGroup);
+ }
+
+ @Override
+ public void preAddRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx, String name)
+ throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preAddRSGroup(ctx, name);
+ }
+
+ @Override
+ public void preRemoveRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx, String name)
+ throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preRemoveRSGroup(ctx, name);
+ }
+
+ @Override
+ public void preBalanceRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx, String groupName,
+ BalanceRequest request) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preBalanceRSGroup(ctx, groupName, request);
+ }
+
+ @Override
+ public void preRemoveServers(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ Set<Address> servers) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preRemoveServers(ctx, servers);
+ }
+
+ @Override
+ public void preRenameRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx, String oldName,
+ String newName) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preRenameRSGroup(ctx, oldName, newName);
+ }
+
+ @Override
+ public void preUpdateRSGroupConfig(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ String groupName, Map<String, String> configuration) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preUpdateRSGroupConfig(ctx, groupName, configuration);
+ }
+
+ @Override
+ public void preAddReplicationPeer(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ String peerId, ReplicationPeerConfig peerConfig) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preAddReplicationPeer(ctx, peerId, peerConfig);
+ }
+
+ @Override
+ public void preRemoveReplicationPeer(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ String peerId) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preRemoveReplicationPeer(ctx, peerId);
+ }
+
+ @Override
+ public void preEnableReplicationPeer(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ String peerId) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preEnableReplicationPeer(ctx, peerId);
+ }
+
+ @Override
+ public void preDisableReplicationPeer(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ String peerId) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preDisableReplicationPeer(ctx, peerId);
+ }
+
+ @Override
+ public void preUpdateReplicationPeerConfig(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ String peerId, ReplicationPeerConfig peerConfig) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preUpdateReplicationPeerConfig(ctx, peerId, peerConfig);
+ }
+
+ @Override
+ public void preTransitReplicationPeerSyncReplicationState(
+ ObserverContext<MasterCoprocessorEnvironment> ctx, String peerId, SyncReplicationState state)
+ throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preTransitReplicationPeerSyncReplicationState(ctx, peerId, state);
+ }
+
+ @Override
+ public void preGrant(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ UserPermission userPermission, boolean mergeExistingPermissions) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preGrant(ctx, userPermission, mergeExistingPermissions);
+ }
+
+ @Override
+ public void preRevoke(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ UserPermission userPermission) throws IOException {
+ internalReadOnlyGuard();
+ MasterObserver.super.preRevoke(ctx, userPermission);
+ }
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/RegionReadOnlyController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/RegionReadOnlyController.java
new file mode 100644
index 00000000000..411b4459f12
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/RegionReadOnlyController.java
@@ -0,0 +1,371 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.access;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Optional;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.CompareOperator;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.CheckAndMutate;
+import org.apache.hadoop.hbase.client.CheckAndMutateResult;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.RegionObserver;
+import org.apache.hadoop.hbase.filter.ByteArrayComparable;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.regionserver.FlushLifeCycleTracker;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
+import org.apache.hadoop.hbase.regionserver.ScanOptions;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.wal.WALEdit;
+import org.apache.hadoop.hbase.wal.WALKey;
+import org.apache.yetus.audience.InterfaceAudience;
+
+@CoreCoprocessor
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
+public class RegionReadOnlyController extends AbstractReadOnlyController
+ implements RegionCoprocessor, RegionObserver {
+
+ private boolean isOnMeta(final ObserverContext<? extends RegionCoprocessorEnvironment> c) {
+ return TableName.isMetaTableName(c.getEnvironment().getRegionInfo().getTable());
+ }
+
+ @Override
+ public Optional<RegionObserver> getRegionObserver() {
+ return Optional.of(this);
+ }
+
+ @Override
+ public void preFlushScannerOpen(ObserverContext<? extends RegionCoprocessorEnvironment> c,
+ Store store, ScanOptions options, FlushLifeCycleTracker tracker) throws IOException {
+ if (!isOnMeta(c)) {
+ internalReadOnlyGuard();
+ }
+ RegionObserver.super.preFlushScannerOpen(c, store, options, tracker);
+ }
+
+ @Override
+ public void preFlush(final ObserverContext<? extends RegionCoprocessorEnvironment> c,
+ FlushLifeCycleTracker tracker) throws IOException {
+ if (!isOnMeta(c)) {
+ internalReadOnlyGuard();
+ }
+ RegionObserver.super.preFlush(c, tracker);
+ }
+
+ @Override
+ public InternalScanner preFlush(ObserverContext<? extends RegionCoprocessorEnvironment> c,
+ Store store, InternalScanner scanner, FlushLifeCycleTracker tracker) throws IOException {
+ if (!isOnMeta(c)) {
+ internalReadOnlyGuard();
+ }
+ return RegionObserver.super.preFlush(c, store, scanner, tracker);
+ }
+
+ @Override
+ public void preMemStoreCompaction(ObserverContext<? extends RegionCoprocessorEnvironment> c,
+ Store store) throws IOException {
+ internalReadOnlyGuard();
+ RegionObserver.super.preMemStoreCompaction(c, store);
+ }
+
+ @Override
+ public void preMemStoreCompactionCompactScannerOpen(
+ ObserverContext<? extends RegionCoprocessorEnvironment> c, Store store, ScanOptions options)
+ throws IOException {
+ internalReadOnlyGuard();
+ RegionObserver.super.preMemStoreCompactionCompactScannerOpen(c, store, options);
+ }
+
+ @Override
+ public InternalScanner preMemStoreCompactionCompact(
+ ObserverContext<? extends RegionCoprocessorEnvironment> c, Store store, InternalScanner scanner)
+ throws IOException {
+ internalReadOnlyGuard();
+ return RegionObserver.super.preMemStoreCompactionCompact(c, store, scanner);
+ }
+
+ @Override
+ public void preCompactSelection(ObserverContext<? extends RegionCoprocessorEnvironment> c,
+ Store store, List<? extends StoreFile> candidates, CompactionLifeCycleTracker tracker)
+ throws IOException {
+ if (!isOnMeta(c)) {
+ internalReadOnlyGuard();
+ }
+ RegionObserver.super.preCompactSelection(c, store, candidates, tracker);
+ }
+
+ @Override
+ public void preCompactScannerOpen(ObserverContext<? extends RegionCoprocessorEnvironment> c,
+ Store store, ScanType scanType, ScanOptions options, CompactionLifeCycleTracker tracker,
+ CompactionRequest request) throws IOException {
+ if (!isOnMeta(c)) {
+ internalReadOnlyGuard();
+ }
+ RegionObserver.super.preCompactScannerOpen(c, store, scanType, options, tracker, request);
+ }
+
+ @Override
+ public InternalScanner preCompact(ObserverContext<? extends RegionCoprocessorEnvironment> c,
+ Store store, InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
+ CompactionRequest request) throws IOException {
+ if (!isOnMeta(c)) {
+ internalReadOnlyGuard();
+ }
+ return RegionObserver.super.preCompact(c, store, scanner, scanType, tracker, request);
+ }
+
+ @Override
+ public void prePut(ObserverContext<? extends RegionCoprocessorEnvironment> c, Put put,
+ WALEdit edit, Durability durability) throws IOException {
+ if (!isOnMeta(c)) {
+ internalReadOnlyGuard();
+ }
+ RegionObserver.super.prePut(c, put, edit, durability);
+ }
+
+ @Override
+ public void prePut(ObserverContext<? extends RegionCoprocessorEnvironment> c, Put put,
+ WALEdit edit) throws IOException {
+ if (!isOnMeta(c)) {
+ internalReadOnlyGuard();
+ }
+ RegionObserver.super.prePut(c, put, edit);
+ }
+
+ @Override
+ public void preDelete(ObserverContext<? extends RegionCoprocessorEnvironment> c, Delete delete,
+ WALEdit edit, Durability durability) throws IOException {
+ if (!isOnMeta(c)) {
+ internalReadOnlyGuard();
+ }
+ RegionObserver.super.preDelete(c, delete, edit, durability);
+ }
+
+ @Override
+ public void preDelete(ObserverContext<? extends RegionCoprocessorEnvironment> c, Delete delete,
+ WALEdit edit) throws IOException {
+ if (!isOnMeta(c)) {
+ internalReadOnlyGuard();
+ }
+ RegionObserver.super.preDelete(c, delete, edit);
+ }
+
+ @Override
+ public void preBatchMutate(ObserverContext<? extends RegionCoprocessorEnvironment> c,
+ MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {
+ if (!isOnMeta(c)) {
+ internalReadOnlyGuard();
+ }
+ RegionObserver.super.preBatchMutate(c, miniBatchOp);
+ }
+
+ @Override
+ public boolean preCheckAndPut(ObserverContext<? extends RegionCoprocessorEnvironment> c,
+ byte[] row, byte[] family, byte[] qualifier, CompareOperator op, ByteArrayComparable comparator,
+ Put put, boolean result) throws IOException {
+ if (!isOnMeta(c)) {
+ internalReadOnlyGuard();
+ }
+ return RegionObserver.super.preCheckAndPut(c, row, family, qualifier, op, comparator, put,
+ result);
+ }
+
+ @Override
+ public boolean preCheckAndPut(ObserverContext<? extends RegionCoprocessorEnvironment> c,
+ byte[] row, Filter filter, Put put, boolean result) throws IOException {
+ if (!isOnMeta(c)) {
+ internalReadOnlyGuard();
+ }
+ return RegionObserver.super.preCheckAndPut(c, row, filter, put, result);
+ }
+
+ @Override
+ public boolean preCheckAndPutAfterRowLock(
+ ObserverContext<? extends RegionCoprocessorEnvironment> c, byte[] row, byte[] family,
+ byte[] qualifier, CompareOperator op, ByteArrayComparable comparator, Put put, boolean result)
+ throws IOException {
+ if (!isOnMeta(c)) {
+ internalReadOnlyGuard();
+ }
+ return RegionObserver.super.preCheckAndPutAfterRowLock(c, row, family, qualifier, op,
+ comparator, put, result);
+ }
+
+ @Override
+ public boolean preCheckAndPutAfterRowLock(
+ ObserverContext<? extends RegionCoprocessorEnvironment> c, byte[] row, Filter filter, Put put,
+ boolean result) throws IOException {
+ if (!isOnMeta(c)) {
+ internalReadOnlyGuard();
+ }
+ return RegionObserver.super.preCheckAndPutAfterRowLock(c, row, filter, put, result);
+ }
+
+ @Override
+ public boolean preCheckAndDelete(ObserverContext<? extends RegionCoprocessorEnvironment> c,
+ byte[] row, byte[] family, byte[] qualifier, CompareOperator op, ByteArrayComparable comparator,
+ Delete delete, boolean result) throws IOException {
+ if (!isOnMeta(c)) {
+ internalReadOnlyGuard();
+ }
+ return RegionObserver.super.preCheckAndDelete(c, row, family, qualifier, op, comparator, delete,
+ result);
+ }
+
+ @Override
+ public boolean preCheckAndDelete(ObserverContext<? extends RegionCoprocessorEnvironment> c,
+ byte[] row, Filter filter, Delete delete, boolean result) throws IOException {
+ if (!isOnMeta(c)) {
+ internalReadOnlyGuard();
+ }
+ return RegionObserver.super.preCheckAndDelete(c, row, filter, delete, result);
+ }
+
+ @Override
+ public boolean preCheckAndDeleteAfterRowLock(
+ ObserverContext<? extends RegionCoprocessorEnvironment> c, byte[] row, byte[] family,
+ byte[] qualifier, CompareOperator op, ByteArrayComparable comparator, Delete delete,
+ boolean result) throws IOException {
+ if (!isOnMeta(c)) {
+ internalReadOnlyGuard();
+ }
+ return RegionObserver.super.preCheckAndDeleteAfterRowLock(c, row, family, qualifier, op,
+ comparator, delete, result);
+ }
+
+ @Override
+ public boolean preCheckAndDeleteAfterRowLock(
+ ObserverContext<? extends RegionCoprocessorEnvironment> c, byte[] row, Filter filter,
+ Delete delete, boolean result) throws IOException {
+ if (!isOnMeta(c)) {
+ internalReadOnlyGuard();
+ }
+ return RegionObserver.super.preCheckAndDeleteAfterRowLock(c, row, filter, delete, result);
+ }
+
+ @Override
+ public CheckAndMutateResult preCheckAndMutate(
+ ObserverContext<? extends RegionCoprocessorEnvironment> c, CheckAndMutate checkAndMutate,
+ CheckAndMutateResult result) throws IOException {
+ internalReadOnlyGuard();
+ return RegionObserver.super.preCheckAndMutate(c, checkAndMutate, result);
+ }
+
+ @Override
+ public CheckAndMutateResult preCheckAndMutateAfterRowLock(
+ ObserverContext<? extends RegionCoprocessorEnvironment> c, CheckAndMutate checkAndMutate,
+ CheckAndMutateResult result) throws IOException {
+ internalReadOnlyGuard();
+ return RegionObserver.super.preCheckAndMutateAfterRowLock(c, checkAndMutate, result);
+ }
+
+ @Override
+ public Result preAppend(ObserverContext<? extends RegionCoprocessorEnvironment> c, Append append)
+ throws IOException {
+ internalReadOnlyGuard();
+ return RegionObserver.super.preAppend(c, append);
+ }
+
+ @Override
+ public Result preAppend(ObserverContext<? extends RegionCoprocessorEnvironment> c, Append append,
+ WALEdit edit) throws IOException {
+ internalReadOnlyGuard();
+ return RegionObserver.super.preAppend(c, append, edit);
+ }
+
+ @Override
+ public Result preAppendAfterRowLock(ObserverContext<? extends RegionCoprocessorEnvironment> c,
+ Append append) throws IOException {
+ internalReadOnlyGuard();
+ return RegionObserver.super.preAppendAfterRowLock(c, append);
+ }
+
+ @Override
+ public Result preIncrement(ObserverContext<? extends RegionCoprocessorEnvironment> c,
+ Increment increment) throws IOException {
+ internalReadOnlyGuard();
+ return RegionObserver.super.preIncrement(c, increment);
+ }
+
+ @Override
+ public Result preIncrement(ObserverContext<? extends RegionCoprocessorEnvironment> c,
+ Increment increment, WALEdit edit) throws IOException {
+ internalReadOnlyGuard();
+ return RegionObserver.super.preIncrement(c, increment, edit);
+ }
+
+ @Override
+ public Result preIncrementAfterRowLock(ObserverContext<? extends RegionCoprocessorEnvironment> c,
+ Increment increment) throws IOException {
+ internalReadOnlyGuard();
+ return RegionObserver.super.preIncrementAfterRowLock(c, increment);
+ }
+
+ @Override
+ public void preReplayWALs(ObserverContext<? extends RegionCoprocessorEnvironment> ctx,
+ RegionInfo info, Path edits) throws IOException {
+ if (!isOnMeta(ctx)) {
+ internalReadOnlyGuard();
+ }
+ RegionObserver.super.preReplayWALs(ctx, info, edits);
+ }
+
+ @Override
+ public void preBulkLoadHFile(ObserverContext<? extends RegionCoprocessorEnvironment> ctx,
+ List<Pair<byte[], String>> familyPaths) throws IOException {
+ internalReadOnlyGuard();
+ RegionObserver.super.preBulkLoadHFile(ctx, familyPaths);
+ }
+
+ @Override
+ public void preCommitStoreFile(ObserverContext<? extends RegionCoprocessorEnvironment> ctx,
+ byte[] family, List<Pair<Path, Path>> pairs) throws IOException {
+ internalReadOnlyGuard();
+ RegionObserver.super.preCommitStoreFile(ctx, family, pairs);
+ }
+
+ @Override
+ public void preWALAppend(ObserverContext<? extends RegionCoprocessorEnvironment> ctx, WALKey key,
+ WALEdit edit) throws IOException {
+ // Only allow this operation for meta table
+ if (!TableName.isMetaTableName(key.getTableName())) {
+ internalReadOnlyGuard();
+ }
+ RegionObserver.super.preWALAppend(ctx, key, edit);
+ }
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/RegionServerReadOnlyController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/RegionServerReadOnlyController.java
new file mode 100644
index 00000000000..2cd69f6f0d9
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/RegionServerReadOnlyController.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.access;
+
+import java.io.IOException;
+import java.util.Optional;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
+
+@CoreCoprocessor
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
+public class RegionServerReadOnlyController extends AbstractReadOnlyController
+ implements RegionServerObserver, RegionServerCoprocessor {
+
+ @Override
+ public Optional<RegionServerObserver> getRegionServerObserver() {
+ return Optional.of(this);
+ }
+
+ @Override
+ public void preRollWALWriterRequest(ObserverContext<RegionServerCoprocessorEnvironment> ctx)
+ throws IOException {
+ internalReadOnlyGuard();
+ RegionServerObserver.super.preRollWALWriterRequest(ctx);
+ }
+
+ @Override
+ public void preReplicationSinkBatchMutate(ObserverContext<RegionServerCoprocessorEnvironment> ctx,
+ AdminProtos.WALEntry walEntry, Mutation mutation) throws IOException {
+ internalReadOnlyGuard();
+ RegionServerObserver.super.preReplicationSinkBatchMutate(ctx, walEntry, mutation);
+ }
+
+ @Override
+ public void preReplicateLogEntries(ObserverContext<RegionServerCoprocessorEnvironment> ctx)
+ throws IOException {
+ internalReadOnlyGuard();
+ RegionServerObserver.super.preReplicateLogEntries(ctx);
+ }
+
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/BulkLoadHFilesTool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/BulkLoadHFilesTool.java
index 4d6f57e22ed..1889ea61b0b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/BulkLoadHFilesTool.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/BulkLoadHFilesTool.java
@@ -660,21 +660,21 @@ public class BulkLoadHFilesTool extends Configured implements BulkLoadHFiles, To
private void checkRegionIndexValid(int idx, List<Pair<byte[], byte[]>> startEndKeys,
TableName tableName) throws IOException {
if (idx < 0) {
- throw new IOException("The first region info for table " + tableName
- + " can't be found in hbase:meta.Please use hbck tool to fix it first.");
+ throw new IOException("The first region info for table " + tableName + " can't be found in "
+ + TableName.META_TABLE_NAME + ". Please use hbck tool to fix it" + " first.");
} else if (
(idx == startEndKeys.size() - 1)
&& !Bytes.equals(startEndKeys.get(idx).getSecond(), HConstants.EMPTY_BYTE_ARRAY)
) {
- throw new IOException("The last region info for table " + tableName
- + " can't be found in hbase:meta.Please use hbck tool to fix it first.");
+ throw new IOException("The last region info for table " + tableName + " can't be found in "
+ + TableName.META_TABLE_NAME + ". Please use hbck tool to fix it" + " first.");
} else if (
idx + 1 < startEndKeys.size() && !(Bytes.compareTo(startEndKeys.get(idx).getSecond(),
startEndKeys.get(idx + 1).getFirst()) == 0)
) {
throw new IOException("The endkey of one region for table " + tableName
- + " is not equal to the startkey of the next region in hbase:meta."
- + "Please use hbck tool to fix it first.");
+ + " is not equal to the startkey of the next region in " + TableName.META_TABLE_NAME + "."
+ + " Please use hbck tool to fix it first.");
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConfigurationUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConfigurationUtil.java
index 555d1fa5a8a..aed30e46af7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConfigurationUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConfigurationUtil.java
@@ -22,6 +22,7 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.util.StringUtils;
import org.apache.yetus.audience.InterfaceAudience;
@@ -114,4 +115,9 @@ public final class ConfigurationUtil {
}
return rtn;
}
+
+ public static boolean isReadOnlyModeEnabled(Configuration conf) {
+ return conf.getBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY,
+ HConstants.HBASE_GLOBAL_READONLY_ENABLED_DEFAULT);
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CoprocessorConfigurationUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CoprocessorConfigurationUtil.java
index 18ded4319ad..bbd1e6755d5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CoprocessorConfigurationUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CoprocessorConfigurationUtil.java
@@ -17,10 +17,19 @@
*/
package org.apache.hadoop.hbase.util;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.security.access.BulkLoadReadOnlyController;
+import org.apache.hadoop.hbase.security.access.EndpointReadOnlyController;
+import org.apache.hadoop.hbase.security.access.MasterReadOnlyController;
+import org.apache.hadoop.hbase.security.access.RegionReadOnlyController;
+import org.apache.hadoop.hbase.security.access.RegionServerReadOnlyController;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
@@ -102,4 +111,80 @@ public final class CoprocessorConfigurationUtil {
}
return false;
}
+
+ private static List<String> getCoprocessorsFromConfig(Configuration conf,
+ String configurationKey) {
+ String[] existing = conf.getStrings(configurationKey);
+ return existing != null ? new ArrayList<>(Arrays.asList(existing)) : new ArrayList<>();
+ }
+
+ public static void addCoprocessors(Configuration conf, String configurationKey,
+ List<String> coprocessorsToAdd) {
+ List<String> existing = getCoprocessorsFromConfig(conf, configurationKey);
+
+ boolean isModified = false;
+
+ for (String coprocessor : coprocessorsToAdd) {
+ if (!existing.contains(coprocessor)) {
+ existing.add(coprocessor);
+ isModified = true;
+ }
+ }
+
+ if (isModified) {
+ conf.setStrings(configurationKey, existing.toArray(new String[0]));
+ }
+ }
+
+ public static void removeCoprocessors(Configuration conf, String configurationKey,
+ List<String> coprocessorsToRemove) {
+ List<String> existing = getCoprocessorsFromConfig(conf, configurationKey);
+
+ if (existing.isEmpty()) {
+ return;
+ }
+
+ boolean isModified = false;
+
+ for (String coprocessor : coprocessorsToRemove) {
+ if (existing.contains(coprocessor)) {
+ existing.remove(coprocessor);
+ isModified = true;
+ }
+ }
+
+ if (isModified) {
+ conf.setStrings(configurationKey, existing.toArray(new String[0]));
+ }
+ }
+
+ private static List<String> getReadOnlyCoprocessors(String configurationKey) {
+ return switch (configurationKey) {
+ case CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY -> List
+ .of(MasterReadOnlyController.class.getName());
+
+ case CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY -> List
+ .of(RegionServerReadOnlyController.class.getName());
+
+ case CoprocessorHost.REGION_COPROCESSOR_CONF_KEY -> List.of(
+ RegionReadOnlyController.class.getName(), BulkLoadReadOnlyController.class.getName(),
+ EndpointReadOnlyController.class.getName());
+
+ default -> throw new IllegalArgumentException(
+ "Unsupported coprocessor configuration key: " + configurationKey);
+ };
+ }
+
+ public static void syncReadOnlyConfigurations(boolean readOnlyMode, Configuration conf,
+ String configurationKey) {
+ conf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, readOnlyMode);
+
+ List<String> cpList = getReadOnlyCoprocessors(configurationKey);
+ // If readonly is true then add the coprocessor of master
+ if (readOnlyMode) {
+ CoprocessorConfigurationUtil.addCoprocessors(conf, configurationKey, cpList);
+ } else {
+ CoprocessorConfigurationUtil.removeCoprocessors(conf, configurationKey, cpList);
+ }
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
index 75bf721ef41..e38ce97b026 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
@@ -153,14 +153,14 @@ public class FSTableDescriptors implements TableDescriptors {
}
TableDescriptorBuilder builder = createMetaTableDescriptorBuilder(conf);
TableDescriptor td = StoreFileTrackerFactory.updateWithTrackerConfigs(conf, builder.build());
- LOG.info("Creating new hbase:meta table descriptor {}", td);
+ LOG.info("Creating new {} table descriptor {}", TableName.META_TABLE_NAME, td);
TableName tableName = td.getTableName();
Path tableDir = CommonFSUtils.getTableDir(rootdir, tableName);
Path p = writeTableDescriptor(fs, td, tableDir, null);
if (p == null) {
- throw new IOException("Failed update hbase:meta table descriptor");
+ throw new IOException("Failed update " + TableName.META_TABLE_NAME + " table descriptor");
}
- LOG.info("Updated hbase:meta table descriptor to {}", p);
+ LOG.info("Updated {} table descriptor to {}", TableName.META_TABLE_NAME, p);
return td;
}
@@ -236,7 +236,7 @@ public class FSTableDescriptors implements TableDescriptors {
cachehits++;
return cachedtdm;
}
- // we do not need to go to fs any more
+ // we do not need to go to the fs anymore
if (fsvisited) {
return null;
}
@@ -270,7 +270,8 @@ public class FSTableDescriptors implements TableDescriptors {
LOG.info("Fetching table descriptors from the filesystem.");
final long startTime = EnvironmentEdgeManager.currentTime();
AtomicBoolean allvisited = new AtomicBoolean(usecache);
- List<Path> tableDirs = FSUtils.getTableDirs(fs, rootdir);
+ List<Path> tableDirs =
+ FSUtils.getTableDirs(fs, rootdir).stream().filter(FSUtils::isLocalMetaTable).toList();
if (!tableDescriptorParallelLoadEnable) {
for (Path dir : tableDirs) {
internalGet(dir, tds, allvisited);
@@ -668,8 +669,8 @@ public class FSTableDescriptors implements TableDescriptors {
* @param htd description of the table to write
* @param forceCreation if <tt>true</tt>,then even if previous table descriptor is present it will
* be overwritten
- * @return <tt>true</tt> if the we successfully created the file, <tt>false</tt> if the file
- * already exists and we weren't forcing the descriptor creation.
+ * @return <tt>true</tt> if we successfully created the file, <tt>false</tt> if the file already
+ * exists, and we weren't forcing the descriptor creation.
* @throws IOException if a filesystem error occurs
*/
public boolean createTableDescriptorForTableDirectory(Path tableDir, TableDescriptor htd,
@@ -688,8 +689,8 @@ public class FSTableDescriptors implements TableDescriptors {
* @param htd description of the table to write
* @param forceCreation if <tt>true</tt>,then even if previous table descriptor is present it will
* be overwritten
- * @return <tt>true</tt> if the we successfully created the file, <tt>false</tt> if the file
- * already exists and we weren't forcing the descriptor creation.
+ * @return <tt>true</tt> if we successfully created the file, <tt>false</tt> if the file already
+ * exists, and we weren't forcing the descriptor creation.
* @throws IOException if a filesystem error occurs
*/
public static boolean createTableDescriptorForTableDirectory(FileSystem fs, Path tableDir,
@@ -706,4 +707,14 @@ public class FSTableDescriptors implements TableDescriptors {
}
return writeTableDescriptor(fs, htd, tableDir, opt.map(Pair::getFirst).orElse(null)) != null;
}
+
+ /**
+ * Invalidates the table descriptor cache.
+ */
+ @Override
+ public void invalidateTableDescriptorCache() {
+ LOG.info("Invalidating table descriptor cache.");
+ this.fsvisited = false;
+ this.cache.clear();
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 3b446826b77..adc5e39fdcd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -31,6 +31,7 @@ import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.InetSocketAddress;
import java.net.URI;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -65,6 +66,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.ActiveClusterSuffix;
import org.apache.hadoop.hbase.ClusterId;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HDFSBlocksDistribution;
@@ -541,15 +543,15 @@ public final class FSUtils {
* @return <code>true</code> if the file exists, otherwise <code>false</code>
* @throws IOException if checking the FileSystem fails
*/
- public static boolean checkClusterIdExists(FileSystem fs, Path rootdir, long wait)
- throws IOException {
+ public static boolean checkFileExistsInHbaseRootDir(FileSystem fs, Path rootdir, String file,
+ long wait) throws IOException {
while (true) {
try {
- Path filePath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
+ Path filePath = new Path(rootdir, file);
return fs.exists(filePath);
} catch (IOException ioe) {
if (wait > 0L) {
- LOG.warn("Unable to check cluster ID file in {}, retrying in {}ms", rootdir, wait, ioe);
+ LOG.warn("Unable to check file {} in {}, retrying in {}ms", file, rootdir, wait, ioe);
try {
Thread.sleep(wait);
} catch (InterruptedException e) {
@@ -611,6 +613,46 @@ public final class FSUtils {
return clusterId;
}
+ public static ActiveClusterSuffix getActiveClusterSuffix(FileSystem fs, Path rootdir)
+ throws IOException {
+ Path idPath = new Path(rootdir, HConstants.ACTIVE_CLUSTER_SUFFIX_FILE_NAME);
+ ActiveClusterSuffix cs = null;
+ FileStatus status = fs.exists(idPath) ? fs.getFileStatus(idPath) : null;
+ if (status != null) {
+ int len = Ints.checkedCast(status.getLen());
+ byte[] content = new byte[len];
+ FSDataInputStream in = fs.open(idPath);
+ try {
+ in.readFully(content);
+ } catch (EOFException eof) {
+ LOG.warn("Cluster Suffix file {} is empty ", idPath);
+ } finally {
+ in.close();
+ }
+ try {
+ cs = ActiveClusterSuffix.parseFrom(content);
+ } catch (DeserializationException e) {
+ throw new IOException("content=" + Bytes.toString(content), e);
+ }
+ // If not pb'd, make it so.
+ if (!ProtobufUtil.isPBMagicPrefix(content)) {
+ String data = null;
+ in = fs.open(idPath);
+ try {
+ data = in.readUTF();
+ cs = new ActiveClusterSuffix(data);
+ } catch (EOFException eof) {
+ LOG.warn("Active Cluster Suffix File {} is empty ", idPath);
+ } finally {
+ in.close();
+ }
+ }
+ return cs;
+ } else {
+ throw new FileNotFoundException("Active Cluster Suffix File " + idPath + " not found");
+ }
+ }
+
/**
* */
private static void rewriteAsPb(final FileSystem fs, final Path rootdir, final Path p,
@@ -638,31 +680,57 @@ public final class FSUtils {
*/
public static void setClusterId(final FileSystem fs, final Path rootdir,
final ClusterId clusterId, final long wait) throws IOException {
-
final Path idFile = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
final Path tempDir = new Path(rootdir, HConstants.HBASE_TEMP_DIRECTORY);
final Path tempIdFile = new Path(tempDir, HConstants.CLUSTER_ID_FILE_NAME);
LOG.debug("Create cluster ID file [{}] with ID: {}", idFile, clusterId);
+ writeClusterInfo(fs, rootdir, idFile, tempIdFile, clusterId.toByteArray(), wait);
+ }
+
+ /**
+ * Writes a user provided suffix for this cluster to the "active_cluster_suffix.id" file in the
+ * HBase root directory. If any operations on the ID file fails, and {@code wait} is a positive
+ * value, the method will retry to produce the ID file until the thread is forcibly interrupted.
+ */
+
+ public static void setActiveClusterSuffix(final FileSystem fs, final Path rootdir, byte[] bdata,
+ final long wait) throws IOException {
+ final Path idFile = new Path(rootdir, HConstants.ACTIVE_CLUSTER_SUFFIX_FILE_NAME);
+ final Path tempDir = new Path(rootdir, HConstants.HBASE_TEMP_DIRECTORY);
+ final Path tempIdFile = new Path(tempDir, HConstants.ACTIVE_CLUSTER_SUFFIX_FILE_NAME);
+ String fsuffix = new String(bdata, StandardCharsets.US_ASCII);
+ LOG.debug("Create Active cluster Suffix file [{}] with Suffix: {}", idFile, fsuffix);
+ writeClusterInfo(fs, rootdir, idFile, tempIdFile, bdata, wait);
+ }
+
+ /**
+ * Writes information about this cluster to the specified file. For ex, it is used for writing
+ * cluster id in "hbase.id" file in the HBase root directory. Also, used for writing active
+ * cluster suffix in "active_cluster_suffix.id" file. If any operations on the ID file fails, and
+ * {@code wait} is a positive value, the method will retry to produce the ID file until the thread
+ * is forcibly interrupted.
+ */
+
+ private static void writeClusterInfo(final FileSystem fs, final Path rootdir, final Path idFile,
+ final Path tempIdFile, byte[] fileData, final long wait) throws IOException {
while (true) {
Optional<IOException> failure = Optional.empty();
- LOG.debug("Write the cluster ID file to a temporary location: {}", tempIdFile);
+ LOG.debug("Write the file to a temporary location: {}", tempIdFile);
try (FSDataOutputStream s = fs.create(tempIdFile)) {
- s.write(clusterId.toByteArray());
+ s.write(fileData);
} catch (IOException ioe) {
failure = Optional.of(ioe);
}
if (!failure.isPresent()) {
try {
- LOG.debug("Move the temporary cluster ID file to its target location [{}]:[{}]",
- tempIdFile, idFile);
+ LOG.debug("Move the temporary file to its target location [{}]:[{}]", tempIdFile, idFile);
if (!fs.rename(tempIdFile, idFile)) {
- failure =
- Optional.of(new IOException("Unable to move temp cluster ID file to " + idFile));
+ failure = Optional.of(new IOException("Unable to move temp file to " + idFile));
}
} catch (IOException ioe) {
failure = Optional.of(ioe);
@@ -672,8 +740,7 @@ public final class FSUtils {
if (failure.isPresent()) {
final IOException cause = failure.get();
if (wait > 0L) {
- LOG.warn("Unable to create cluster ID file in {}, retrying in {}ms", rootdir, wait,
- cause);
+ LOG.warn("Unable to create file in {}, retrying in {}ms", rootdir, wait, cause);
try {
Thread.sleep(wait);
} catch (InterruptedException e) {
@@ -1005,6 +1072,25 @@ public final class FSUtils {
return tabledirs;
}
+ /**
+ * A filter to exclude meta tables belonging to foreign clusters. This is essential in a
+ * read-replica setup where multiple clusters share the same fs.
+ * @param tablePath The Path to the table directory.
+ * @return {@code true} if the path is a regular table or the cluster's own meta table.
+ * {@code false} if it is a meta table belonging to a different cluster.
+ */
+ public static boolean isLocalMetaTable(Path tablePath) {
+ if (tablePath == null) {
+ return false;
+ }
+ String dirName = tablePath.getName();
+ if (dirName.startsWith(TableName.META_TABLE_NAME.getQualifierAsString())) {
+ return TableName.valueOf(TableName.META_TABLE_NAME.getNamespaceAsString(), dirName)
+ .equals(TableName.META_TABLE_NAME);
+ }
+ return true;
+ }
+
/**
* Filter for all dirs that don't start with '.'
*/
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index c3eafa7c11d..bfdbcb3c788 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -660,17 +660,19 @@ public class HBaseFsck extends Configured implements Closeable {
reportUnknownServers();
// Check if hbase:meta is found only once and in the right place
if (!checkMetaRegion()) {
- String errorMsg = "hbase:meta table is not consistent. ";
+ String errorMsg = TableName.META_TABLE_NAME + " table is not consistent. ";
if (shouldFixAssignments()) {
- errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
+ errorMsg += "HBCK will try fixing it. Rerun once " + TableName.META_TABLE_NAME + " is back "
+ + "to consistent state.";
} else {
- errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
+ errorMsg += "Run HBCK with proper fix options to fix " + TableName.META_TABLE_NAME
+ + " inconsistency.";
}
errors.reportError(errorMsg + " Exiting...");
return -2;
}
// Not going with further consistency check for tables when hbase:meta itself is not consistent.
- LOG.info("Loading regionsinfo from the hbase:meta table");
+ LOG.info("Loading regionsinfo from the {} table", TableName.META_TABLE_NAME);
boolean success = loadMetaEntries();
if (!success) return -1;
@@ -1219,7 +1221,7 @@ public class HBaseFsck extends Configured implements Closeable {
* TODO -- need to add tests for this.
*/
private void reportEmptyMetaCells() {
- errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: "
+ errors.print("Number of empty REGIONINFO_QUALIFIER rows in " + TableName.META_TABLE_NAME + ": "
+ emptyRegionInfoQualifiers.size());
if (details) {
for (Result r : emptyRegionInfoQualifiers) {
@@ -1371,7 +1373,7 @@ public class HBaseFsck extends Configured implements Closeable {
*/
public void fixEmptyMetaCells() throws IOException {
if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
- LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
+ LOG.info("Trying to fix empty REGIONINFO_QUALIFIER {} rows.", TableName.META_TABLE_NAME);
for (Result region : emptyRegionInfoQualifiers) {
deleteMetaRegion(region.getRow());
errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
@@ -2019,9 +2021,11 @@ public class HBaseFsck extends Configured implements Closeable {
}
RegionInfo hri = h.getRegion();
if (hri == null) {
- LOG.warn("Unable to close region " + hi.getRegionNameAsString()
- + " because hbase:meta had invalid or missing " + HConstants.CATALOG_FAMILY_STR + ":"
- + Bytes.toString(HConstants.REGIONINFO_QUALIFIER) + " qualifier value.");
+ LOG.warn(
+ "Unable to close region " + hi.getRegionNameAsString()
+ + " because {} had invalid or missing " + HConstants.CATALOG_FAMILY_STR + ":"
+ + Bytes.toString(HConstants.REGIONINFO_QUALIFIER) + " qualifier value.",
+ TableName.META_TABLE_NAME);
continue;
}
// close the region -- close files and remove assignment
@@ -2140,8 +2144,9 @@ public class HBaseFsck extends Configured implements Closeable {
assert false : "Entry for region with no data";
} else if (!inMeta && !inHdfs && isDeployed) {
errors.reportError(ERROR_CODE.NOT_IN_META_HDFS,
- "Region " + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but "
- + "deployed on " + Joiner.on(", ").join(hbi.getDeployedOn()));
+ "Region " + descriptiveName + ", key=" + key + ", not on HDFS or in "
+ + TableName.META_TABLE_NAME + " but " + "deployed on "
+ + Joiner.on(", ").join(hbi.getDeployedOn()));
if (shouldFixAssignments()) {
undeployRegions(hbi);
}
@@ -2155,8 +2160,9 @@ public class HBaseFsck extends Configured implements Closeable {
+ " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
return;
}
- errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region " + descriptiveName
- + " on HDFS, but not listed in hbase:meta " + "or deployed on any region server");
+ errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
+ "Region " + descriptiveName + " on HDFS, but not listed in " + TableName.META_TABLE_NAME
+ + " or deployed on any region server");
// restore region consistency of an adopted orphan
if (shouldFixMeta()) {
if (!hbi.isHdfsRegioninfoPresent()) {
@@ -2196,7 +2202,7 @@ public class HBaseFsck extends Configured implements Closeable {
}
}
}
- LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
+ LOG.info("Patching {} with .regioninfo: " + hbi.getHdfsHRI(), TableName.META_TABLE_NAME);
int numReplicas = admin.getDescriptor(hbi.getTableName()).getRegionReplication();
HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().keySet(),
@@ -2224,7 +2230,8 @@ public class HBaseFsck extends Configured implements Closeable {
return;
}
- LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
+ LOG.info("Patching {} with with .regioninfo: " + hbi.getHdfsHRI(),
+ TableName.META_TABLE_NAME);
int numReplicas = admin.getDescriptor(hbi.getTableName()).getRegionReplication();
HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().keySet(),
@@ -2301,9 +2308,9 @@ public class HBaseFsck extends Configured implements Closeable {
}
} else if (inMeta && inHdfs && isMultiplyDeployed) {
errors.reportError(ERROR_CODE.MULTI_DEPLOYED,
- "Region " + descriptiveName + " is listed in hbase:meta on region server "
- + hbi.getMetaEntry().regionServer + " but is multiply assigned to region servers "
- + Joiner.on(", ").join(hbi.getDeployedOn()));
+ "Region " + descriptiveName + " is listed in " + TableName.META_TABLE_NAME
+ + " on region server " + hbi.getMetaEntry().regionServer + " but is multiply assigned"
+ + " to region servers " + Joiner.on(", ").join(hbi.getDeployedOn()));
// If we are trying to fix the errors
if (shouldFixAssignments()) {
errors.print("Trying to fix assignment error...");
@@ -2313,8 +2320,8 @@ public class HBaseFsck extends Configured implements Closeable {
}
} else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META,
- "Region " + descriptiveName + " listed in hbase:meta on region server "
- + hbi.getMetaEntry().regionServer + " but found on region server "
+ "Region " + descriptiveName + " listed in " + TableName.META_TABLE_NAME
+ + " on region server " + hbi.getMetaEntry().regionServer + " but found on region server "
+ hbi.getDeployedOn().get(0));
// If we are trying to fix the errors
if (shouldFixAssignments()) {
@@ -2614,11 +2621,12 @@ public class HBaseFsck extends Configured implements Closeable {
if (servers.isEmpty()) {
assignMetaReplica(i);
} else if (servers.size() > 1) {
- errors.reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta, replicaId "
- + metaHbckRegionInfo.getReplicaId() + " is found on more than one region.");
+ errors.reportError(ERROR_CODE.MULTI_META_REGION,
+ TableName.META_TABLE_NAME + ", replicaId " + metaHbckRegionInfo.getReplicaId()
+ + " is found on more than one region.");
if (shouldFixAssignments()) {
- errors.print("Trying to fix a problem with hbase:meta, replicaId "
- + metaHbckRegionInfo.getReplicaId() + "..");
+ errors.print("Trying to fix a problem with " + TableName.META_TABLE_NAME
+ + ", replicaId " + metaHbckRegionInfo.getReplicaId() + "..");
setShouldRerun();
// try fix it (treat is a dupe assignment)
HBaseFsckRepair.fixMultiAssignment(connection,
@@ -2631,11 +2639,11 @@ public class HBaseFsck extends Configured implements Closeable {
for (Map.Entry<Integer, HbckRegionInfo> entry : metaRegions.entrySet()) {
noProblem = false;
errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
- "hbase:meta replicas are deployed in excess. Configured " + metaReplication + ", deployed "
- + metaRegions.size());
+ TableName.META_TABLE_NAME + " replicas are deployed in excess. Configured "
+ + metaReplication + ", deployed " + metaRegions.size());
if (shouldFixAssignments()) {
- errors.print(
- "Trying to undeploy excess replica, replicaId: " + entry.getKey() + " of hbase:meta..");
+ errors.print("Trying to undeploy excess replica, replicaId: " + entry.getKey() + " of "
+ + TableName.META_TABLE_NAME + "..");
setShouldRerun();
unassignMetaReplica(entry.getValue());
}
@@ -2655,9 +2663,9 @@ public class HBaseFsck extends Configured implements Closeable {
private void assignMetaReplica(int replicaId)
throws IOException, KeeperException, InterruptedException {
errors.reportError(ERROR_CODE.NO_META_REGION,
- "hbase:meta, replicaId " + replicaId + " is not found on any region.");
+ TableName.META_TABLE_NAME + ", replicaId " + replicaId + " is not found on any region.");
if (shouldFixAssignments()) {
- errors.print("Trying to fix a problem with hbase:meta..");
+ errors.print("Trying to fix a problem with " + TableName.META_TABLE_NAME + "..");
setShouldRerun();
// try to fix it (treat it as unassigned region)
RegionInfo h = RegionReplicaUtil
@@ -2693,7 +2701,7 @@ public class HBaseFsck extends Configured implements Closeable {
if (rl == null) {
emptyRegionInfoQualifiers.add(result);
errors.reportError(ERROR_CODE.EMPTY_META_CELL,
- "Empty REGIONINFO_QUALIFIER found in hbase:meta");
+ "Empty REGIONINFO_QUALIFIER found in " + TableName.META_TABLE_NAME);
return true;
}
ServerName sn = null;
@@ -2703,7 +2711,7 @@ public class HBaseFsck extends Configured implements Closeable {
) {
emptyRegionInfoQualifiers.add(result);
errors.reportError(ERROR_CODE.EMPTY_META_CELL,
- "Empty REGIONINFO_QUALIFIER found in hbase:meta");
+ "Empty REGIONINFO_QUALIFIER found in " + TableName.META_TABLE_NAME);
return true;
}
RegionInfo hri = rl.getRegionLocation(RegionInfo.DEFAULT_REPLICA_ID).getRegion();
@@ -2731,7 +2739,8 @@ public class HBaseFsck extends Configured implements Closeable {
} else if (previous.getMetaEntry() == null) {
previous.setMetaEntry(m);
} else {
- throw new IOException("Two entries in hbase:meta are same " + previous);
+ throw new IOException(
+ "Two entries in " + TableName.META_TABLE_NAME + " are same " + previous);
}
}
List<RegionInfo> mergeParents = CatalogFamilyFormat.getMergeRegions(result.rawCells());
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java
index c1f98edd75a..5f7e62bd5bc 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.UnknownRegionException;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
@@ -586,13 +587,13 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
// For isolating hbase:meta, it should move explicitly in Ack mode,
// hence the forceMoveRegionByAck = true.
if (!metaSeverName.equals(server)) {
- LOG.info("Region of hbase:meta " + metaRegionInfo.getEncodedName() + " is on server "
- + metaSeverName + " moving to " + server);
+ LOG.info("Region of {} {} is on server {} moving to {}", TableName.META_TABLE_NAME,
+ metaRegionInfo.getEncodedName(), metaSeverName, server);
submitRegionMovesWhileUnloading(metaSeverName, Collections.singletonList(server),
movedRegions, Collections.singletonList(metaRegionInfo), true);
} else {
- LOG.info("Region of hbase:meta " + metaRegionInfo.getEncodedName() + " already exists"
- + " on server : " + server);
+ LOG.info("Region of {} {} already exists on server: {}", TableName.META_TABLE_NAME,
+ metaRegionInfo.getEncodedName(), server);
}
isolateRegionInfoList.add(RegionInfoBuilder.FIRST_META_REGIONINFO);
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableForReplica.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableForReplica.java
new file mode 100644
index 00000000000..7c937eb5c05
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableForReplica.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test {@link org.apache.hadoop.hbase.TestMetaTableForReplica}.
+ */
+@Category({ MiscTests.class, MediumTests.class })
+@SuppressWarnings("deprecation")
+public class TestMetaTableForReplica {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestMetaTableForReplica.class);
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestMetaTableForReplica.class);
+ private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
+ private static Connection connection;
+ private static Field metaTableName;
+ private static Object originalMetaTableName;
+
+ @Rule
+ public TestName name = new TestName();
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ Configuration c = UTIL.getConfiguration();
+ // quicker heartbeat interval for faster DN death notification
+ c.setInt("hbase.ipc.client.connect.max.retries", 1);
+ c.setInt(HConstants.ZK_SESSION_TIMEOUT, 1000);
+ // Start cluster having non-default hbase meta table name
+ UTIL.startMiniCluster(3);
+ connection = ConnectionFactory.createConnection(c);
+ // Save the original value of META_TABLE_NAME before any test runs.x
+ metaTableName = TableName.class.getDeclaredField("META_TABLE_NAME");
+ originalMetaTableName = metaTableName.get(null);
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ connection.close();
+ UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testStateOfMetaForReplica() {
+ HMaster m = UTIL.getMiniHBaseCluster().getMaster();
+ assertTrue(m.waitForMetaOnline());
+ }
+
+ @Test
+ public void testMetaTableNameForReplicaWithoutSuffix() throws IOException {
+ testNameOfMetaForReplica();
+ testGetNonExistentRegionFromMetaFromReplica();
+ testGetExistentRegionFromMetaFromReplica();
+ }
+
+ private void testNameOfMetaForReplica() {
+ // Check the correctness of the meta table for replica
+ String metaTableName = TableName.META_TABLE_NAME.getNameWithNamespaceInclAsString();
+ assertNotNull(metaTableName);
+
+ // Check if name of the meta table for replica is same as the default meta table
+ assertEquals(0,
+ TableName.META_TABLE_NAME.compareTo(TableName.getDefaultNameOfMetaForReplica()));
+ }
+
+ private void testGetNonExistentRegionFromMetaFromReplica() throws IOException {
+ final String name = this.name.getMethodName();
+ LOG.info("Started " + name);
+ Pair<RegionInfo, ServerName> pair =
+ MetaTableAccessor.getRegion(connection, Bytes.toBytes("nonexistent-region"));
+ assertNull(pair);
+ LOG.info("Finished " + name);
+ }
+
+ private void testGetExistentRegionFromMetaFromReplica() throws IOException {
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ LOG.info("Started " + tableName);
+ UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
+ assertEquals(1, MetaTableAccessor.getTableRegions(connection, tableName).size());
+ }
+
+ @Test
+ public void testMetaTableNameForReplicaWithSuffix() throws Exception {
+ // This test actively changes the META_TABLE_NAME to a non-default value and verifies it.
+ Configuration conf = HBaseConfiguration.create();
+ String suffix = "replica1";
+ conf.set(HConstants.HBASE_META_TABLE_SUFFIX, suffix);
+
+ // Re-initialize the static final META_TABLE_NAME for the testing to a non-default value.
+ TableName expectedMetaTableName = TableName.initializeHbaseMetaTableName(conf);
+ setStaticFinalField(metaTableName, expectedMetaTableName);
+
+ TableName currentMetaName = TableName.META_TABLE_NAME;
+ TableName defaultMetaName = TableName.getDefaultNameOfMetaForReplica();
+
+ // The current meta table name is not the default one.
+ assertNotEquals("META_TABLE_NAME should not be the default. ", defaultMetaName,
+ currentMetaName);
+
+ // The current meta table name has the configured suffix.
+ assertEquals("META_TABLE_NAME should have the configured suffix", expectedMetaTableName,
+ currentMetaName);
+
+ // restore default value of META_TABLE_NAME
+ setDefaultMetaTableName();
+ }
+
+ private static void setDefaultMetaTableName() throws Exception {
+ if (originalMetaTableName != null) {
+ setStaticFinalField(metaTableName, originalMetaTableName);
+ }
+ }
+
+ /**
+ * A helper method to modify a static final field using reflection. This is necessary for testing
+ * code that reads a configuration only once during class loading.
+ * @param field The field to modify.
+ * @param newValue The new value to set.
+ * @throws Exception if reflection fails.
+ */
+ private static void setStaticFinalField(Field field, Object newValue) throws Exception {
+ field.setAccessible(true);
+ // Using MethodHandles to get a trusted lookup with the necessary permissions to modify it.
+ // NOTE: For this to work, the JVM running the test must be started with arguments like:
+ // --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
+ var lookup = MethodHandles.privateLookupIn(Field.class, MethodHandles.lookup());
+ var handle = lookup.findVarHandle(Field.class, "modifiers", int.class);
+ handle.set(field, field.getModifiers() & ~Modifier.FINAL);
+ field.set(null, newValue);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRefreshHFilesBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRefreshHFilesBase.java
new file mode 100644
index 00000000000..5338b7a6260
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRefreshHFilesBase.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import static org.apache.hadoop.hbase.HConstants.HBASE_CLIENT_RETRIES_NUMBER;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.RefreshHFilesTableProcedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.JVMClusterUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestRefreshHFilesBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestRefreshHFilesBase.class);
+
+ protected static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
+ protected Admin admin;
+ protected HMaster master;
+ List<HRegionServer> regionServers;
+ protected SingleProcessHBaseCluster cluster;
+ protected ProcedureExecutor<MasterProcedureEnv> procExecutor;
+ protected static Configuration conf;
+ protected static final TableName TEST_TABLE = TableName.valueOf("testRefreshHFilesTable");
+ protected static final String TEST_NAMESPACE = "testRefreshHFilesNamespace";
+ protected static final byte[] TEST_FAMILY = Bytes.toBytes("testRefreshHFilesCF1");
+
+ protected void createTableAndWait(TableName table, byte[] cf)
+ throws IOException, InterruptedException {
+ TEST_UTIL.createTable(table, cf);
+ TEST_UTIL.waitTableAvailable(table);
+ }
+
+ protected void createTableInNamespaceAndWait(String namespace, TableName table, byte[] cf)
+ throws IOException, InterruptedException {
+ TableName fqTableName = TableName.valueOf(namespace + table.getNameAsString());
+ TEST_UTIL.createTable(fqTableName, cf);
+ TEST_UTIL.waitTableAvailable(fqTableName);
+ }
+
+ protected void deleteTable(TableName table) throws IOException {
+ TEST_UTIL.deleteTableIfAny(table);
+ }
+
+ protected void createNamespace(String namespace) throws RuntimeException {
+ try {
+ final NamespaceDescriptor nsd = NamespaceDescriptor.create(namespace).build();
+ // Create the namespace if it doesn’t exist
+ if (
+ Arrays.stream(admin.listNamespaceDescriptors())
+ .noneMatch(ns -> ns.getName().equals(namespace))
+ ) {
+ admin.createNamespace(nsd);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected void deleteNamespace(String namespace) {
+ try {
+ // List table in namespace
+ TableName[] tables = admin.listTableNamesByNamespace(namespace);
+ for (TableName t : tables) {
+ TEST_UTIL.deleteTableIfAny(t);
+ }
+ // Now delete the namespace
+ admin.deleteNamespace(namespace);
+ } catch (Exception e) {
+ LOG.debug(
+ "Unable to delete namespace " + namespace + " post test execution. This isn't a failure");
+ }
+ }
+
+ protected void submitProcedureAndAssertNotFailed(RefreshHFilesTableProcedure procedure) {
+ long procId = procExecutor.submitProcedure(procedure);
+ ProcedureTestingUtility.waitProcedure(procExecutor, procId);
+ ProcedureTestingUtility.assertProcNotFailed(procExecutor.getResult(procId));
+ }
+
+ protected void setReadOnlyMode(boolean isReadOnly) {
+ conf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, isReadOnly);
+ notifyConfigurationObservers();
+ }
+
+ private void notifyConfigurationObservers() {
+ master.getConfigurationManager().notifyAllObservers(TEST_UTIL.getConfiguration());
+ for (HRegionServer rs : regionServers) {
+ rs.getConfigurationManager().notifyAllObservers(TEST_UTIL.getConfiguration());
+ }
+ }
+
+ private void setupReadOnlyConf(boolean addReadOnlyConf) {
+ if (!addReadOnlyConf) return;
+ // Keep ReadOnly property to false at the beginning so that create table succeed.
+ conf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, false);
+ }
+
+ protected void baseSetup(boolean addReadOnlyConf) throws Exception {
+ conf = TEST_UTIL.getConfiguration();
+ // Shorten the run time of failed unit tests by limiting retries and the session timeout
+ // threshold
+ conf.setInt(HBASE_CLIENT_RETRIES_NUMBER, 1);
+ conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 1000);
+ conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 60000);
+ conf.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 120000);
+
+ setupReadOnlyConf(addReadOnlyConf);
+
+ try {
+ // Start the test cluster
+ cluster = TEST_UTIL.startMiniCluster(1);
+ admin = TEST_UTIL.getAdmin();
+ procExecutor = TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
+ master = TEST_UTIL.getHBaseCluster().getMaster();
+ regionServers = cluster.getRegionServerThreads().stream()
+ .map(JVMClusterUtil.RegionServerThread::getRegionServer).collect(Collectors.toList());
+ } catch (Exception e) {
+ TEST_UTIL.shutdownMiniCluster();
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected void baseTearDown() throws Exception {
+ if (admin != null) {
+ admin.close();
+ }
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRefreshHFilesFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRefreshHFilesFromClient.java
new file mode 100644
index 00000000000..c134c6c9804
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRefreshHFilesFromClient.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.NamespaceNotFoundException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.TestRefreshHFilesBase;
+import org.apache.hadoop.hbase.testclassification.ClientTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MediumTests.class, ClientTests.class })
+public class TestRefreshHFilesFromClient extends TestRefreshHFilesBase {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestRefreshHFilesFromClient.class);
+
+ private static final TableName TEST_NONEXISTENT_TABLE =
+ TableName.valueOf("testRefreshHFilesNonExistentTable");
+ private static final String TEST_NONEXISTENT_NAMESPACE = "testRefreshHFilesNonExistentNamespace";
+
+ @Before
+ public void setup() throws Exception {
+ baseSetup(false);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ baseTearDown();
+ }
+
+ @Test
+ public void testRefreshHFilesForTable() throws Exception {
+ try {
+ // Create table in default namespace
+ createTableAndWait(TEST_TABLE, TEST_FAMILY);
+
+ // RefreshHFiles for table
+ Long procId = admin.refreshHFiles(TEST_TABLE);
+ assertTrue(procId >= 0);
+ } catch (Exception e) {
+ Assert.fail("RefreshHFilesForTable Should Not Throw Exception: " + e);
+ throw new RuntimeException(e);
+ } finally {
+ // Delete table name post test execution
+ deleteTable(TEST_TABLE);
+ }
+ }
+
+ // Not creating table hence refresh should throw exception
+ @Test(expected = TableNotFoundException.class)
+ public void testRefreshHFilesForNonExistentTable() throws Exception {
+ // RefreshHFiles for table
+ admin.refreshHFiles(TEST_NONEXISTENT_TABLE);
+ }
+
+ @Test
+ public void testRefreshHFilesForNamespace() throws Exception {
+ try {
+ createNamespace(TEST_NAMESPACE);
+
+ // Create table under test namespace
+ createTableInNamespaceAndWait(TEST_NAMESPACE, TEST_TABLE, TEST_FAMILY);
+
+ // RefreshHFiles for namespace
+ Long procId = admin.refreshHFiles(TEST_NAMESPACE);
+ assertTrue(procId >= 0);
+
+ } catch (Exception e) {
+ Assert.fail("RefreshHFilesForAllNamespace Should Not Throw Exception: " + e);
+ throw new RuntimeException(e);
+ } finally {
+ // Delete namespace post test execution
+ // This will delete all tables under namespace hence no explicit table
+ // deletion for table under namespace is needed.
+ deleteNamespace(TEST_NAMESPACE);
+ }
+ }
+
+ @Test(expected = NamespaceNotFoundException.class)
+ public void testRefreshHFilesForNonExistentNamespace() throws Exception {
+ // RefreshHFiles for namespace
+ admin.refreshHFiles(TEST_NONEXISTENT_NAMESPACE);
+ }
+
+ @Test
+ public void testRefreshHFilesForAllTables() throws Exception {
+ try {
+ // Create table in default namespace
+ createTableAndWait(TEST_TABLE, TEST_FAMILY);
+
+ // Create test namespace
+ createNamespace(TEST_NAMESPACE);
+
+ // Create table under test namespace
+ createTableInNamespaceAndWait(TEST_NAMESPACE, TEST_TABLE, TEST_FAMILY);
+
+ // RefreshHFiles for all the tables
+ Long procId = admin.refreshHFiles();
+ assertTrue(procId >= 0);
+
+ } catch (Exception e) {
+ Assert.fail("RefreshHFilesForAllTables Should Not Throw Exception: " + e);
+ throw new RuntimeException(e);
+ } finally {
+ // Delete table name post test execution
+ deleteTable(TEST_TABLE);
+
+ // Delete namespace post test execution
+ // This will delete all tables under namespace hence no explicit table
+ // deletion for table under namespace is needed.
+ deleteNamespace(TEST_NAMESPACE);
+ }
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
index 09618b3d899..409550f5fd5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
@@ -17,22 +17,41 @@
*/
package org.apache.hadoop.hbase.master;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
import java.io.IOException;
+import java.io.InterruptedIOException;
import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.EnumSet;
import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.ClusterMetrics;
import org.apache.hadoop.hbase.CompatibilityFactory;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.ServerMetricsBuilder;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
import org.apache.hadoop.hbase.StartTestingClusterOption;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.YouAreDeadException;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.RegionStatesCount;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.test.MetricsAssertHelper;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.zookeeper.KeeperException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -61,6 +80,7 @@ public class TestMasterMetrics {
private static final Logger LOG = LoggerFactory.getLogger(TestMasterMetrics.class);
private static final MetricsAssertHelper metricsHelper =
CompatibilityFactory.getInstance(MetricsAssertHelper.class);
+ private static final String COLUMN_FAMILY = "cf";
private static SingleProcessHBaseCluster cluster;
private static HMaster master;
@@ -188,8 +208,187 @@ public class TestMasterMetrics {
}
@Test
- public void testDefaultMasterProcMetrics() throws Exception {
+ public void testDefaultMasterProcMetrics() {
MetricsMasterProcSource masterSource = master.getMasterMetrics().getMetricsProcSource();
metricsHelper.assertGauge("numMasterWALs", master.getNumWALFiles(), masterSource);
}
+
+ // Verifies a foreign meta table does not show up in the table regions state
+ @Test
+ public void testClusterMetricsSkippingForeignMetaTable() throws Exception {
+ TableName replicaMetaTable = TableName.valueOf("hbase", "meta_replica");
+ TableDescriptor replicaMetaDescriptor = TableDescriptorBuilder.newBuilder(replicaMetaTable)
+ .setColumnFamily(ColumnFamilyDescriptorBuilder.of("info")).build();
+ master.getTableDescriptors().update(replicaMetaDescriptor, true);
+ try {
+ Map<TableName, RegionStatesCount> tableRegionStatesCount = getTableRegionStatesCount();
+
+ assertFalse("Foreign meta table should not be present",
+ tableRegionStatesCount.containsKey(replicaMetaTable));
+ assertTrue("Local meta should be present",
+ tableRegionStatesCount.containsKey(TableName.META_TABLE_NAME));
+
+ } finally {
+ master.getTableDescriptors().remove(replicaMetaTable);
+ }
+ }
+
+ // This test adds foreign file descriptors to the cluster's table descriptor cache. It then
+ // verifies the foreign tables do not show up in the table regions state.
+ @Test
+ public void testClusterMetricsSkippingCachedForeignTables() throws Exception {
+ List<TableName> allTables = new ArrayList<>();
+
+ // These tables, including the cluster's meta table, should not be foreign to the cluster.
+ // The cluster should be able to find their state.
+ allTables.add(TableName.META_TABLE_NAME);
+ List<TableName> familiarTables = new ArrayList<>();
+ familiarTables.add(TableName.valueOf(null, "familiarTable1"));
+ familiarTables.add(TableName.valueOf("", "familiarTable2"));
+ familiarTables.add(TableName.valueOf("default", "familiarTable3"));
+ familiarTables.add(TableName.valueOf("familiarNamespace", "familiarTable4"));
+ familiarTables.add(TableName.valueOf("familiarNamespace", "familiarTable5"));
+
+ // Create these "familiar" tables so their state can be found
+ TEST_UTIL.getAdmin().createNamespace(NamespaceDescriptor.create("familiarNamespace").build());
+ for (TableName familiarTable : familiarTables) {
+ TEST_UTIL.createTable(familiarTable, COLUMN_FAMILY);
+ allTables.add(familiarTable);
+ }
+
+ // hbase:meta is a familiar table that was created automatically
+ familiarTables.add(TableName.META_TABLE_NAME);
+
+ // These tables should be foreign to the cluster.
+ // The cluster should not be able to find their state.
+ allTables.add(TableName.valueOf("hbase", "meta_replica"));
+ allTables.add(TableName.valueOf(null, "defaultNamespaceTable1"));
+ allTables.add(TableName.valueOf("", "defaultNamespaceTable2"));
+ allTables.add(TableName.valueOf("default", "defaultNamespaceTable3"));
+ allTables.add(TableName.valueOf("customNamespace", "customNamespaceTable1"));
+ allTables.add(TableName.valueOf("customNamespace", "customNamespaceTable2"));
+ allTables.add(TableName.valueOf("anotherNamespace", "anotherNamespaceTable"));
+ allTables.add(TableName.valueOf("sharedNamespace", "sharedNamespaceTable1"));
+ allTables.add(TableName.valueOf("sharedNamespace", "sharedNamespaceTable2"));
+
+ // Update master's table descriptors to have all tables
+ TableDescriptor foreignTableDescriptor;
+ for (TableName tableName : allTables) {
+ foreignTableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
+ .setColumnFamily(ColumnFamilyDescriptorBuilder.of(COLUMN_FAMILY)).build();
+ master.getTableDescriptors().update(foreignTableDescriptor, true);
+ }
+
+ // The state of the meta table and the familiar tables we created should exist.
+ // The other tables' state should not exist.
+ for (TableName tableName : allTables) {
+ try {
+ Map<TableName, RegionStatesCount> tableRegionStatesCount = getTableRegionStatesCount();
+
+ if (
+ tableName.equals(TableName.META_TABLE_NAME)
+ || tableName.getQualifierAsString().startsWith("familiar")
+ ) {
+ assertTrue("Expected this table's state to exist: " + tableName,
+ tableRegionStatesCount.containsKey(tableName));
+ } else {
+ assertFalse("This foreign table's state should not exist: " + tableName,
+ tableRegionStatesCount.containsKey(tableName));
+ }
+ } finally {
+ if (!TableName.META_TABLE_NAME.equals(tableName) && familiarTables.contains(tableName)) {
+ LOG.debug("Deleting table: {}", tableName);
+ TEST_UTIL.deleteTable(tableName);
+ } else if (!familiarTables.contains(tableName)) {
+ LOG.debug("Removing table descriptor for foreign table: {}", tableName);
+ master.getTableDescriptors().remove(tableName);
+ }
+ }
+ }
+ }
+
+ // This test creates foreign file descriptors on the filesystem in addition to updating the
+ // table descriptor cache. It then verifies the foreign tables do not show up in the
+ // table regions state.
+ @Test
+ public void testClusterMetricsSkippingForeignTablesOnFileSystem() throws IOException {
+ List<TableName> familiarTables = new ArrayList<>();
+ List<TableName> foreignTables = new ArrayList<>();
+ FileSystem fs = TEST_UTIL.getTestFileSystem();
+ Path testDir = TEST_UTIL.getDataTestDirOnTestFS();
+ LOG.info("The test dir is: {}", testDir);
+
+ // Create tables whose state are familiar to this cluster
+ familiarTables.add(TableName.valueOf("testTable1"));
+ familiarTables.add(TableName.valueOf("testTable2"));
+ TEST_UTIL.getAdmin().createNamespace(NamespaceDescriptor.create("myNamespace").build());
+ familiarTables.add(TableName.valueOf("myNamespace", "tableWithNamespace1"));
+ for (TableName tableName : familiarTables) {
+ TEST_UTIL.createTable(tableName, COLUMN_FAMILY);
+ }
+
+ // hbase:meta is a familiar table that was created automatically
+ familiarTables.add(TableName.META_TABLE_NAME);
+
+ // There should now be 4 tables, including hbase:meta
+ Map<String, TableDescriptor> tableDescriptorMap = master.getTableDescriptors().getAll();
+ assertEquals(4, tableDescriptorMap.size());
+ for (TableName tableName : familiarTables) {
+ assertTrue("Expected table descriptor map to contain table: " + tableName, tableDescriptorMap
+ .containsKey(tableName.getNamespaceAsString() + ":" + tableName.getQualifierAsString()));
+ }
+
+ createTableDescriptorOnFileSystem("hbase", "meta_replica", foreignTables);
+ createTableDescriptorOnFileSystem("default", "foreignTable1", foreignTables);
+ createTableDescriptorOnFileSystem("customForeignNs", "customForeignNsTable1", foreignTables);
+
+ // Verify the table descriptors were created on the filesystem
+ for (TableName tableName : foreignTables) {
+ Path tableDescPath = new Path(testDir,
+ "data" + Path.SEPARATOR + tableName.getNamespaceAsString() + Path.SEPARATOR
+ + tableName.getQualifierAsString() + Path.SEPARATOR + FSTableDescriptors.TABLEINFO_DIR);
+ assertTrue("Expected table descriptor directory to exist: " + tableDescPath,
+ fs.exists(tableDescPath));
+ }
+
+ Map<TableName, RegionStatesCount> tableRegionStatesCount = getTableRegionStatesCount();
+
+ // The foreign tables should not be in the table state
+ assertEquals(4, tableRegionStatesCount.size());
+ for (TableName tableName : familiarTables) {
+ assertTrue("Expected table regions state count to contain: " + tableName,
+ tableRegionStatesCount.containsKey(tableName));
+ // Delete unneeded tables
+ if (!TableName.META_TABLE_NAME.equals(tableName)) {
+ LOG.debug("Deleting table: {}", tableName);
+ TEST_UTIL.deleteTable(tableName);
+ }
+ }
+ for (TableName tableName : foreignTables) {
+ assertFalse("Expected table regions state count to NOT contain: " + tableName,
+ tableRegionStatesCount.containsKey(tableName));
+ // Remove unneeded table descriptors
+ LOG.debug("Removing table descriptor for foreign table: {}", tableName);
+ master.getTableDescriptors().remove(tableName);
+ }
+ }
+
+ private Map<TableName, RegionStatesCount> getTableRegionStatesCount()
+ throws InterruptedIOException {
+ ClusterMetrics metrics = master.getClusterMetricsWithoutCoprocessor(
+ EnumSet.of(ClusterMetrics.Option.TABLE_TO_REGIONS_COUNT));
+ return metrics.getTableRegionStatesCount();
+ }
+
+ private void createTableDescriptorOnFileSystem(String namespace, String qualifier,
+ List<TableName> tableNameList) throws IOException {
+ // Create a table descriptor on the filesystem that uses the provided namespace
+ TableName tableName = TableName.valueOf(namespace, qualifier);
+ TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
+ .setColumnFamily(ColumnFamilyDescriptorBuilder.of(COLUMN_FAMILY)).build();
+ // Create the table descriptor on the filesystem and update the file descriptor cache
+ master.getTableDescriptors().update(tableDescriptor, false);
+
+ tableNameList.add(tableName);
+ }
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestHbckChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestHbckChore.java
index 70afeae4c6e..4282f4e38f2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestHbckChore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestHbckChore.java
@@ -31,20 +31,28 @@ import java.util.Map;
import java.util.Optional;
import java.util.concurrent.Future;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.master.TableStateManager;
import org.apache.hadoop.hbase.master.hbck.HbckChore;
import org.apache.hadoop.hbase.master.hbck.HbckReport;
+import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.junit.Before;
@@ -247,4 +255,39 @@ public class TestHbckChore extends TestAssignmentManagerBase {
hbckChoreWithChangedConf.choreForTesting();
assertNull(hbckChoreWithChangedConf.getLastReport());
}
+
+ @Test
+ public void testChoreSkipsForeignMetaTables() throws Exception {
+ FileSystem fs = master.getMasterFileSystem().getFileSystem();
+ Path rootDir = master.getMasterFileSystem().getRootDir();
+ String[] metaTables = { "meta_replica1", "meta" };
+ Path hbaseNamespaceDir = new Path(rootDir, HConstants.BASE_NAMESPACE_DIR + "/hbase");
+ fs.mkdirs(hbaseNamespaceDir);
+
+ for (String metaTable : metaTables) {
+ TableName tableName = TableName.valueOf("hbase", metaTable);
+ Path metaTableDir = new Path(hbaseNamespaceDir, metaTable);
+ fs.mkdirs(metaTableDir);
+ fs.mkdirs(new Path(metaTableDir, FSTableDescriptors.TABLEINFO_DIR));
+ fs.mkdirs(new Path(metaTableDir, "abcdef0123456789"));
+
+ TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
+ .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(HConstants.CATALOG_FAMILY)
+ .setMaxVersions(HConstants.DEFAULT_HBASE_META_VERSIONS).setInMemory(true)
+ .setBlocksize(HConstants.DEFAULT_HBASE_META_BLOCK_SIZE)
+ .setBloomFilterType(BloomType.ROWCOL).build())
+ .build();
+
+ Path tableDir = CommonFSUtils.getTableDir(rootDir, tableName);
+ FSTableDescriptors.createTableDescriptorForTableDirectory(fs, tableDir, tableDescriptor,
+ false);
+ }
+
+ assertTrue("HbckChore should run successfully", hbckChore.runChore());
+ HbckReport report = hbckChore.getLastReport();
+ assertNotNull("HbckReport should not be null", report);
+ boolean hasForeignMetaOrphan = report.getOrphanRegionsOnFS().values().stream()
+ .anyMatch(path -> path.toString().contains("meta_replica1"));
+ assertFalse("HbckChore should not report foreign meta tables as orphans", hasForeignMetaOrphan);
+ }
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRefreshHFilesProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRefreshHFilesProcedure.java
new file mode 100644
index 00000000000..9bb7bf181c3
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRefreshHFilesProcedure.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.procedure;
+
+import java.io.IOException;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.TestRefreshHFilesBase;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MasterTests.class, MediumTests.class })
+public class TestRefreshHFilesProcedure extends TestRefreshHFilesBase {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestRefreshHFilesProcedure.class);
+
+ @Before
+ public void setup() throws Exception {
+ baseSetup(false);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ baseTearDown();
+ }
+
+ @Test
+ public void testRefreshHFilesProcedureForTable() throws IOException {
+ try {
+ // Create table in default namespace
+ createTableAndWait(TEST_TABLE, TEST_FAMILY);
+
+ RefreshHFilesTableProcedure procedure =
+ new RefreshHFilesTableProcedure(procExecutor.getEnvironment(), TEST_TABLE);
+ submitProcedureAndAssertNotFailed(procedure);
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ // Delete table name post test execution
+ deleteTable(TEST_TABLE);
+ }
+ }
+
+ @Test
+ public void testRefreshHFilesProcedureForNamespace() {
+ try {
+ createNamespace(TEST_NAMESPACE);
+
+ // Create table under test namespace
+ createTableInNamespaceAndWait(TEST_NAMESPACE, TEST_TABLE, TEST_FAMILY);
+
+ RefreshHFilesTableProcedure procedure =
+ new RefreshHFilesTableProcedure(procExecutor.getEnvironment(), TEST_NAMESPACE);
+ submitProcedureAndAssertNotFailed(procedure);
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ // Delete namespace post test execution
+ // This will delete all tables under namespace hence no explicit table
+ // deletion for table under namespace is needed.
+ deleteNamespace(TEST_NAMESPACE);
+ }
+ }
+
+ @Test
+ public void testRefreshHFilesProcedureForAllTables() throws IOException {
+ try {
+ // Create table in default namespace
+ createTableAndWait(TEST_TABLE, TEST_FAMILY);
+
+ // Create test namespace
+ createNamespace(TEST_NAMESPACE);
+
+ // Create table under test namespace
+ createTableInNamespaceAndWait(TEST_NAMESPACE, TEST_TABLE, TEST_FAMILY);
+
+ RefreshHFilesTableProcedure procedure =
+ new RefreshHFilesTableProcedure(procExecutor.getEnvironment());
+ submitProcedureAndAssertNotFailed(procedure);
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ // Delete table name post test execution
+ deleteTable(TEST_TABLE);
+
+ // Delete namespace post test execution
+ // This will delete all tables under namespace hence no explicit table
+ // deletion for table under namespace is needed.
+ deleteNamespace(TEST_NAMESPACE);
+ }
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRefreshHFilesProcedureWithReadOnlyConf.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRefreshHFilesProcedureWithReadOnlyConf.java
new file mode 100644
index 00000000000..351081d04fc
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRefreshHFilesProcedureWithReadOnlyConf.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.procedure;
+
+import java.io.IOException;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.TestRefreshHFilesBase;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MasterTests.class, MediumTests.class })
+public class TestRefreshHFilesProcedureWithReadOnlyConf extends TestRefreshHFilesBase {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestRefreshHFilesProcedureWithReadOnlyConf.class);
+
+ @Before
+ public void setup() throws Exception {
+ // When true is passed only setup for readonly property is done.
+ // The initial ReadOnly property will be false for table creation
+ baseSetup(true);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ baseTearDown();
+ }
+
+ @Test
+ public void testRefreshHFilesProcedureForTable() throws IOException {
+ try {
+ // Create table in default namespace
+ createTableAndWait(TEST_TABLE, TEST_FAMILY);
+
+ setReadOnlyMode(true);
+ RefreshHFilesTableProcedure procedure =
+ new RefreshHFilesTableProcedure(procExecutor.getEnvironment(), TEST_TABLE);
+ submitProcedureAndAssertNotFailed(procedure);
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ setReadOnlyMode(false);
+ // Delete table name post test execution
+ deleteTable(TEST_TABLE);
+ }
+ }
+
+ @Test
+ public void testRefreshHFilesProcedureForNamespace() {
+ try {
+ createNamespace(TEST_NAMESPACE);
+
+ // Create table under test namespace
+ createTableInNamespaceAndWait(TEST_NAMESPACE, TEST_TABLE, TEST_FAMILY);
+
+ setReadOnlyMode(true);
+ RefreshHFilesTableProcedure procedure =
+ new RefreshHFilesTableProcedure(procExecutor.getEnvironment(), TEST_NAMESPACE);
+ submitProcedureAndAssertNotFailed(procedure);
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ setReadOnlyMode(false);
+ // Delete namespace post test execution
+ // This will delete all tables under namespace hence no explicit table
+ // deletion for table under namespace is needed.
+ deleteNamespace(TEST_NAMESPACE);
+ }
+ }
+
+ @Test
+ public void testRefreshHFilesProcedureForAllTables() throws IOException {
+ try {
+ // Create table in default namespace
+ createTableAndWait(TEST_TABLE, TEST_FAMILY);
+
+ // Create test namespace
+ createNamespace(TEST_NAMESPACE);
+
+ // Create table under test namespace
+ createTableInNamespaceAndWait(TEST_NAMESPACE, TEST_TABLE, TEST_FAMILY);
+
+ setReadOnlyMode(true);
+ RefreshHFilesTableProcedure procedure =
+ new RefreshHFilesTableProcedure(procExecutor.getEnvironment());
+ submitProcedureAndAssertNotFailed(procedure);
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ setReadOnlyMode(false);
+ // Delete table name post test execution
+ deleteTable(TEST_TABLE);
+
+ // Delete namespace post test execution
+ // This will delete all tables under namespace hence no explicit table
+ // deletion for table under namespace is needed.
+ deleteNamespace(TEST_NAMESPACE);
+ }
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRefreshMetaProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRefreshMetaProcedure.java
new file mode 100644
index 00000000000..e419d1df6ad
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRefreshMetaProcedure.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.procedure;
+
+import static org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.assertProcNotFailed;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MasterTests.class, MediumTests.class })
+public class TestRefreshMetaProcedure {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestRefreshMetaProcedure.class);
+
+ private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
+ private ProcedureExecutor<MasterProcedureEnv> procExecutor;
+ List<RegionInfo> activeRegions;
+ TableName tableName = TableName.valueOf("testRefreshMeta");
+
+ @Before
+ public void setup() throws Exception {
+ TEST_UTIL.getConfiguration().set("USE_META_REPLICAS", "false");
+ TEST_UTIL.startMiniCluster();
+ procExecutor = TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
+ byte[][] splitKeys =
+ new byte[][] { Bytes.toBytes("split1"), Bytes.toBytes("split2"), Bytes.toBytes("split3") };
+ TEST_UTIL.createTable(tableName, Bytes.toBytes("cf"), splitKeys);
+ TEST_UTIL.waitTableAvailable(tableName);
+ TEST_UTIL.getAdmin().flush(tableName);
+ activeRegions = TEST_UTIL.getAdmin().getRegions(tableName);
+ assertFalse(activeRegions.isEmpty());
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testRefreshMetaProcedureExecutesSuccessfully() {
+ RefreshMetaProcedure procedure = new RefreshMetaProcedure(procExecutor.getEnvironment());
+ long procId = procExecutor.submitProcedure(procedure);
+ ProcedureTestingUtility.waitProcedure(procExecutor, procId);
+ assertProcNotFailed(procExecutor.getResult(procId));
+ }
+
+ @Test
+ public void testGetCurrentRegions() throws Exception {
+ RefreshMetaProcedure procedure = new RefreshMetaProcedure(procExecutor.getEnvironment());
+ List<RegionInfo> regions = procedure.getCurrentRegions(TEST_UTIL.getConnection());
+ assertFalse("Should have found regions in meta", regions.isEmpty());
+ assertTrue("Should include test table region",
+ regions.stream().anyMatch(r -> r.getTable().getNameAsString().equals("testRefreshMeta")));
+ }
+
+ @Test
+ public void testScanBackingStorage() throws Exception {
+ RefreshMetaProcedure procedure = new RefreshMetaProcedure(procExecutor.getEnvironment());
+
+ List<RegionInfo> fsRegions = procedure.scanBackingStorage(TEST_UTIL.getConnection());
+
+ assertTrue("All regions from meta should be found in the storage",
+ activeRegions.stream().allMatch(reg -> fsRegions.stream()
+ .anyMatch(r -> r.getRegionNameAsString().equals(reg.getRegionNameAsString()))));
+ }
+
+ @Test
+ public void testHasBoundaryChanged() throws Exception {
+ RefreshMetaProcedure procedure = new RefreshMetaProcedure(procExecutor.getEnvironment());
+ RegionInfo region1 = RegionInfoBuilder.newBuilder(tableName)
+ .setStartKey(Bytes.toBytes("start1")).setEndKey(Bytes.toBytes("end1")).build();
+
+ RegionInfo region2 = RegionInfoBuilder.newBuilder(tableName)
+ .setStartKey(Bytes.toBytes("start2")).setEndKey(Bytes.toBytes("end1")).build();
+
+ RegionInfo region3 = RegionInfoBuilder.newBuilder(tableName)
+ .setStartKey(Bytes.toBytes("start1")).setEndKey(Bytes.toBytes("end2")).build();
+
+ assertTrue("Different start keys should have been detected",
+ procedure.hasBoundaryChanged(region1, region2));
+
+ assertTrue("Different end keys should have been detected",
+ procedure.hasBoundaryChanged(region1, region3));
+
+ assertFalse("Identical boundaries should not have been identified",
+ procedure.hasBoundaryChanged(region1, region1));
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRefreshMetaProcedureIntegration.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRefreshMetaProcedureIntegration.java
new file mode 100644
index 00000000000..208c2b04beb
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRefreshMetaProcedureIntegration.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.procedure;
+
+import static org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.assertProcNotFailed;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Stream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MetaTableAccessor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.client.TableState;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MasterTests.class, LargeTests.class })
+public class TestRefreshMetaProcedureIntegration {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestRefreshMetaProcedureIntegration.class);
+
+ private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
+ private Admin admin;
+ private ProcedureExecutor<MasterProcedureEnv> procExecutor;
+ private HMaster master;
+ private HRegionServer regionServer;
+
+ @Before
+ public void setup() throws Exception {
+ // Start in active mode
+ TEST_UTIL.getConfiguration().setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, false);
+
+ TEST_UTIL.startMiniCluster();
+ admin = TEST_UTIL.getAdmin();
+ procExecutor = TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
+ master = TEST_UTIL.getHBaseCluster().getMaster();
+ regionServer = TEST_UTIL.getHBaseCluster().getRegionServerThreads().get(0).getRegionServer();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (admin != null) {
+ admin.close();
+ }
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testRestoreMissingRegionInMeta() throws Exception {
+
+ TableName tableName = TableName.valueOf("replicaTestTable");
+
+ createTableWithData(tableName);
+
+ List<RegionInfo> activeRegions = admin.getRegions(tableName);
+ assertTrue("Should have at least 2 regions after split", activeRegions.size() >= 2);
+
+ Table metaTable = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
+ RegionInfo regionToRemove = activeRegions.get(0);
+ admin.unassign(regionToRemove.getRegionName(), false);
+ Thread.sleep(1000);
+
+ org.apache.hadoop.hbase.client.Delete delete =
+ new org.apache.hadoop.hbase.client.Delete(regionToRemove.getRegionName());
+ metaTable.delete(delete);
+ metaTable.close();
+
+ List<RegionInfo> regionsAfterDrift = admin.getRegions(tableName);
+ assertEquals("Should have one less region in meta after simulating drift",
+ activeRegions.size() - 1, regionsAfterDrift.size());
+
+ setReadOnlyMode(true);
+
+ boolean writeBlocked = false;
+ try {
+ Table readOnlyTable = TEST_UTIL.getConnection().getTable(tableName);
+ Put testPut = new Put(Bytes.toBytes("test_readonly"));
+ testPut.addColumn(Bytes.toBytes("cf1"), Bytes.toBytes("qual"), Bytes.toBytes("should_fail"));
+ readOnlyTable.put(testPut);
+ readOnlyTable.close();
+ } catch (Exception e) {
+ if (e.getMessage().contains("Operation not allowed in Read-Only Mode")) {
+ writeBlocked = true;
+ }
+ }
+ assertTrue("Write operations should be blocked in read-only mode", writeBlocked);
+
+ Long procId = admin.refreshMeta();
+
+ waitForProcedureCompletion(procId);
+
+ List<RegionInfo> regionsAfterRefresh = admin.getRegions(tableName);
+ assertEquals("Missing regions should be restored by refresh_meta", activeRegions.size(),
+ regionsAfterRefresh.size());
+
+ boolean regionRestored = regionsAfterRefresh.stream()
+ .anyMatch(r -> r.getRegionNameAsString().equals(regionToRemove.getRegionNameAsString()));
+ assertTrue("Missing region should be restored by refresh_meta", regionRestored);
+
+ setReadOnlyMode(false);
+
+ Table activeTable = TEST_UTIL.getConnection().getTable(tableName);
+ Put testPut = new Put(Bytes.toBytes("test_active_again"));
+ testPut.addColumn(Bytes.toBytes("cf1"), Bytes.toBytes("qual"),
+ Bytes.toBytes("active_mode_again"));
+ activeTable.put(testPut);
+ activeTable.close();
+ }
+
+ @Test
+ public void testPhantomTableCleanup() throws Exception {
+ TableName table1 = TableName.valueOf("table1");
+ TableName phantomTable = TableName.valueOf("phantomTable");
+ createTableWithData(table1);
+ createTableWithData(phantomTable);
+
+ assertTrue("Table1 should have multiple regions", admin.getRegions(table1).size() >= 2);
+ assertTrue("phantomTable should have multiple regions",
+ admin.getRegions(phantomTable).size() >= 2);
+
+ deleteTableFromFilesystem(phantomTable);
+ List<TableName> tablesBeforeRefresh = Arrays.asList(admin.listTableNames());
+ assertTrue("phantomTable should still be listed before refresh_meta",
+ tablesBeforeRefresh.contains(phantomTable));
+ assertTrue("Table1 should still be listed", tablesBeforeRefresh.contains(table1));
+
+ setReadOnlyMode(true);
+ Long procId = admin.refreshMeta();
+ waitForProcedureCompletion(procId);
+
+ List<TableName> tablesAfterRefresh = Arrays.asList(admin.listTableNames());
+
+ assertFalse("phantomTable should be removed after refresh_meta",
+ tablesAfterRefresh.contains(phantomTable));
+ assertTrue("Table1 should still be listed", tablesAfterRefresh.contains(table1));
+ assertTrue("phantomTable should have no regions after refresh_meta",
+ admin.getRegions(phantomTable).isEmpty());
+ setReadOnlyMode(false);
+ }
+
+ @Test
+ public void testRestoreTableStateForOrphanRegions() throws Exception {
+ TableName tableName = TableName.valueOf("t1");
+ createTableInFilesystem(tableName);
+
+ assertEquals("No tables should exist", 0,
+ Stream.of(admin.listTableNames()).filter(tn -> tn.equals(tableName)).count());
+
+ setReadOnlyMode(true);
+ Long procId = admin.refreshMeta();
+ waitForProcedureCompletion(procId);
+
+ TableState tableState = MetaTableAccessor.getTableState(admin.getConnection(), tableName);
+ assert tableState != null;
+ assertEquals("Table state should be ENABLED", TableState.State.ENABLED, tableState.getState());
+ assertEquals("The list should show the new table from the FS", 1,
+ Stream.of(admin.listTableNames()).filter(tn -> tn.equals(tableName)).count());
+ assertFalse("Should have at least 1 region", admin.getRegions(tableName).isEmpty());
+ setReadOnlyMode(false);
+ }
+
+ private void createTableInFilesystem(TableName tableName) throws IOException {
+ FileSystem fs = TEST_UTIL.getTestFileSystem();
+ Path rootDir = CommonFSUtils.getRootDir(TEST_UTIL.getConfiguration());
+ Path tableDir = CommonFSUtils.getTableDir(rootDir, tableName);
+ fs.mkdirs(tableDir);
+
+ TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
+ TEST_UTIL.getHBaseCluster().getMaster().getTableDescriptors()
+ .update(builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf1")).build(), false);
+
+ Path regionDir = new Path(tableDir, "dab6d1e1c88787c13b97647f11b2c907");
+ Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
+ fs.mkdirs(regionDir);
+
+ RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).setStartKey(new byte[0])
+ .setEndKey(new byte[0]).setRegionId(1757100253228L).build();
+ byte[] regionInfoContent = RegionInfo.toDelimitedByteArray(regionInfo);
+ try (FSDataOutputStream out = fs.create(regionInfoFile, true)) {
+ out.write(regionInfoContent);
+ }
+ }
+
+ private void deleteTableFromFilesystem(TableName tableName) throws IOException {
+ FileSystem fs = TEST_UTIL.getTestFileSystem();
+ Path rootDir = CommonFSUtils.getRootDir(TEST_UTIL.getConfiguration());
+ Path tableDir = CommonFSUtils.getTableDir(rootDir, tableName);
+ if (fs.exists(tableDir)) {
+ fs.delete(tableDir, true);
+ }
+ }
+
+ private void createTableWithData(TableName tableName) throws Exception {
+ TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
+ builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf1"));
+ byte[] splitKeyBytes = Bytes.toBytes("split_key");
+ admin.createTable(builder.build(), new byte[][] { splitKeyBytes });
+ TEST_UTIL.waitTableAvailable(tableName);
+ try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
+ for (int i = 0; i < 100; i++) {
+ Put put = new Put(Bytes.toBytes("row_" + String.format("%05d", i)));
+ put.addColumn(Bytes.toBytes("cf1"), Bytes.toBytes("qual"), Bytes.toBytes("value_" + i));
+ table.put(put);
+ }
+ }
+ admin.flush(tableName);
+ }
+
+ private void waitForProcedureCompletion(Long procId) {
+ assertTrue("Procedure ID should be positive", procId > 0);
+ TEST_UTIL.waitFor(1000, () -> {
+ try {
+ return procExecutor.isFinished(procId);
+ } catch (Exception e) {
+ return false;
+ }
+ });
+ assertProcNotFailed(procExecutor.getResult(procId));
+ }
+
+ private void setReadOnlyMode(boolean isReadOnly) {
+ TEST_UTIL.getConfiguration().setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY,
+ isReadOnly);
+ notifyConfigurationObservers();
+ }
+
+ private void notifyConfigurationObservers() {
+ master.getConfigurationManager().notifyAllObservers(TEST_UTIL.getConfiguration());
+ regionServer.getConfigurationManager().notifyAllObservers(TEST_UTIL.getConfiguration());
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionInitialize.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionInitialize.java
index 8b6f4670732..20bd4ada48d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionInitialize.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionInitialize.java
@@ -24,8 +24,11 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.TableDescriptor;
@@ -114,4 +117,19 @@ public class TestMasterRegionInitialize extends MasterRegionTestBase {
// but the data should have been cleaned up
assertTrue(region.get(new Get(row)).isEmpty());
}
+
+ @Test
+ public void testMasterRegionDirSuffix() {
+ String currentMasterRegionDirName = MasterRegionFactory.getMasterRegionDirName();
+ assertEquals("Default master region directory should be MasterData", "MasterData",
+ currentMasterRegionDirName);
+
+ Configuration confWithSuffix = HBaseConfiguration.create();
+ String suffix = "replica1";
+ confWithSuffix.set(HConstants.HBASE_META_TABLE_SUFFIX, suffix);
+ String dirNameWithSuffix = MasterRegionFactory.initMasterRegionDirName(confWithSuffix);
+ String expectedDirName = "MasterData_" + suffix;
+ assertEquals("Directory name should have suffix when configured", expectedDirName,
+ dirNameWithSuffix);
+ }
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestHFileProcedurePrettyPrinter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestHFileProcedurePrettyPrinter.java
index b85897a1afa..0a99f27eb60 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestHFileProcedurePrettyPrinter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestHFileProcedurePrettyPrinter.java
@@ -102,7 +102,7 @@ public class TestHFileProcedurePrettyPrinter extends RegionProcedureStoreTestBas
store.cleanup();
store.region.flush(true);
Path tableDir = CommonFSUtils.getTableDir(
- new Path(htu.getDataTestDir(), MasterRegionFactory.MASTER_STORE_DIR),
+ new Path(htu.getDataTestDir(), MasterRegionFactory.getMasterRegionDirName()),
MasterRegionFactory.TABLE_NAME);
FileSystem fs = tableDir.getFileSystem(htu.getConfiguration());
Path regionDir =
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestWALProcedurePrettyPrinter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestWALProcedurePrettyPrinter.java
index d942c65b8c7..2875c9f467b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestWALProcedurePrettyPrinter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestWALProcedurePrettyPrinter.java
@@ -64,7 +64,7 @@ public class TestWALProcedurePrettyPrinter extends RegionProcedureStoreTestBase
}
store.cleanup();
Path walParentDir = new Path(htu.getDataTestDir(),
- MasterRegionFactory.MASTER_STORE_DIR + "/" + HConstants.HREGION_LOGDIR_NAME);
+ MasterRegionFactory.getMasterRegionDirName() + "/" + HConstants.HREGION_LOGDIR_NAME);
FileSystem fs = walParentDir.getFileSystem(htu.getConfiguration());
Path walDir = fs.listStatus(walParentDir)[0].getPath();
Path walFile = fs.listStatus(walDir)[0].getPath();
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestActiveClusterSuffix.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestActiveClusterSuffix.java
new file mode 100644
index 00000000000..df036f08f00
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestActiveClusterSuffix.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseCommonTestingUtil;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.hbase.util.JVMClusterUtil;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test Active Cluster Suffix file.
+ */
+@Category({ RegionServerTests.class, MediumTests.class })
+public class TestActiveClusterSuffix {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestActiveClusterSuffix.class);
+
+ private final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
+
+ private JVMClusterUtil.RegionServerThread rst;
+
+ @Before
+ public void setUp() throws Exception {
+ TEST_UTIL.getConfiguration().setBoolean(ShutdownHook.RUN_SHUTDOWN_HOOK, false);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ if (rst != null && rst.getRegionServer() != null) {
+ rst.getRegionServer().stop("end of test");
+ rst.join();
+ }
+ }
+
+ @Test
+ public void testActiveClusterSuffixCreated() throws Exception {
+ TEST_UTIL.startMiniZKCluster();
+ TEST_UTIL.startMiniDFSCluster(1);
+ TEST_UTIL.startMiniHBaseCluster();
+
+ Path rootDir = CommonFSUtils.getRootDir(TEST_UTIL.getConfiguration());
+ FileSystem fs = rootDir.getFileSystem(TEST_UTIL.getConfiguration());
+ Path filePath = new Path(rootDir, HConstants.ACTIVE_CLUSTER_SUFFIX_FILE_NAME);
+
+ assertTrue(filePath + " should exists ", fs.exists(filePath));
+ assertTrue(filePath + " should not be empty ", fs.getFileStatus(filePath).getLen() > 0);
+
+ MasterFileSystem mfs = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem();
+ // Compute string using currently set suffix and the cluster id
+ String cluster_suffix1 =
+ new String(mfs.getSuffixFileDataToCompare(), StandardCharsets.US_ASCII);
+ // Compute string member variable
+ String cluster_suffix2 = mfs.getActiveClusterSuffix().toString();
+ assertEquals(cluster_suffix1, cluster_suffix2);
+ }
+
+ @Test
+ public void testSuffixFileOnRestart() throws Exception {
+ TEST_UTIL.startMiniZKCluster();
+ TEST_UTIL.startMiniDFSCluster(1);
+ TEST_UTIL.createRootDir();
+ TEST_UTIL.getConfiguration().set(HConstants.HBASE_META_TABLE_SUFFIX, "Test");
+
+ String clusterId = HBaseCommonTestingUtil.getRandomUUID().toString();
+ String cluster_suffix = clusterId + ":" + TEST_UTIL.getConfiguration()
+ .get(HConstants.HBASE_META_TABLE_SUFFIX, HConstants.HBASE_META_TABLE_SUFFIX_DEFAULT_VALUE);
+
+ writeIdFile(clusterId, HConstants.CLUSTER_ID_FILE_NAME);
+ writeIdFile(cluster_suffix, HConstants.ACTIVE_CLUSTER_SUFFIX_FILE_NAME);
+
+ try {
+ TEST_UTIL.startMiniHBaseCluster();
+ } catch (IOException ioe) {
+ Assert.fail("Can't start mini hbase cluster.");
+ }
+
+ MasterFileSystem mfs = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem();
+ // Compute using file contents
+ String cluster_suffix1 =
+ new String(mfs.getSuffixFileDataToCompare(), StandardCharsets.US_ASCII);
+ // Compute using config
+ String cluster_suffix2 = mfs.getSuffixFromConfig();
+
+ assertEquals(cluster_suffix1, cluster_suffix2);
+ assertEquals(cluster_suffix, cluster_suffix1);
+ }
+
+ @Test
+ public void testVerifyErrorWhenSuffixNotMatched() throws Exception {
+ TEST_UTIL.startMiniZKCluster();
+ TEST_UTIL.startMiniDFSCluster(1);
+ TEST_UTIL.createRootDir();
+ TEST_UTIL.getConfiguration().setInt("hbase.master.start.timeout.localHBaseCluster", 10000);
+ String cluster_suffix = String.valueOf("2df92f65-d801-46e6-b892-c2bae2df3c21:test");
+ writeIdFile(cluster_suffix, HConstants.ACTIVE_CLUSTER_SUFFIX_FILE_NAME);
+ // Exception: as config in the file and the one set by the user are not matching
+ boolean threwIOE = false;
+ try {
+ TEST_UTIL.startMiniHBaseCluster();
+ } catch (IOException ioe) {
+ threwIOE = true;
+ } finally {
+ assertTrue("The master should have thrown an exception", threwIOE);
+ }
+ }
+
+ private void writeIdFile(String id, String fileName) throws Exception {
+ Path rootDir = CommonFSUtils.getRootDir(TEST_UTIL.getConfiguration());
+ FileSystem fs = rootDir.getFileSystem(TEST_UTIL.getConfiguration());
+ Path filePath = new Path(rootDir, fileName);
+ FSDataOutputStream s = null;
+ try {
+ s = fs.create(filePath);
+ s.writeUTF(id);
+ } finally {
+ if (s != null) {
+ s.close();
+ }
+ }
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSplitReadOnly.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSplitReadOnly.java
new file mode 100644
index 00000000000..065d42cd24c
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSplitReadOnly.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ RegionServerTests.class, SmallTests.class })
+public class TestCompactSplitReadOnly {
+
+ private CompactSplit compactSplit;
+ private Configuration conf;
+
+ @Before
+ public void setUp() {
+ conf = new Configuration();
+ // enable read-only mode
+ conf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, true);
+ // create CompactSplit with conf-only constructor (available for tests)
+ compactSplit = new CompactSplit(conf);
+ }
+
+ @After
+ public void tearDown() {
+ // ensure thread pools are shutdown to avoid leakage
+ compactSplit.interruptIfNecessary();
+ }
+
+ @Test
+ public void testRequestSystemCompactionIgnoredWhenReadOnly() throws IOException {
+ // Mock HRegion and HStore minimal behavior
+ HRegion region = mock(HRegion.class);
+ HStore store = mock(HStore.class);
+
+ // Ensure compaction queues start empty
+ assertEquals(0, compactSplit.getCompactionQueueSize());
+
+ // Call requestSystemCompaction for a single store (selectNow = false)
+ compactSplit.requestSystemCompaction(region, store, "test-readonly");
+
+ // Because read-only mode is enabled, no compaction should be queued
+ assertEquals(0, compactSplit.getCompactionQueueSize());
+ }
+
+ @Test
+ public void testSelectCompactionIgnoredWhenReadOnly() throws IOException {
+ HRegion region = mock(HRegion.class);
+ HStore store = mock(HStore.class);
+
+ // Ensure compaction queues start empty
+ assertEquals(0, compactSplit.getCompactionQueueSize());
+
+ // Call requestCompaction which uses selectNow = true and should call selectCompaction
+ compactSplit.requestCompaction(region, store, "test-select-readonly", 0,
+ CompactionLifeCycleTracker.DUMMY, (User) null);
+
+ // Because read-only mode is enabled, selectCompaction should be skipped and no task queued
+ assertEquals(0, compactSplit.getCompactionQueueSize());
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DummyStoreFileTrackerForReadOnlyMode.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DummyStoreFileTrackerForReadOnlyMode.java
new file mode 100644
index 00000000000..5f8268b3ac7
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DummyStoreFileTrackerForReadOnlyMode.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.storefiletracker;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+
+public class DummyStoreFileTrackerForReadOnlyMode extends StoreFileTrackerBase {
+ private boolean readOnlyUsed = false;
+ private boolean compactionExecuted = false;
+ private boolean addExecuted = false;
+ private boolean setExecuted = false;
+
+ public DummyStoreFileTrackerForReadOnlyMode(Configuration conf, boolean isPrimaryReplica) {
+ super(conf, isPrimaryReplica, null);
+ }
+
+ @Override
+ protected void doAddCompactionResults(Collection<StoreFileInfo> compactedFiles,
+ Collection<StoreFileInfo> newFiles) {
+ compactionExecuted = true;
+ }
+
+ @Override
+ protected void doSetStoreFiles(Collection<StoreFileInfo> files) throws IOException {
+ setExecuted = true;
+ }
+
+ @Override
+ protected List<StoreFileInfo> doLoadStoreFiles(boolean readOnly) {
+ readOnlyUsed = readOnly;
+ return Collections.emptyList();
+ }
+
+ @Override
+ protected void doAddNewStoreFiles(Collection<StoreFileInfo> newFiles) throws IOException {
+ addExecuted = true;
+ }
+
+ boolean wasReadOnlyLoad() {
+ return readOnlyUsed;
+ }
+
+ boolean wasCompactionExecuted() {
+ return compactionExecuted;
+ }
+
+ boolean wasAddExecuted() {
+ return addExecuted;
+ }
+
+ boolean wasSetExecuted() {
+ return setExecuted;
+ }
+
+ @Override
+ public boolean requireWritingToTmpDirFirst() {
+ return false;
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerBaseReadOnlyMode.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerBaseReadOnlyMode.java
new file mode 100644
index 00000000000..8a1cee55832
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerBaseReadOnlyMode.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.storefiletracker;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Collections;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.TestRefreshHFilesBase;
+import org.apache.hadoop.hbase.master.procedure.TestRefreshHFilesProcedureWithReadOnlyConf;
+import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ RegionServerTests.class, SmallTests.class })
+public class TestStoreFileTrackerBaseReadOnlyMode extends TestRefreshHFilesBase {
+ private DummyStoreFileTrackerForReadOnlyMode tracker;
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestRefreshHFilesProcedureWithReadOnlyConf.class);
+
+ @Before
+ public void setup() throws Exception {
+ // When true is passed only setup for readonly property is done.
+ // The initial ReadOnly property will be false for table creation
+ baseSetup(true);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ baseTearDown();
+ }
+
+ @Test
+ public void testLoadReadOnlyWhenGlobalReadOnlyEnabled() throws Exception {
+ try {
+ setReadOnlyMode(true);
+ tracker = new DummyStoreFileTrackerForReadOnlyMode(conf, true);
+ tracker.load();
+ assertTrue("Tracker should be in read-only mode", tracker.wasReadOnlyLoad());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ setReadOnlyMode(false);
+ }
+ }
+
+ @Test
+ public void testReplaceSkippedWhenGlobalReadOnlyEnabled() throws Exception {
+ try {
+ setReadOnlyMode(true);
+ tracker = new DummyStoreFileTrackerForReadOnlyMode(conf, true);
+ tracker.replace(Collections.emptyList(), Collections.emptyList());
+ assertFalse("Compaction should not be executed in readonly mode",
+ tracker.wasCompactionExecuted());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ setReadOnlyMode(false);
+ }
+ }
+
+ @Test
+ public void testReplaceExecutedWhenWritable() throws Exception {
+ tracker = new DummyStoreFileTrackerForReadOnlyMode(conf, true);
+ tracker.replace(Collections.emptyList(), Collections.emptyList());
+ assertTrue("Compaction should run when not readonly", tracker.wasCompactionExecuted());
+ }
+
+ @Test
+ public void testAddSkippedWhenGlobalReadOnlyEnabled() throws Exception {
+ try {
+ setReadOnlyMode(true);
+ tracker = new DummyStoreFileTrackerForReadOnlyMode(conf, true);
+ tracker.add(Collections.emptyList());
+ assertFalse("Add should not be executed in readonly mode", tracker.wasAddExecuted());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ setReadOnlyMode(false);
+ }
+ }
+
+ @Test
+ public void testAddExecutedWhenWritable() throws Exception {
+ tracker = new DummyStoreFileTrackerForReadOnlyMode(conf, true);
+ tracker.add(Collections.emptyList());
+ assertTrue("Add should run when not readonly", tracker.wasAddExecuted());
+ }
+
+ @Test
+ public void testSetSkippedWhenGlobalReadOnlyEnabled() throws Exception {
+ try {
+ setReadOnlyMode(true);
+ tracker = new DummyStoreFileTrackerForReadOnlyMode(conf, true);
+ tracker.set(Collections.emptyList());
+ assertFalse("Set should not be executed in readonly mode", tracker.wasSetExecuted());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ setReadOnlyMode(false);
+ }
+ }
+
+ @Test
+ public void testSetExecutedWhenWritable() throws Exception {
+ tracker = new DummyStoreFileTrackerForReadOnlyMode(conf, true);
+ tracker.set(Collections.emptyList());
+ assertTrue("Set should run when not readonly", tracker.wasSetExecuted());
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testCreateWriterThrowExceptionWhenGlobalReadOnlyEnabled() throws Exception {
+ try {
+ setReadOnlyMode(true);
+ tracker = new DummyStoreFileTrackerForReadOnlyMode(conf, true);
+ CreateStoreFileWriterParams params = CreateStoreFileWriterParams.create().maxKeyCount(4)
+ .isCompaction(false).includeMVCCReadpoint(true).includesTag(false).shouldDropBehind(false);
+ tracker.createWriter(params);
+ } finally {
+ setReadOnlyMode(false);
+ }
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java
index b0e3eb051fc..cf4f1f0e838 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java
@@ -1011,4 +1011,23 @@ public class VerifyingRSGroupAdmin implements Admin, Closeable {
return admin.isReplicationPeerModificationEnabled();
}
+ @Override
+ public Long refreshMeta() throws IOException {
+ return admin.refreshMeta();
+ }
+
+ @Override
+ public Long refreshHFiles(final TableName tableName) throws IOException {
+ return admin.refreshHFiles(tableName);
+ }
+
+ @Override
+ public Long refreshHFiles(final String namespace) throws IOException {
+ return admin.refreshHFiles(namespace);
+ }
+
+ @Override
+ public Long refreshHFiles() throws IOException {
+ return admin.refreshHFiles();
+ }
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestCanStartHBaseInReadOnlyMode.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestCanStartHBaseInReadOnlyMode.java
new file mode 100644
index 00000000000..82bca89f443
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestCanStartHBaseInReadOnlyMode.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.access;
+
+import static org.apache.hadoop.hbase.HConstants.HBASE_CLIENT_RETRIES_NUMBER;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.SecurityTests;
+import org.junit.*;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+@Category({ SecurityTests.class, LargeTests.class })
+@SuppressWarnings("deprecation")
+public class TestCanStartHBaseInReadOnlyMode {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestCanStartHBaseInReadOnlyMode.class);
+
+ private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
+ private static Configuration conf;
+
+ @Rule
+ public TestName name = new TestName();
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ conf = TEST_UTIL.getConfiguration();
+
+ // Shorten the run time of failed unit tests by limiting retries and the session timeout
+ // threshold
+ conf.setInt(HBASE_CLIENT_RETRIES_NUMBER, 0);
+ conf.setInt("hbase.master.init.timeout.localHBaseCluster", 10000);
+
+ // Enable Read-Only mode to prove the cluster can be started in this state
+ conf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, true);
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testCanStartHBaseInReadOnlyMode() throws Exception {
+ TEST_UTIL.startMiniCluster(1);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyController.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyController.java
new file mode 100644
index 00000000000..31e5f8193ba
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyController.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.access;
+
+import static org.apache.hadoop.hbase.HConstants.HBASE_CLIENT_RETRIES_NUMBER;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.SecurityTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Category({ SecurityTests.class, LargeTests.class })
+@SuppressWarnings("deprecation")
+public class TestReadOnlyController {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestReadOnlyController.class);
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestReadOnlyController.class);
+ private final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
+ private static final TableName TEST_TABLE = TableName.valueOf("read_only_test_table");
+ private static final byte[] TEST_FAMILY = Bytes.toBytes("read_only_table_col_fam");
+ private static HRegionServer hRegionServer;
+ private static HMaster hMaster;
+ private static Configuration conf;
+ private static Connection connection;
+ private static SingleProcessHBaseCluster cluster;
+
+ private static Table testTable;
+ @Rule
+ public TestName name = new TestName();
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ @Before
+ public void beforeClass() throws Exception {
+ conf = TEST_UTIL.getConfiguration();
+
+ // Shorten the run time of failed unit tests by limiting retries and the session timeout
+ // threshold
+ conf.setInt(HBASE_CLIENT_RETRIES_NUMBER, 1);
+ conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 1000);
+
+ // Set up test class with Read-Only mode disabled so a table can be created
+ conf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, false);
+
+ try {
+ // Start the test cluster
+ cluster = TEST_UTIL.startMiniCluster(1);
+
+ hMaster = cluster.getMaster();
+ hRegionServer = cluster.getRegionServerThreads().get(0).getRegionServer();
+ connection = ConnectionFactory.createConnection(conf);
+
+ // Create a test table
+ testTable = TEST_UTIL.createTable(TEST_TABLE, TEST_FAMILY);
+ } catch (Exception e) {
+ // Delete the created table, and clean up the connection and cluster before throwing an
+ // exception
+ disableReadOnlyMode();
+ TEST_UTIL.deleteTable(TEST_TABLE);
+ connection.close();
+ TEST_UTIL.shutdownMiniCluster();
+ throw new RuntimeException(e);
+ }
+ }
+
+ @After
+ public void afterClass() throws Exception {
+ if (connection != null) {
+ connection.close();
+ }
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ private static void enableReadOnlyMode() {
+ // Dynamically enable Read-Only mode if it is not active
+ if (!isReadOnlyModeEnabled()) {
+ LOG.info("Dynamically enabling Read-Only mode by setting {} to true",
+ HConstants.HBASE_GLOBAL_READONLY_ENABLED_DEFAULT);
+ conf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, true);
+ notifyObservers();
+ }
+ }
+
+ private static void disableReadOnlyMode() {
+ // Dynamically disable Read-Only mode if it is active
+ if (isReadOnlyModeEnabled()) {
+ LOG.info("Dynamically disabling Read-Only mode by setting {} to false",
+ HConstants.HBASE_GLOBAL_READONLY_ENABLED_DEFAULT);
+ conf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, false);
+ notifyObservers();
+ }
+ }
+
+ private static boolean isReadOnlyModeEnabled() {
+ return conf.getBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY,
+ HConstants.HBASE_GLOBAL_READONLY_ENABLED_DEFAULT);
+ }
+
+ private static void notifyObservers() {
+ LOG.info("Notifying observers about configuration changes");
+ hMaster.getConfigurationManager().notifyAllObservers(conf);
+ hRegionServer.getConfigurationManager().notifyAllObservers(conf);
+ }
+
+ // The test case for successfully creating a table with Read-Only mode disabled happens when
+ // setting up the test class, so we only need a test function for a failed table creation.
+ @Test
+ public void testCannotCreateTableWithReadOnlyEnabled() throws IOException {
+ // Expect an IOException to result from the createTable attempt since Read-Only mode is enabled
+ enableReadOnlyMode();
+ TableName newTable = TableName.valueOf("bad_read_only_test_table");
+ exception.expect(IOException.class);
+ exception.expectMessage("Operation not allowed in Read-Only Mode");
+
+ // This should throw the IOException
+ TEST_UTIL.createTable(newTable, TEST_FAMILY);
+ }
+
+ @Test
+ public void testPutWithReadOnlyDisabled() throws IOException {
+ // Successfully put a row in the table since Read-Only mode is disabled
+ disableReadOnlyMode();
+ final byte[] row2 = Bytes.toBytes("row2");
+ final byte[] value = Bytes.toBytes("efgh");
+ Put put = new Put(row2);
+ put.addColumn(TEST_FAMILY, null, value);
+ testTable.put(put);
+ }
+
+ @Test
+ public void testCannotPutWithReadOnlyEnabled() throws IOException {
+ // Prepare a Put command with Read-Only mode enabled
+ enableReadOnlyMode();
+ final byte[] row1 = Bytes.toBytes("row1");
+ final byte[] value = Bytes.toBytes("abcd");
+ Put put = new Put(row1);
+ put.addColumn(TEST_FAMILY, null, value);
+
+ // Expect an IOException to result from the Put attempt
+ exception.expect(IOException.class);
+ exception.expectMessage("Operation not allowed in Read-Only Mode");
+
+ // This should throw the IOException
+ testTable.put(put);
+ }
+
+ @Test
+ public void testBatchPutWithReadOnlyDisabled() throws IOException, InterruptedException {
+ // Successfully create and run a batch Put operation with Read-Only mode disabled
+ disableReadOnlyMode();
+ List<Row> actions = new ArrayList<>();
+ actions.add(new Put(Bytes.toBytes("row10")).addColumn(TEST_FAMILY, null, Bytes.toBytes("10")));
+ actions.add(new Delete(Bytes.toBytes("row10")));
+ testTable.batch(actions, null);
+ }
+
+ @Test
+ public void testCannotBatchPutWithReadOnlyEnabled() throws IOException, InterruptedException {
+ // Create a batch Put operation that is expected to fail with Read-Only mode enabled
+ enableReadOnlyMode();
+ List<Row> actions = new ArrayList<>();
+ actions.add(new Put(Bytes.toBytes("row11")).addColumn(TEST_FAMILY, null, Bytes.toBytes("11")));
+ actions.add(new Delete(Bytes.toBytes("row11")));
+
+ // Expect an IOException to result from the batch Put attempt
+ exception.expect(IOException.class);
+ exception.expectMessage("Operation not allowed in Read-Only Mode");
+
+ // This should throw the IOException
+ testTable.batch(actions, null);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyControllerBulkLoadObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyControllerBulkLoadObserver.java
new file mode 100644
index 00000000000..9ac008fdaf8
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyControllerBulkLoadObserver.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.access;
+
+import static org.mockito.Mockito.mock;
+
+import java.io.IOException;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.testclassification.SecurityTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+// Tests methods of BulkLoad Observer which are implemented in ReadOnlyController,
+// by mocking the coprocessor environment and dependencies
+@Category({ SecurityTests.class, SmallTests.class })
+public class TestReadOnlyControllerBulkLoadObserver {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestReadOnlyControllerBulkLoadObserver.class);
+
+ BulkLoadReadOnlyController bulkLoadReadOnlyController;
+
+ // Region Server Coprocessor mocking variables
+ ObserverContext<RegionCoprocessorEnvironment> ctx;
+
+ @Before
+ public void setup() throws Exception {
+ bulkLoadReadOnlyController = new BulkLoadReadOnlyController();
+
+ // mocking variables initialization
+ ctx = mock(ObserverContext.class);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPrePrepareBulkLoadReadOnlyException() throws IOException {
+ bulkLoadReadOnlyController.prePrepareBulkLoad(ctx);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCleanupBulkLoadReadOnlyException() throws IOException {
+ bulkLoadReadOnlyController.preCleanupBulkLoad(ctx);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyControllerCoprocessorLoading.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyControllerCoprocessorLoading.java
new file mode 100644
index 00000000000..4846e84bcd6
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyControllerCoprocessorLoading.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.access;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
+import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.SecurityTests;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@RunWith(Parameterized.class)
+@Category({ SecurityTests.class, MediumTests.class })
+public class TestReadOnlyControllerCoprocessorLoading {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestReadOnlyControllerCoprocessorLoading.class);
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestReadOnlyController.class);
+ private HBaseTestingUtil TEST_UTIL;
+
+ Configuration conf;
+ TableName tableName = TableName.valueOf("test_table");
+ HMaster master;
+ HRegionServer regionServer;
+ HRegion region;
+
+ private final boolean initialReadOnlyMode;
+
+ public TestReadOnlyControllerCoprocessorLoading(boolean initialReadOnlyMode) {
+ this.initialReadOnlyMode = initialReadOnlyMode;
+ }
+
+ @Before
+ public void setup() throws Exception {
+ TEST_UTIL = new HBaseTestingUtil();
+ if (TEST_UTIL.getMiniHBaseCluster() != null) {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ private void setupMiniCluster(boolean isReadOnlyEnabled) throws Exception {
+ conf = TEST_UTIL.getConfiguration();
+ conf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, isReadOnlyEnabled);
+ TEST_UTIL.startMiniCluster(1);
+
+ master = TEST_UTIL.getMiniHBaseCluster().getMaster();
+ regionServer = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
+ }
+
+ private void createTable() throws Exception {
+ // create a table to get at a region
+ TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
+ .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf")).build();
+ TEST_UTIL.getAdmin().createTable(desc);
+
+ List<HRegion> regions = regionServer.getRegions(tableName);
+ assertFalse(regions.isEmpty());
+ region = regions.get(0);
+ }
+
+ private void setReadOnlyMode(boolean isReadOnlyEnabled) {
+ // Create a new configuration to micic client server behavior
+ // otherwise the existing conf object is shared with the cluster
+ // and can cause side effects on other tests if not reset properly.
+ // This way we can ensure that only the coprocessor loading is tested
+ // without impacting other tests.
+ HBaseTestingUtil NEW_TEST_UTIL = new HBaseTestingUtil();
+ Configuration newConf = NEW_TEST_UTIL.getConfiguration();
+ // Set the read-only enabled config dynamically after cluster startup
+ newConf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, isReadOnlyEnabled);
+ master.getConfigurationManager().notifyAllObservers(newConf);
+ regionServer.getConfigurationManager().notifyAllObservers(newConf);
+ }
+
+ private void verifyMasterReadOnlyControllerLoading(boolean isReadOnlyEnabled) throws Exception {
+ MasterCoprocessorHost masterCPHost = master.getMasterCoprocessorHost();
+ if (isReadOnlyEnabled) {
+ assertNotNull(
+ MasterReadOnlyController.class.getName()
+ + " should be loaded at startup when readonly is true.",
+ masterCPHost.findCoprocessor(MasterReadOnlyController.class.getName()));
+ } else {
+ assertNull(
+ MasterReadOnlyController.class.getName()
+ + " should not be loaded at startup when readonly support property is false.",
+ masterCPHost.findCoprocessor(MasterReadOnlyController.class.getName()));
+ }
+ }
+
+ private void verifyRegionServerReadOnlyControllerLoading(boolean isReadOnlyEnabled)
+ throws Exception {
+ RegionServerCoprocessorHost rsCPHost = regionServer.getRegionServerCoprocessorHost();
+ if (isReadOnlyEnabled) {
+ assertNotNull(
+ RegionServerReadOnlyController.class.getName()
+ + " should be loaded at startup when readonly is true.",
+ rsCPHost.findCoprocessor(RegionServerReadOnlyController.class.getName()));
+ } else {
+ assertNull(
+ RegionServerReadOnlyController.class.getName()
+ + " should not be loaded at startup when readonly support property is false.",
+ rsCPHost.findCoprocessor(RegionServerReadOnlyController.class.getName()));
+ }
+ }
+
+ private void verifyRegionReadOnlyControllerLoading(boolean isReadOnlyEnabled) throws Exception {
+ RegionCoprocessorHost regionCPHost = region.getCoprocessorHost();
+
+ if (isReadOnlyEnabled) {
+ assertNotNull(
+ RegionReadOnlyController.class.getName()
+ + " should be loaded at startup when readonly is true.",
+ regionCPHost.findCoprocessor(RegionReadOnlyController.class.getName()));
+ assertNotNull(
+ EndpointReadOnlyController.class.getName()
+ + " should be loaded at startup when readonly is true.",
+ regionCPHost.findCoprocessor(EndpointReadOnlyController.class.getName()));
+ assertNotNull(
+ BulkLoadReadOnlyController.class.getName()
+ + " should be loaded at startup when readonly is true.",
+ regionCPHost.findCoprocessor(BulkLoadReadOnlyController.class.getName()));
+ } else {
+ assertNull(
+ RegionReadOnlyController.class.getName()
+ + " should not be loaded at startup when readonly support property is false",
+ regionCPHost.findCoprocessor(RegionReadOnlyController.class.getName()));
+ assertNull(
+ EndpointReadOnlyController.class.getName()
+ + " should not be loaded at startup when readonly support property is false",
+ regionCPHost.findCoprocessor(EndpointReadOnlyController.class.getName()));
+ assertNull(
+ BulkLoadReadOnlyController.class.getName()
+ + " should not be loaded at startup when readonly support property is false",
+ regionCPHost.findCoprocessor(BulkLoadReadOnlyController.class.getName()));
+ }
+ }
+
+ private void verifyReadOnlyState(boolean isReadOnlyEnabled) throws Exception {
+ verifyMasterReadOnlyControllerLoading(isReadOnlyEnabled);
+ verifyRegionServerReadOnlyControllerLoading(isReadOnlyEnabled);
+ verifyRegionReadOnlyControllerLoading(isReadOnlyEnabled);
+ }
+
+ @Test
+ public void testReadOnlyControllerStartupBehavior() throws Exception {
+ setupMiniCluster(initialReadOnlyMode);
+ // Table creation is needed to get a region and verify region coprocessor loading hence we can't
+ // test region coprocessor loading at startup.
+ // This will get covered in the dynamic loading test where we will also verify that the
+ // coprocessors are loaded at after table creation dynamically.
+ verifyMasterReadOnlyControllerLoading(initialReadOnlyMode);
+ verifyRegionServerReadOnlyControllerLoading(initialReadOnlyMode);
+ }
+
+ @Test
+ public void testReadOnlyControllerLoadedWhenEnabledDynamically() throws Exception {
+ setupMiniCluster(initialReadOnlyMode);
+ if (!initialReadOnlyMode) {
+ createTable();
+ }
+ boolean isReadOnlyEnabled = true;
+ setReadOnlyMode(isReadOnlyEnabled);
+ verifyMasterReadOnlyControllerLoading(isReadOnlyEnabled);
+ verifyRegionServerReadOnlyControllerLoading(isReadOnlyEnabled);
+ if (!initialReadOnlyMode) {
+ verifyRegionReadOnlyControllerLoading(isReadOnlyEnabled);
+ }
+ }
+
+ @Test
+ public void testReadOnlyControllerUnloadedWhenDisabledDynamically() throws Exception {
+ setupMiniCluster(initialReadOnlyMode);
+ boolean isReadOnlyEnabled = false;
+ setReadOnlyMode(isReadOnlyEnabled);
+ createTable();
+ verifyMasterReadOnlyControllerLoading(isReadOnlyEnabled);
+ verifyRegionServerReadOnlyControllerLoading(isReadOnlyEnabled);
+ verifyRegionReadOnlyControllerLoading(isReadOnlyEnabled);
+ }
+
+ @Test
+ public void testReadOnlyControllerLoadUnloadedWhenMultipleReadOnlyToggle() throws Exception {
+ setupMiniCluster(initialReadOnlyMode);
+
+ // Ensure region exists before validation
+ setReadOnlyMode(false);
+ createTable();
+ verifyReadOnlyState(false);
+
+ // Define toggle sequence
+ boolean[] toggleSequence = new boolean[] { true, false, // basic toggle
+ true, true, // idempotent enable
+ false, false // idempotent disable
+ };
+
+ for (int i = 0; i < toggleSequence.length; i++) {
+ boolean state = toggleSequence[i];
+ LOG.info("Toggling read-only mode to {} (step {})", state, i);
+
+ setReadOnlyMode(state);
+ verifyReadOnlyState(state);
+ }
+ }
+
+ @Parameters(name = "initialReadOnlyMode={0}")
+ public static Collection<Object[]> parameters() {
+ return Arrays.asList(new Object[][] { { Boolean.TRUE }, { Boolean.FALSE } });
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyControllerEndpointObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyControllerEndpointObserver.java
new file mode 100644
index 00000000000..460d1fd2aae
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyControllerEndpointObserver.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.access;
+
+import static org.mockito.Mockito.mock;
+
+import java.io.IOException;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.testclassification.SecurityTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hbase.thirdparty.com.google.protobuf.Message;
+import org.apache.hbase.thirdparty.com.google.protobuf.Service;
+
+// Tests methods of Endpoint Observer which are implemented in ReadOnlyController,
+// by mocking the coprocessor environment and dependencies.
+@Category({ SecurityTests.class, SmallTests.class })
+public class TestReadOnlyControllerEndpointObserver {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestReadOnlyControllerEndpointObserver.class);
+
+ EndpointReadOnlyController endpointReadOnlyController;
+ // Region Server Coprocessor mocking variables.
+ ObserverContext<? extends RegionCoprocessorEnvironment> ctx;
+ Service service;
+ String methodName;
+ Message request;
+
+ @Before
+ public void setup() throws Exception {
+ endpointReadOnlyController = new EndpointReadOnlyController();
+
+ // mocking variables initialization
+ ctx = mock(ObserverContext.class);
+ service = mock(Service.class);
+ methodName = "testMethod";
+ request = mock(Message.class);
+
+ // Linking the mocks
+ }
+
+ @After
+ public void tearDown() throws Exception {
+
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreEndpointInvocationReadOnlyException() throws IOException {
+ endpointReadOnlyController.preEndpointInvocation(ctx, service, methodName, request);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyControllerMasterObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyControllerMasterObserver.java
new file mode 100644
index 00000000000..215edb49317
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyControllerMasterObserver.java
@@ -0,0 +1,393 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.access;
+
+import static org.mockito.Mockito.mock;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.BalanceRequest;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.net.Address;
+import org.apache.hadoop.hbase.quotas.GlobalQuotaSettings;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.SyncReplicationState;
+import org.apache.hadoop.hbase.testclassification.SecurityTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+// Tests methods of Master Observer which are implemented in ReadOnlyController,
+// by mocking the coprocessor environment and dependencies
+
+@Category({ SecurityTests.class, SmallTests.class })
+public class TestReadOnlyControllerMasterObserver {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestReadOnlyControllerMasterObserver.class);
+
+ MasterReadOnlyController MasterReadOnlyController;
+
+ // Master Coprocessor mocking variables
+ ObserverContext<MasterCoprocessorEnvironment> c, ctx;
+ TableDescriptor desc;
+ RegionInfo[] regions;
+ TableName tableName;
+ TableDescriptor currentDescriptor, newDescriptor;
+ String dstSFT;
+ byte[] family;
+ byte[] splitRow;
+ byte[] splitKey;
+ List<Mutation> metaEntries;
+ RegionInfo[] regionsToMerge;
+ SnapshotDescription snapshot;
+ TableDescriptor tableDescriptor;
+ NamespaceDescriptor ns;
+ NamespaceDescriptor currentNsDescriptor, newNsDescriptor;
+ String namespace;
+ String userName;
+ GlobalQuotaSettings quotas;
+ String regionServer;
+ Set<Address> servers;
+ Set<TableName> tables;
+ String targetGroup;
+ String name;
+ String groupName;
+ BalanceRequest request;
+ String oldName, newName;
+ Map<String, String> configuration;
+ String peerId;
+ ReplicationPeerConfig peerConfig;
+ SyncReplicationState state;
+ UserPermission userPermission;
+ boolean mergeExistingPermissions;
+
+ @Before
+ public void setup() throws Exception {
+ MasterReadOnlyController = new MasterReadOnlyController();
+
+ // mocking variables initialization
+ c = mock(ObserverContext.class);
+ // ctx is created to make naming variable in sync with the Observer interface
+ // methods where 'ctx' is used as the ObserverContext variable name instead of 'c'.
+ // otherwise both are one and the same
+ ctx = c;
+ desc = mock(TableDescriptor.class);
+ regions = new RegionInfo[] {};
+ tableName = TableName.valueOf("testTable");
+ currentDescriptor = mock(TableDescriptor.class);
+ newDescriptor = mock(TableDescriptor.class);
+ dstSFT = "dstSFT";
+ family = Bytes.toBytes("testFamily");
+ splitRow = Bytes.toBytes("splitRow");
+ splitKey = Bytes.toBytes("splitKey");
+ metaEntries = List.of();
+ regionsToMerge = new RegionInfo[] {};
+ snapshot = mock(SnapshotDescription.class);
+ tableDescriptor = mock(TableDescriptor.class);
+ ns = mock(NamespaceDescriptor.class);
+ currentNsDescriptor = mock(NamespaceDescriptor.class);
+ newNsDescriptor = mock(NamespaceDescriptor.class);
+ namespace = "testNamespace";
+ userName = "testUser";
+ quotas = mock(GlobalQuotaSettings.class);
+ regionServer = "testRegionServer";
+ servers = Set.of();
+ tables = Set.of();
+ targetGroup = "targetGroup";
+ name = "testRSGroup";
+ groupName = "testGroupName";
+ request = BalanceRequest.newBuilder().build();
+ oldName = "oldRSGroupName";
+ newName = "newRSGroupName";
+ configuration = Map.of();
+ peerId = "testPeerId";
+ peerConfig = mock(ReplicationPeerConfig.class);
+ state = SyncReplicationState.NONE;
+ userPermission = mock(UserPermission.class);
+ mergeExistingPermissions = false;
+
+ // Linking the mocks:
+ }
+
+ @After
+ public void tearDown() throws Exception {
+
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCreateTableRegionsInfosReadOnlyException() throws IOException {
+ MasterReadOnlyController.preCreateTableRegionsInfos(ctx, desc);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCreateTableReadOnlyException() throws IOException {
+ MasterReadOnlyController.preCreateTable(ctx, desc, regions);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCreateTableActionReadOnlyException() throws IOException {
+ MasterReadOnlyController.preCreateTableAction(ctx, desc, regions);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreDeleteTableReadOnlyException() throws IOException {
+ MasterReadOnlyController.preDeleteTable(ctx, tableName);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreDeleteTableActionReadOnlyException() throws IOException {
+ MasterReadOnlyController.preDeleteTableAction(ctx, tableName);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreTruncateTableReadOnlyException() throws IOException {
+ MasterReadOnlyController.preTruncateTable(ctx, tableName);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreTruncateTableActionReadOnlyException() throws IOException {
+ MasterReadOnlyController.preTruncateTableAction(ctx, tableName);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreModifyTableReadOnlyException() throws IOException {
+ MasterReadOnlyController.preModifyTable(ctx, tableName, currentDescriptor, newDescriptor);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreModifyTableStoreFileTrackerReadOnlyException() throws IOException {
+ MasterReadOnlyController.preModifyTableStoreFileTracker(ctx, tableName, dstSFT);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreModifyColumnFamilyStoreFileTrackerReadOnlyException() throws IOException {
+ MasterReadOnlyController.preModifyColumnFamilyStoreFileTracker(ctx, tableName, family, dstSFT);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreModifyTableActionReadOnlyException() throws IOException {
+ MasterReadOnlyController.preModifyTableAction(ctx, tableName, currentDescriptor, newDescriptor);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreSplitRegionReadOnlyException() throws IOException {
+ MasterReadOnlyController.preSplitRegion(c, tableName, splitRow);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreSplitRegionActionReadOnlyException() throws IOException {
+ MasterReadOnlyController.preSplitRegionAction(c, tableName, splitRow);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreSplitRegionBeforeMETAActionReadOnlyException() throws IOException {
+ MasterReadOnlyController.preSplitRegionBeforeMETAAction(ctx, splitKey, metaEntries);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreSplitRegionAfterMETAActionReadOnlyException() throws IOException {
+ MasterReadOnlyController.preSplitRegionAfterMETAAction(ctx);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreMergeRegionsActionReadOnlyException() throws IOException {
+ MasterReadOnlyController.preMergeRegionsAction(ctx, regionsToMerge);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreMergeRegionsCommitActionReadOnlyException() throws IOException {
+ MasterReadOnlyController.preMergeRegionsCommitAction(ctx, regionsToMerge, metaEntries);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreSnapshotReadOnlyException() throws IOException {
+ MasterReadOnlyController.preSnapshot(ctx, snapshot, tableDescriptor);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCloneSnapshotReadOnlyException() throws IOException {
+ MasterReadOnlyController.preCloneSnapshot(ctx, snapshot, tableDescriptor);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreRestoreSnapshotReadOnlyException() throws IOException {
+ MasterReadOnlyController.preRestoreSnapshot(ctx, snapshot, tableDescriptor);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreDeleteSnapshotReadOnlyException() throws IOException {
+ MasterReadOnlyController.preDeleteSnapshot(ctx, snapshot);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCreateNamespaceReadOnlyException() throws IOException {
+ MasterReadOnlyController.preCreateNamespace(ctx, ns);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreModifyNamespaceReadOnlyException() throws IOException {
+ MasterReadOnlyController.preModifyNamespace(ctx, currentNsDescriptor, newNsDescriptor);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreDeleteNamespaceReadOnlyException() throws IOException {
+ MasterReadOnlyController.preDeleteNamespace(ctx, namespace);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreMasterStoreFlushReadOnlyException() throws IOException {
+ MasterReadOnlyController.preMasterStoreFlush(ctx);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreSetUserQuotaReadOnlyException() throws IOException {
+ MasterReadOnlyController.preSetUserQuota(ctx, userName, quotas);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreSetUserQuotaOnTableReadOnlyException() throws IOException {
+ MasterReadOnlyController.preSetUserQuota(ctx, userName, tableName, quotas);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreSetUserQuotaOnNamespaceReadOnlyException() throws IOException {
+ MasterReadOnlyController.preSetUserQuota(ctx, userName, namespace, quotas);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreSetTableQuotaReadOnlyException() throws IOException {
+ MasterReadOnlyController.preSetTableQuota(ctx, tableName, quotas);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreSetNamespaceQuotaReadOnlyException() throws IOException {
+ MasterReadOnlyController.preSetNamespaceQuota(ctx, namespace, quotas);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreSetRegionServerQuotaReadOnlyException() throws IOException {
+ MasterReadOnlyController.preSetRegionServerQuota(ctx, regionServer, quotas);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreMergeRegionsReadOnlyException() throws IOException {
+ MasterReadOnlyController.preMergeRegions(ctx, regionsToMerge);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreMoveServersAndTablesReadOnlyException() throws IOException {
+ MasterReadOnlyController.preMoveServersAndTables(ctx, servers, tables, targetGroup);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreMoveServersReadOnlyException() throws IOException {
+ MasterReadOnlyController.preMoveServers(ctx, servers, targetGroup);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreMoveTablesReadOnlyException() throws IOException {
+ MasterReadOnlyController.preMoveTables(ctx, tables, targetGroup);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreAddRSGroupReadOnlyException() throws IOException {
+ MasterReadOnlyController.preAddRSGroup(ctx, name);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreRemoveRSGroupReadOnlyException() throws IOException {
+ MasterReadOnlyController.preRemoveRSGroup(ctx, name);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreBalanceRSGroupReadOnlyException() throws IOException {
+ MasterReadOnlyController.preBalanceRSGroup(ctx, groupName, request);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreRemoveServersReadOnlyException() throws IOException {
+ MasterReadOnlyController.preRemoveServers(ctx, servers);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreRenameRSGroupReadOnlyException() throws IOException {
+ MasterReadOnlyController.preRenameRSGroup(ctx, oldName, newName);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreUpdateRSGroupConfigReadOnlyException() throws IOException {
+ MasterReadOnlyController.preUpdateRSGroupConfig(ctx, groupName, configuration);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreAddReplicationPeerReadOnlyException() throws IOException {
+ MasterReadOnlyController.preAddReplicationPeer(ctx, peerId, peerConfig);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreRemoveReplicationPeerReadOnlyException() throws IOException {
+ MasterReadOnlyController.preRemoveReplicationPeer(ctx, peerId);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreEnableReplicationPeerReadOnlyException() throws IOException {
+ MasterReadOnlyController.preEnableReplicationPeer(ctx, peerId);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreDisableReplicationPeerReadOnlyException() throws IOException {
+ MasterReadOnlyController.preDisableReplicationPeer(ctx, peerId);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreUpdateReplicationPeerConfigReadOnlyException() throws IOException {
+ MasterReadOnlyController.preUpdateReplicationPeerConfig(ctx, peerId, peerConfig);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreTransitReplicationPeerSyncReplicationStateReadOnlyException()
+ throws IOException {
+ MasterReadOnlyController.preTransitReplicationPeerSyncReplicationState(ctx, peerId, state);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreGrantReadOnlyException() throws IOException {
+ MasterReadOnlyController.preGrant(ctx, userPermission, mergeExistingPermissions);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreRevokeReadOnlyException() throws IOException {
+ MasterReadOnlyController.preRevoke(ctx, userPermission);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyControllerRegionObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyControllerRegionObserver.java
new file mode 100644
index 00000000000..d7176b90055
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyControllerRegionObserver.java
@@ -0,0 +1,483 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.access;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.CompareOperator;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.CheckAndMutate;
+import org.apache.hadoop.hbase.client.CheckAndMutateResult;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.filter.ByteArrayComparable;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.regionserver.FlushLifeCycleTracker;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
+import org.apache.hadoop.hbase.regionserver.ScanOptions;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.testclassification.SecurityTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.wal.WALEdit;
+import org.apache.hadoop.hbase.wal.WALKey;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+// Tests methods of Region Observer interface which are implemented in ReadOnlyController,
+// by mocking the coprocessor environment and dependencies.
+// V1 and V2 means version 1 and version 2 of the coprocessor method signature.
+// For example, prePut has 2 versions:
+// V1: prePut(ObserverContext<RegionCoprocessorEnvironment> c, Put put, WALEdit edit)
+// V2: prePut(ObserverContext<RegionCoprocessorEnvironment> c, Put put, WALEdit edit, Durability durability)
+@Category({ SecurityTests.class, SmallTests.class })
+public class TestReadOnlyControllerRegionObserver {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestReadOnlyControllerRegionObserver.class);
+
+ RegionReadOnlyController regionReadOnlyController;
+
+ // Region Coprocessor mocking variables
+ ObserverContext<RegionCoprocessorEnvironment> c, ctx;
+ RegionCoprocessorEnvironment env;
+ RegionInfo regionInfo;
+ Store store;
+ InternalScanner scanner;
+ ScanOptions options;
+ FlushLifeCycleTracker flushLifeCycleTracker;
+ List<StoreFile> candidates;
+ CompactionLifeCycleTracker compactionLifeCycleTracker;
+ ScanType scanType;
+ CompactionRequest compactionRequest;
+ TableName tableName;
+ Put put;
+ WALEdit edit;
+ Durability durability;
+ Delete delete;
+ MiniBatchOperationInProgress<Mutation> miniBatchOp;
+ byte[] row;
+ byte[] family;
+ byte[] qualifier;
+ Filter filter;
+ CompareOperator op;
+ ByteArrayComparable comparator;
+ boolean result;
+ CheckAndMutate checkAndMutate;
+ CheckAndMutateResult checkAndMutateResult;
+ Append append;
+ Increment increment;
+ RegionInfo info;
+ Path edits;
+ List<Pair<byte[], String>> familyPaths;
+ List<Pair<Path, Path>> pairs;
+ WALKey key;
+
+ @Before
+ public void setup() throws Exception {
+ regionReadOnlyController = new RegionReadOnlyController();
+
+ // mocking variables initialization
+ c = mock(ObserverContext.class);
+ // ctx is created to make naming variable in sync with the Observer interface
+ // methods where 'ctx' is used as the ObserverContext variable name instead of 'c'.
+ // otherwise both are one and the same
+ ctx = c;
+ env = mock(RegionCoprocessorEnvironment.class);
+ regionInfo = mock(RegionInfo.class);
+ store = mock(Store.class);
+ scanner = mock(InternalScanner.class);
+ options = mock(ScanOptions.class);
+ flushLifeCycleTracker = mock(FlushLifeCycleTracker.class);
+ compactionLifeCycleTracker = mock(CompactionLifeCycleTracker.class);
+ StoreFile sf1 = mock(StoreFile.class);
+ StoreFile sf2 = mock(StoreFile.class);
+ candidates = List.of(sf1, sf2);
+ scanType = ScanType.COMPACT_DROP_DELETES;
+ compactionRequest = mock(CompactionRequest.class);
+ tableName = TableName.valueOf("testTable");
+ put = mock(Put.class);
+ edit = mock(WALEdit.class);
+ durability = Durability.USE_DEFAULT;
+ delete = mock(Delete.class);
+ miniBatchOp = mock(MiniBatchOperationInProgress.class);
+ row = Bytes.toBytes("test-row");
+ family = Bytes.toBytes("test-family");
+ qualifier = Bytes.toBytes("test-qualifier");
+ filter = mock(Filter.class);
+ op = CompareOperator.NO_OP;
+ comparator = mock(ByteArrayComparable.class);
+ result = false;
+ checkAndMutate = CheckAndMutate
+ .newBuilder(Bytes.toBytes("test-row")).ifEquals(Bytes.toBytes("test-family"),
+ Bytes.toBytes("test-qualifier"), Bytes.toBytes("test-value"))
+ .build(new Put(Bytes.toBytes("test-row")));
+ checkAndMutateResult = mock(CheckAndMutateResult.class);
+ append = mock(Append.class);
+ increment = mock(Increment.class);
+ edits = mock(Path.class);
+ familyPaths = List.of(new Pair<>(Bytes.toBytes("test-family"), "/path/to/hfile1"),
+ new Pair<>(Bytes.toBytes("test-family"), "/path/to/hfile2"));
+ pairs = List.of(new Pair<>(mock(Path.class), mock(Path.class)),
+ new Pair<>(mock(Path.class), mock(Path.class)));
+ key = mock(WALKey.class);
+
+ // Linking the mocks:
+ when(c.getEnvironment()).thenReturn(env);
+ when(env.getRegionInfo()).thenReturn(regionInfo);
+ when(regionInfo.getTable()).thenReturn(tableName);
+ when(key.getTableName()).thenReturn(tableName);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+
+ }
+
+ private void mockOperationForMetaTable() {
+ when(regionInfo.getTable()).thenReturn(TableName.META_TABLE_NAME);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreFlushV1ReadOnlyException() throws IOException {
+ regionReadOnlyController.preFlush(c, flushLifeCycleTracker);
+ }
+
+ @Test
+ public void testPreFlushV1ReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.preFlush(c, flushLifeCycleTracker);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreFlushV2ReadOnlyException() throws IOException {
+ regionReadOnlyController.preFlush(c, store, scanner, flushLifeCycleTracker);
+ }
+
+ @Test
+ public void testPreFlushV2ReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.preFlush(c, store, scanner, flushLifeCycleTracker);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreFlushScannerOpenReadOnlyException() throws IOException {
+ regionReadOnlyController.preFlushScannerOpen(c, store, options, flushLifeCycleTracker);
+ }
+
+ @Test
+ public void testPreFlushScannerOpenReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.preFlushScannerOpen(c, store, options, flushLifeCycleTracker);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreMemStoreCompactionReadOnlyException() throws IOException {
+ regionReadOnlyController.preMemStoreCompaction(c, store);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreMemStoreCompactionCompactScannerOpenReadOnlyException() throws IOException {
+ regionReadOnlyController.preMemStoreCompactionCompactScannerOpen(c, store, options);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreMemStoreCompactionCompactReadOnlyException() throws IOException {
+ regionReadOnlyController.preMemStoreCompactionCompact(c, store, scanner);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCompactSelectionReadOnlyException() throws IOException {
+ regionReadOnlyController.preCompactSelection(c, store, candidates, compactionLifeCycleTracker);
+ }
+
+ @Test
+ public void testPreCompactSelectionReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.preCompactSelection(c, store, candidates, compactionLifeCycleTracker);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCompactScannerOpenReadOnlyException() throws IOException {
+ regionReadOnlyController.preCompactScannerOpen(c, store, scanType, options,
+ compactionLifeCycleTracker, compactionRequest);
+ }
+
+ @Test
+ public void testPreCompactScannerOpenReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.preCompactScannerOpen(c, store, scanType, options,
+ compactionLifeCycleTracker, compactionRequest);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCompactReadOnlyException() throws IOException {
+ regionReadOnlyController.preCompact(c, store, scanner, scanType, compactionLifeCycleTracker,
+ compactionRequest);
+ }
+
+ @Test
+ public void testPreCompactReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.preCompact(c, store, scanner, scanType, compactionLifeCycleTracker,
+ compactionRequest);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPrePutV1ReadOnlyException() throws IOException {
+ regionReadOnlyController.prePut(c, put, edit);
+ }
+
+ @Test
+ public void testPrePutV1ReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.prePut(c, put, edit);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPrePutV2ReadOnlyException() throws IOException {
+ regionReadOnlyController.prePut(c, put, edit, durability);
+ }
+
+ @Test
+ public void testPrePutV2ReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.prePut(c, put, edit, durability);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreDeleteV1ReadOnlyException() throws IOException {
+ regionReadOnlyController.preDelete(c, delete, edit);
+ }
+
+ @Test
+ public void testPreDeleteV1ReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.preDelete(c, delete, edit);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreDeleteV2ReadOnlyException() throws IOException {
+ regionReadOnlyController.preDelete(c, delete, edit, durability);
+ }
+
+ @Test
+ public void testPreDeleteV2ReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.preDelete(c, delete, edit, durability);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreBatchMutateReadOnlyException() throws IOException {
+ regionReadOnlyController.preBatchMutate(c, miniBatchOp);
+ }
+
+ @Test
+ public void testPreBatchMutateReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.preBatchMutate(c, miniBatchOp);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCheckAndPutV1ReadOnlyException() throws IOException {
+ regionReadOnlyController.preCheckAndPut(c, row, family, qualifier, op, comparator, put, result);
+ }
+
+ @Test
+ public void testPreCheckAndPutV1ReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.preCheckAndPut(c, row, family, qualifier, op, comparator, put, result);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCheckAndPutV2ReadOnlyException() throws IOException {
+ regionReadOnlyController.preCheckAndPut(c, row, filter, put, result);
+ }
+
+ @Test
+ public void testPreCheckAndPutV2ReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.preCheckAndPut(c, row, filter, put, result);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCheckAndPutAfterRowLockV1ReadOnlyException() throws IOException {
+ regionReadOnlyController.preCheckAndPutAfterRowLock(c, row, family, qualifier, op, comparator,
+ put, result);
+ }
+
+ @Test
+ public void testPreCheckAndPutAfterRowLockV1ReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.preCheckAndPutAfterRowLock(c, row, family, qualifier, op, comparator,
+ put, result);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCheckAndPutAfterRowLockV2ReadOnlyException() throws IOException {
+ regionReadOnlyController.preCheckAndPutAfterRowLock(c, row, filter, put, result);
+ }
+
+ @Test
+ public void testPreCheckAndPutAfterRowLockV2ReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.preCheckAndPutAfterRowLock(c, row, filter, put, result);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCheckAndDeleteV1ReadOnlyException() throws IOException {
+ regionReadOnlyController.preCheckAndDelete(c, row, family, qualifier, op, comparator, delete,
+ result);
+ }
+
+ @Test
+ public void testPreCheckAndDeleteV1ReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.preCheckAndDelete(c, row, family, qualifier, op, comparator, delete,
+ result);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCheckAndDeleteV2ReadOnlyException() throws IOException {
+ regionReadOnlyController.preCheckAndDelete(c, row, filter, delete, result);
+ }
+
+ @Test
+ public void testPreCheckAndDeleteV2ReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.preCheckAndDelete(c, row, filter, delete, result);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCheckAndDeleteAfterRowLockV1ReadOnlyException() throws IOException {
+ regionReadOnlyController.preCheckAndDeleteAfterRowLock(c, row, family, qualifier, op,
+ comparator, delete, result);
+ }
+
+ @Test
+ public void testPreCheckAndDeleteAfterRowLockV1ReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.preCheckAndDeleteAfterRowLock(c, row, family, qualifier, op,
+ comparator, delete, result);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCheckAndDeleteAfterRowLockV2ReadOnlyException() throws IOException {
+ regionReadOnlyController.preCheckAndDeleteAfterRowLock(c, row, filter, delete, result);
+ }
+
+ @Test
+ public void testPreCheckAndDeleteAfterRowLockV2ReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.preCheckAndDeleteAfterRowLock(c, row, filter, delete, result);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCheckAndMutateReadOnlyException() throws IOException {
+ regionReadOnlyController.preCheckAndMutate(c, checkAndMutate, checkAndMutateResult);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCheckAndMutateAfterRowLockReadOnlyException() throws IOException {
+ regionReadOnlyController.preCheckAndMutateAfterRowLock(c, checkAndMutate, checkAndMutateResult);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreAppendV1ReadOnlyException() throws IOException {
+ regionReadOnlyController.preAppend(c, append);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreAppendV2ReadOnlyException() throws IOException {
+ regionReadOnlyController.preAppend(c, append, edit);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreAppendAfterRowLockReadOnlyException() throws IOException {
+ regionReadOnlyController.preAppendAfterRowLock(c, append);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreIncrementV1ReadOnlyException() throws IOException {
+ regionReadOnlyController.preIncrement(c, increment);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreIncrementV2ReadOnlyException() throws IOException {
+ regionReadOnlyController.preIncrement(c, increment, edit);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreIncrementAfterRowLockReadOnlyException() throws IOException {
+ regionReadOnlyController.preIncrementAfterRowLock(c, increment);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreReplayWALsReadOnlyException() throws IOException {
+ regionReadOnlyController.preReplayWALs(ctx, info, edits);
+ }
+
+ @Test
+ public void testPreReplayWALsReadOnlyMetaNoException() throws IOException {
+ mockOperationForMetaTable();
+ regionReadOnlyController.preReplayWALs(ctx, info, edits);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreBulkLoadHFileReadOnlyException() throws IOException {
+ regionReadOnlyController.preBulkLoadHFile(ctx, familyPaths);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreCommitStoreFileReadOnlyException() throws IOException {
+ regionReadOnlyController.preCommitStoreFile(ctx, family, pairs);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreWALAppendReadOnlyException() throws IOException {
+ regionReadOnlyController.preWALAppend(ctx, key, edit);
+ }
+
+ @Test
+ public void testPreWALAppendReadOnlyMetaNoException() throws IOException {
+ when(key.getTableName()).thenReturn(TableName.META_TABLE_NAME);
+ regionReadOnlyController.preWALAppend(ctx, key, edit);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyControllerRegionServerObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyControllerRegionServerObserver.java
new file mode 100644
index 00000000000..382e55a0c40
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyControllerRegionServerObserver.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.access;
+
+import static org.mockito.Mockito.mock;
+
+import java.io.IOException;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
+import org.apache.hadoop.hbase.testclassification.SecurityTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
+
+// Tests methods of Region Server Observer which are implemented in ReadOnlyController,
+// by mocking the coprocessor environment and dependencies
+@Category({ SecurityTests.class, SmallTests.class })
+public class TestReadOnlyControllerRegionServerObserver {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestReadOnlyControllerRegionServerObserver.class);
+
+ RegionServerReadOnlyController regionServerReadOnlyController;
+
+ // Region Server Coprocessor mocking variables
+ ObserverContext<RegionServerCoprocessorEnvironment> ctx;
+ AdminProtos.WALEntry walEntry;
+ Mutation mutation;
+
+ @Before
+ public void setup() throws Exception {
+ regionServerReadOnlyController = new RegionServerReadOnlyController();
+
+ // mocking variables initialization
+ ctx = mock(ObserverContext.class);
+ walEntry = AdminProtos.WALEntry.newBuilder()
+ .setKey(WALProtos.WALKey.newBuilder().setTableName(ByteString.copyFromUtf8("test"))
+ .setEncodedRegionName(ByteString.copyFromUtf8("regionA")).setLogSequenceNumber(100)
+ .setWriteTime(2).build())
+ .build();
+ mutation = mock(Mutation.class);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreRollWALWriterRequestReadOnlyException() throws IOException {
+ regionServerReadOnlyController.preRollWALWriterRequest(ctx);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreReplicationSinkBatchMutateReadOnlyException() throws IOException {
+ regionServerReadOnlyController.preReplicationSinkBatchMutate(ctx, walEntry, mutation);
+ }
+
+ @Test(expected = DoNotRetryIOException.class)
+ public void testPreReplicateLogEntriesReadOnlyException() throws IOException {
+ regionServerReadOnlyController.preReplicateLogEntries(ctx);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyManageActiveClusterFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyManageActiveClusterFile.java
new file mode 100644
index 00000000000..eaeda593ea4
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestReadOnlyManageActiveClusterFile.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.access;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.SecurityTests;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Category({ SecurityTests.class, MediumTests.class })
+public class TestReadOnlyManageActiveClusterFile {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestReadOnlyManageActiveClusterFile.class);
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestReadOnlyManageActiveClusterFile.class);
+ private final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
+ private static final int NUM_RS = 1;
+ private static Configuration conf;
+ HMaster master;
+ HRegionServer regionServer;
+
+ MasterFileSystem mfs;
+ Path rootDir;
+ FileSystem fs;
+ Path activeClusterFile;
+
+ @Before
+ public void setup() throws Exception {
+ conf = TEST_UTIL.getConfiguration();
+
+ // Set up test class with Read-Only mode disabled so a table can be created
+ conf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, false);
+ // Start the test cluster
+ TEST_UTIL.startMiniCluster(NUM_RS);
+ master = TEST_UTIL.getMiniHBaseCluster().getMaster();
+ regionServer = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
+
+ mfs = master.getMasterFileSystem();
+ rootDir = mfs.getRootDir();
+ fs = mfs.getFileSystem();
+ activeClusterFile = new Path(rootDir, HConstants.ACTIVE_CLUSTER_SUFFIX_FILE_NAME);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ private void setReadOnlyMode(boolean enabled) {
+ conf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, enabled);
+ master.getConfigurationManager().notifyAllObservers(conf);
+ regionServer.getConfigurationManager().notifyAllObservers(conf);
+ }
+
+ private void restartCluster() throws IOException, InterruptedException {
+ TEST_UTIL.getMiniHBaseCluster().shutdown();
+ TEST_UTIL.restartHBaseCluster(NUM_RS);
+ TEST_UTIL.waitUntilNoRegionsInTransition();
+
+ master = TEST_UTIL.getMiniHBaseCluster().getMaster();
+ regionServer = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
+
+ MasterFileSystem mfs = master.getMasterFileSystem();
+ fs = mfs.getFileSystem();
+ activeClusterFile = new Path(mfs.getRootDir(), HConstants.ACTIVE_CLUSTER_SUFFIX_FILE_NAME);
+ }
+
+ private void overwriteExistingFile() throws IOException {
+ try (FSDataOutputStream out = fs.create(activeClusterFile, true)) {
+ out.writeBytes("newClusterId");
+ }
+ }
+
+ private boolean activeClusterIdFileExists() throws IOException {
+ return fs.exists(activeClusterFile);
+ }
+
+ @Test
+ public void testActiveClusterIdFileCreationWhenReadOnlyDisabled()
+ throws IOException, InterruptedException {
+ setReadOnlyMode(false);
+ assertTrue(activeClusterIdFileExists());
+ }
+
+ @Test
+ public void testActiveClusterIdFileDeletionWhenReadOnlyEnabled()
+ throws IOException, InterruptedException {
+ setReadOnlyMode(true);
+ assertFalse(activeClusterIdFileExists());
+ }
+
+ @Test
+ public void testDeleteActiveClusterIdFileWhenSwitchingToReadOnlyIfOwnedByCluster()
+ throws IOException, InterruptedException {
+ // At the start cluster is in active mode hence set readonly mode and restart
+ conf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, true);
+ // Restart the cluster to trigger the deletion of the active cluster ID file
+ restartCluster();
+ // Should delete the active cluster ID file since it is owned by the cluster
+ assertFalse(activeClusterIdFileExists());
+ }
+
+ @Test
+ public void testDoNotDeleteActiveClusterIdFileWhenSwitchingToReadOnlyIfNotOwnedByCluster()
+ throws IOException, InterruptedException {
+ // Change the content of Active Cluster file to simulate the scenario where the file is not
+ // owned by the cluster and then set readonly mode and restart
+ overwriteExistingFile();
+ // At the start cluster is in active mode hence set readonly mode and restart
+ conf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, true);
+ // Restart the cluster to trigger the deletion of the active cluster ID file
+ restartCluster();
+ // As Active cluster file is not owned by the cluster, it should not be deleted even when
+ // switching to readonly mode
+ assertTrue(activeClusterIdFileExists());
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorConfigurationUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorConfigurationUtil.java
new file mode 100644
index 00000000000..eeda7194982
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorConfigurationUtil.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.security.access.BulkLoadReadOnlyController;
+import org.apache.hadoop.hbase.security.access.EndpointReadOnlyController;
+import org.apache.hadoop.hbase.security.access.MasterReadOnlyController;
+import org.apache.hadoop.hbase.security.access.RegionReadOnlyController;
+import org.apache.hadoop.hbase.security.access.RegionServerReadOnlyController;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestCoprocessorConfigurationUtil {
+
+ private Configuration conf;
+ private String key;
+
+ @Before
+ public void setUp() {
+ conf = new Configuration();
+ key = "test.key";
+ }
+
+ @Test
+ public void testAddCoprocessorsEmptyCPList() {
+ CoprocessorConfigurationUtil.addCoprocessors(conf, key, List.of("cp1", "cp2"));
+ assertArrayEquals(new String[] { "cp1", "cp2" }, conf.getStrings(key));
+ }
+
+ @Test
+ public void testAddCoprocessorsNonEmptyCPList() {
+ conf.setStrings(key, "cp1");
+ CoprocessorConfigurationUtil.addCoprocessors(conf, key, List.of("cp1", "cp2"));
+ assertArrayEquals(new String[] { "cp1", "cp2" }, conf.getStrings(key));
+ }
+
+ @Test
+ public void testAddCoprocessorsNoChange() {
+ conf.setStrings(key, "cp1");
+ CoprocessorConfigurationUtil.addCoprocessors(conf, key, List.of("cp1"));
+ assertArrayEquals(new String[] { "cp1" }, conf.getStrings(key));
+ }
+
+ @Test
+ public void testAddCoprocessorsIdempotent() {
+ CoprocessorConfigurationUtil.addCoprocessors(conf, key, List.of("cp1", "cp2"));
+ // Call again
+ CoprocessorConfigurationUtil.addCoprocessors(conf, key, List.of("cp1", "cp2"));
+ assertArrayEquals(new String[] { "cp1", "cp2" }, conf.getStrings(key));
+ }
+
+ @Test
+ public void testAddCoprocessorsIdempotentWithOverlap() {
+ CoprocessorConfigurationUtil.addCoprocessors(conf, key, List.of("cp1"));
+ CoprocessorConfigurationUtil.addCoprocessors(conf, key, List.of("cp1", "cp2"));
+ CoprocessorConfigurationUtil.addCoprocessors(conf, key, List.of("cp2"));
+ assertArrayEquals(new String[] { "cp1", "cp2" }, conf.getStrings(key));
+ }
+
+ @Test
+ public void testRemoveCoprocessorsEmptyCPList() {
+ CoprocessorConfigurationUtil.removeCoprocessors(conf, key, List.of("cp1"));
+ assertNull(conf.getStrings(key));
+ }
+
+ @Test
+ public void testRemoveCoprocessorsNonEmptyCPList() {
+ conf.setStrings(key, "cp1", "cp2", "cp3");
+ CoprocessorConfigurationUtil.removeCoprocessors(conf, key, List.of("cp2"));
+ assertArrayEquals(new String[] { "cp1", "cp3" }, conf.getStrings(key));
+ }
+
+ @Test
+ public void testRemoveCoprocessorsNoChange() {
+ conf.setStrings(key, "cp1");
+ CoprocessorConfigurationUtil.removeCoprocessors(conf, key, List.of("cp2"));
+ assertArrayEquals(new String[] { "cp1" }, conf.getStrings(key));
+ }
+
+ @Test
+ public void testRemoveCoprocessorsIdempotent() {
+ conf.setStrings(key, "cp1", "cp2");
+ CoprocessorConfigurationUtil.removeCoprocessors(conf, key, List.of("cp2"));
+ // Call again
+ CoprocessorConfigurationUtil.removeCoprocessors(conf, key, List.of("cp2"));
+ assertArrayEquals(new String[] { "cp1" }, conf.getStrings(key));
+ }
+
+ @Test
+ public void testRemoveCoprocessorsIdempotentWhenNotPresent() {
+ conf.setStrings(key, "cp1");
+ CoprocessorConfigurationUtil.removeCoprocessors(conf, key, List.of("cp2"));
+ CoprocessorConfigurationUtil.removeCoprocessors(conf, key, List.of("cp2"));
+ assertArrayEquals(new String[] { "cp1" }, conf.getStrings(key));
+ }
+
+ private void assertEnable(String key, List<String> expected) {
+ CoprocessorConfigurationUtil.syncReadOnlyConfigurations(true, conf, key);
+ assertTrue(conf.getBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, false));
+ String[] result = conf.getStrings(key);
+ assertNotNull(result);
+ assertEquals(expected.size(), result.length);
+ assertTrue(Arrays.asList(result).containsAll(expected));
+ }
+
+ @Test
+ public void testSyncReadOnlyConfigurationsReadOnlyEnableAllKeys() {
+ assertEnable(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
+ List.of(MasterReadOnlyController.class.getName()));
+
+ assertEnable(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY,
+ List.of(RegionServerReadOnlyController.class.getName()));
+ assertEnable(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
+ List.of(RegionReadOnlyController.class.getName(), BulkLoadReadOnlyController.class.getName(),
+ EndpointReadOnlyController.class.getName()));
+ }
+
+ private void assertDisable(String key, List<String> initialCoprocs) {
+ conf.setStrings(key, initialCoprocs.toArray(new String[0]));
+ conf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, true);
+ CoprocessorConfigurationUtil.syncReadOnlyConfigurations(false, conf, key);
+ assertFalse(conf.getBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, true));
+ String[] result = conf.getStrings(key);
+ assertTrue(result == null || result.length == 0);
+ }
+
+ @Test
+ public void testSyncReadOnlyConfigurationsReadOnlyDisableAllKeys() {
+ assertDisable(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
+ List.of(MasterReadOnlyController.class.getName()));
+
+ assertDisable(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY,
+ List.of(RegionServerReadOnlyController.class.getName()));
+
+ assertDisable(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
+ List.of(RegionReadOnlyController.class.getName(), BulkLoadReadOnlyController.class.getName(),
+ EndpointReadOnlyController.class.getName()));
+ }
+
+ @Test
+ public void testSyncReadOnlyConfigurationsReadOnlyEnablePreservesExistingCoprocessors() {
+ String key = CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY;
+ conf.setStrings(key, "existingCp");
+ CoprocessorConfigurationUtil.syncReadOnlyConfigurations(true, conf, key);
+ List<String> result = Arrays.asList(conf.getStrings(key));
+ assertTrue(result.contains("existingCp"));
+ assertTrue(result.contains(MasterReadOnlyController.class.getName()));
+ }
+
+ @Test
+ public void testSyncReadOnlyConfigurationsReadOnlyDisableRemovesOnlyReadOnlyCoprocessor() {
+ Configuration conf = new Configuration(false);
+ String key = CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY;
+ String existingCp = "org.example.OtherCP";
+ conf.setStrings(key, existingCp, MasterReadOnlyController.class.getName());
+ CoprocessorConfigurationUtil.syncReadOnlyConfigurations(false, conf, key);
+ String[] cps = conf.getStrings(key);
+ assertNotNull(cps);
+ assertEquals(1, cps.length);
+ assertEquals(existingCp, cps[0]);
+ }
+
+ @Test
+ public void testSyncReadOnlyConfigurationsIsIdempotent() {
+ Configuration conf = new Configuration(false);
+ String key = CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY;
+ CoprocessorConfigurationUtil.syncReadOnlyConfigurations(true, conf, key);
+ CoprocessorConfigurationUtil.syncReadOnlyConfigurations(true, conf, key);
+ String[] cps = conf.getStrings(key);
+ assertNotNull(cps);
+ assertEquals(1, cps.length);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java
index 5e2b4b52950..6cf83fba8a0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.junit.AfterClass;
@@ -483,6 +484,38 @@ public class TestFSTableDescriptors {
assertFalse(fs.exists(brokenFile));
}
+ @Test
+ public void testFSTableDescriptorsSkipsForeignMetaTables() throws Exception {
+ FileSystem fs = FileSystem.get(UTIL.getConfiguration());
+ String[] metaTables = { "meta_replica1", "meta" };
+ Path hbaseNamespaceDir = new Path(testDir, HConstants.BASE_NAMESPACE_DIR + "/hbase");
+ fs.mkdirs(hbaseNamespaceDir);
+
+ for (String metaTable : metaTables) {
+ TableName tableName = TableName.valueOf("hbase", metaTable);
+ Path metaTableDir = new Path(hbaseNamespaceDir, metaTable);
+ fs.mkdirs(metaTableDir);
+ fs.mkdirs(new Path(metaTableDir, FSTableDescriptors.TABLEINFO_DIR));
+ fs.mkdirs(new Path(metaTableDir, "abcdef0123456789"));
+
+ TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
+ .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(HConstants.CATALOG_FAMILY)
+ .setMaxVersions(HConstants.DEFAULT_HBASE_META_VERSIONS).setInMemory(true)
+ .setBlocksize(HConstants.DEFAULT_HBASE_META_BLOCK_SIZE)
+ .setBloomFilterType(BloomType.ROWCOL).build())
+ .build();
+
+ Path tableDir = CommonFSUtils.getTableDir(testDir, tableName);
+ FSTableDescriptors.createTableDescriptorForTableDirectory(fs, tableDir, tableDescriptor,
+ false);
+ }
+ FSTableDescriptors tableDescriptors = new FSTableDescriptors(fs, testDir);
+ Map<String, TableDescriptor> allTables = tableDescriptors.getAll();
+
+ assertFalse("Should not contain meta_replica1", allTables.containsKey("hbase:meta_replica1"));
+ assertTrue("Should include the local hbase:meta", allTables.containsKey("hbase:meta"));
+ }
+
private static class FSTableDescriptorsTest extends FSTableDescriptors {
public FSTableDescriptorsTest(FileSystem fs, Path rootdir) {
diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb
index 8827db0daef..970cd2fcd4d 100644
--- a/hbase-shell/src/main/ruby/hbase/admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -1927,6 +1927,32 @@ module Hbase
def list_tables_by_state(isEnabled)
@admin.listTableNamesByState(isEnabled).map(&:getNameAsString)
end
+
+ #----------------------------------------------------------------------------------------------
+ # Refresh hbase:meta table by syncing with the backing storage
+ def refresh_meta()
+ @admin.refreshMeta()
+ end
+
+ #----------------------------------------------------------------------------------------------
+ # Refresh HFiles for the table
+ def refresh_hfiles(args = {})
+ table_name = args.fetch(TABLE_NAME, nil)
+ namespace = args.fetch(NAMESPACE, nil)
+ if namespace && table_name
+ raise ArgumentError, "Specify either a TABLE_NAME or a NAMESPACE, not both"
+ elsif namespace == "" || table_name == ""
+ raise ArgumentError, "TABLE_NAME or NAMESPACE cannot be empty string"
+ elsif namespace.is_a?(Array) || table_name.is_a?(Array)
+ raise ArgumentError, "TABLE_NAME or NAMESPACE must be a single string, not an array"
+ elsif namespace
+ @admin.refreshHFiles(namespace)
+ elsif table_name
+ @admin.refreshHFiles(org.apache.hadoop.hbase.TableName.valueOf(table_name))
+ else
+ @admin.refreshHFiles()
+ end
+ end
end
# rubocop:enable Metrics/ClassLength
end
diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb
index c408bf06b22..0e278be2cba 100644
--- a/hbase-shell/src/main/ruby/shell.rb
+++ b/hbase-shell/src/main/ruby/shell.rb
@@ -496,6 +496,8 @@ Shell.load_command_group(
decommission_regionservers
recommission_regionserver
truncate_region
+ refresh_meta
+ refresh_hfiles
],
# TODO: remove older hlog_roll command
aliases: {
diff --git a/hbase-shell/src/main/ruby/shell/commands/refresh_hfiles.rb b/hbase-shell/src/main/ruby/shell/commands/refresh_hfiles.rb
new file mode 100644
index 00000000000..1ec0c74381a
--- /dev/null
+++ b/hbase-shell/src/main/ruby/shell/commands/refresh_hfiles.rb
@@ -0,0 +1,64 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+ module Commands
+ class RefreshHfiles < Command
+ def help
+ return <<-EOF
+Refresh HFiles/storefiles for table(s) or namespaces.
+
+Allowed Syntax:
+hbase> refresh_hfiles
+hbase> refresh_hfiles 'TABLE_NAME' => 'test_table'
+hbase> refresh_hfiles 'TABLE_NAME' => 'namespace:test_table'
+hbase> refresh_hfiles 'NAMESPACE' => 'test_namespace'
+
+Behavior:
+- Without any argument, it refreshes HFiles for all user tables in HBase.
+- If 'TABLE_NAME' is provided: Refreshes HFiles for that specific table.
+ - If provided without a namespace qualifier (e.g., 'TABLE_NAME' => 'test_table'),
+ it refreshes HFiles for that table in the default namespace.
+ - If provided with a namespace qualifier (e.g., 'TABLE_NAME' => 'namespace:test_table'),
+ it refreshes HFiles for the table in the specified namespace.
+- With 'NAMESPACE', it refreshes HFiles for all tables in the given namespace.
+On successful submission, it returns the procedure ID (procId). Otherwise, it throws an exception.
+
+Important Note:
+This command should ideally be run on a read-replica cluster,
+ and only after successfully executing refresh_meta.
+
+Not Allowed:
+hbase> refresh_hfiles 'TABLE_NAME' => 'test_table', 'NAMESPACE' => 'test_namespace'
+
+Passing both 'TABLE_NAME' and 'NAMESPACE' is not allowed to avoid ambiguity.
+Otherwise, it is unclear whether the user intends to:
+1. Refresh HFiles for 'test_table' under 'test_namespace',
+or
+2. Refresh HFiles for 'test_table' under the default namespace and
+ all tables under 'test_namespace'.
+To prevent such confusion, only one argument should be provided per command.
+
+EOF
+ end
+ def command(args = {})
+ admin.refresh_hfiles(args)
+ end
+ end
+ end
+end
diff --git a/hbase-shell/src/main/ruby/shell/commands/refresh_meta.rb b/hbase-shell/src/main/ruby/shell/commands/refresh_meta.rb
new file mode 100644
index 00000000000..8c5acb49dc5
--- /dev/null
+++ b/hbase-shell/src/main/ruby/shell/commands/refresh_meta.rb
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+ module Commands
+ class RefreshMeta < Command
+ def help
+ <<-EOF
+Refresh the hbase:meta table by syncing with backing storage.
+This command is used in Read Replica clusters to pick up new
+tables and regions from the shared storage.
+Examples:
+
+ hbase> refresh_meta
+
+The command returns a procedure ID that can be used to track the progress
+of the meta table refresh operation.
+EOF
+ end
+
+ def command
+ proc_id = admin.refresh_meta
+ formatter.row(["Refresh meta procedure submitted. Procedure ID: #{proc_id}"])
+ proc_id
+ end
+ end
+ end
+end
diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java
index 76a8b41481b..f29d6b5771d 100644
--- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java
+++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java
@@ -1374,6 +1374,11 @@ public class ThriftAdmin implements Admin {
throw new NotImplementedException("restoreBackupSystemTable not supported in ThriftAdmin");
}
+ @Override
+ public Long refreshMeta() throws IOException {
+ throw new NotImplementedException("refreshMeta not supported in ThriftAdmin");
+ }
+
@Override
public boolean replicationPeerModificationSwitch(boolean on, boolean drainProcedures)
throws IOException {
@@ -1386,4 +1391,19 @@ public class ThriftAdmin implements Admin {
throw new NotImplementedException(
"isReplicationPeerModificationEnabled not supported in ThriftAdmin");
}
+
+ @Override
+ public Long refreshHFiles(final TableName tableName) throws IOException {
+ throw new NotImplementedException("refreshHFiles not supported in ThriftAdmin");
+ }
+
+ @Override
+ public Long refreshHFiles(final String namespace) throws IOException {
+ throw new NotImplementedException("refreshHFiles not supported in ThriftAdmin");
+ }
+
+ @Override
+ public Long refreshHFiles() throws IOException {
+ throw new NotImplementedException("refreshHFiles not supported in ThriftAdmin");
+ }
}
diff --git a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java
index d34600bc5d3..87feb20b770 100644
--- a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java
+++ b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java
@@ -21,6 +21,7 @@ import com.google.errorprone.annotations.RestrictedApi;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.master.RegionState;
@@ -165,11 +166,12 @@ public final class MetaTableLocator {
public static void setMetaLocation(ZKWatcher zookeeper, ServerName serverName, int replicaId,
RegionState.State state) throws KeeperException {
if (serverName == null) {
- LOG.warn("Tried to set null ServerName in hbase:meta; skipping -- ServerName required");
+ LOG.warn("Tried to set null ServerName in {}; skipping -- ServerName required",
+ TableName.META_TABLE_NAME);
return;
}
- LOG.info("Setting hbase:meta replicaId={} location in ZooKeeper as {}, state={}", replicaId,
- serverName, state);
+ LOG.info("Setting {} replicaId={} location in ZooKeeper as {}, state={}",
+ TableName.META_TABLE_NAME, replicaId, serverName, state);
// Make the MetaRegionServer pb and then get its bytes and save this as
// the znode content.
MetaRegionServer pbrsr =
@@ -180,10 +182,10 @@ public final class MetaTableLocator {
ZKUtil.setData(zookeeper, zookeeper.getZNodePaths().getZNodeForReplica(replicaId), data);
} catch (KeeperException.NoNodeException nne) {
if (replicaId == RegionInfo.DEFAULT_REPLICA_ID) {
- LOG.debug("hbase:meta region location doesn't exist, create it");
+ LOG.debug("{} region location doesn't exist, create it", TableName.META_TABLE_NAME);
} else {
- LOG.debug(
- "hbase:meta region location doesn't exist for replicaId=" + replicaId + ", create it");
+ LOG.debug("{} region location doesn't exist for replicaId={}, create it",
+ TableName.META_TABLE_NAME, replicaId);
}
ZKUtil.createAndWatch(zookeeper, zookeeper.getZNodePaths().getZNodeForReplica(replicaId),
data);
@@ -233,9 +235,10 @@ public final class MetaTableLocator {
public static void deleteMetaLocation(ZKWatcher zookeeper, int replicaId) throws KeeperException {
if (replicaId == RegionInfo.DEFAULT_REPLICA_ID) {
- LOG.info("Deleting hbase:meta region location in ZooKeeper");
+ LOG.info("Deleting {} region location in ZooKeeper", TableName.META_TABLE_NAME);
} else {
- LOG.info("Deleting hbase:meta for {} region location in ZooKeeper", replicaId);
+ LOG.info("Deleting {} for {} region location in ZooKeeper", TableName.META_TABLE_NAME,
+ replicaId);
}
try {
// Just delete the node. Don't need any watches.
diff --git a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKDump.java b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKDump.java
index cd2b0b1a014..ed164d714a2 100644
--- a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKDump.java
+++ b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKDump.java
@@ -32,6 +32,7 @@ import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
@@ -74,7 +75,7 @@ public final class ZKDump {
sb.append("\n ").append(child);
}
}
- sb.append("\nRegion server holding hbase:meta:");
+ sb.append("\nRegion server holding ").append(TableName.META_TABLE_NAME).append(":");
sb.append("\n ").append(MetaTableLocator.getMetaRegionLocation(zkWatcher));
int numMetaReplicas = zkWatcher.getMetaReplicaNodes().size();
for (int i = 1; i < numMetaReplicas; i++) {