You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2020/01/11 17:20:41 UTC
[hbase] branch master updated: Revert "HBASE-23055 Alter hbase:meta"
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new c8496bc Revert "HBASE-23055 Alter hbase:meta"
c8496bc is described below
commit c8496bc694052db41854270e0fdc263e6f50f878
Author: stack <st...@apache.org>
AuthorDate: Sat Jan 11 09:17:17 2020 -0800
Revert "HBASE-23055 Alter hbase:meta"
This reverts commit 9abdb7b5ae4de0b6d839f5c5d85e9bb899b5edbd.
---
.../org/apache/hadoop/hbase/MetaTableAccessor.java | 28 ++---
.../apache/hadoop/hbase/client/AsyncRegistry.java | 15 ---
.../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 58 ++++-------
.../hadoop/hbase/client/ZKAsyncRegistry.java | 62 +----------
.../hadoop/hbase/shaded/protobuf/ProtobufUtil.java | 43 --------
.../apache/hadoop/hbase/zookeeper/ZNodePaths.java | 59 +++++------
.../java/org/apache/hadoop/hbase/HConstants.java | 7 ++
.../apache/hadoop/hbase/util/CommonFSUtils.java | 2 -
.../org/apache/hadoop/hbase/TableDescriptors.java | 20 +++-
.../org/apache/hadoop/hbase/master/HMaster.java | 2 +-
.../hbase/master/MirroringTableStateManager.java | 5 +-
.../hadoop/hbase/master/TableStateManager.java | 115 +++++++++++++++------
.../hbase/master/assignment/RegionStateStore.java | 3 +-
.../master/procedure/CreateTableProcedure.java | 7 +-
.../master/procedure/DisableTableProcedure.java | 10 +-
.../master/procedure/EnableTableProcedure.java | 109 +++++++++----------
.../master/replication/AbstractPeerProcedure.java | 4 +-
.../master/replication/ModifyPeerProcedure.java | 4 +-
.../hbase/master/zksyncer/MetaLocationSyncer.java | 6 +-
.../RejectReplicationRequestStateChecker.java | 5 +-
.../hadoop/hbase/snapshot/SnapshotManifest.java | 5 +-
.../hadoop/hbase/util/FSTableDescriptors.java | 114 ++++++++++++--------
.../apache/hadoop/hbase/util/ZKDataMigrator.java | 34 +++++-
.../apache/hadoop/hbase/HBaseTestingUtility.java | 6 +-
.../org/apache/hadoop/hbase/TestHBaseMetaEdit.java | 110 --------------------
.../org/apache/hadoop/hbase/client/TestAdmin2.java | 17 +++
.../client/TestAsyncAccessControlAdminApi.java | 2 +-
.../hbase/client/TestAsyncTableAdminApi2.java | 13 +++
.../hbase/client/TestAsyncTableAdminApi3.java | 12 +++
.../hadoop/hbase/client/TestZKAsyncRegistry.java | 28 -----
.../hadoop/hbase/util/TestFSTableDescriptors.java | 14 +--
.../src/main/ruby/shell/commands/describe.rb | 13 +--
.../hadoop/hbase/zookeeper/MetaTableLocator.java | 18 +++-
.../org/apache/hadoop/hbase/zookeeper/ZKUtil.java | 2 +-
34 files changed, 419 insertions(+), 533 deletions(-)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index 71cbdb5..cba3f2d 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hbase;
-
import edu.umd.cs.findbugs.annotations.NonNull;
import edu.umd.cs.findbugs.annotations.Nullable;
import java.io.ByteArrayOutputStream;
@@ -81,7 +80,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
/**
@@ -306,18 +304,11 @@ public class MetaTableAccessor {
*/
public static HRegionLocation getRegionLocation(Connection connection, RegionInfo regionInfo)
throws IOException {
- return getRegionLocation(getCatalogFamilyRow(connection, regionInfo),
- regionInfo, regionInfo.getReplicaId());
- }
-
- /**
- * @return Return the {@link HConstants#CATALOG_FAMILY} row from hbase:meta.
- */
- public static Result getCatalogFamilyRow(Connection connection, RegionInfo ri)
- throws IOException {
- Get get = new Get(getMetaKeyForRegion(ri));
+ byte[] row = getMetaKeyForRegion(regionInfo);
+ Get get = new Get(row);
get.addFamily(HConstants.CATALOG_FAMILY);
- return get(getMetaHTable(connection), get);
+ Result r = get(getMetaHTable(connection), get);
+ return getRegionLocation(r, regionInfo, regionInfo.getReplicaId());
}
/** Returns the row key to use for this regionInfo */
@@ -949,8 +940,7 @@ public class MetaTableAccessor {
* @return A ServerName instance or null if necessary fields not found or empty.
*/
@Nullable
- // for use by HMaster#getTableRegionRow which is used for testing only
- @InterfaceAudience.Private
+ @InterfaceAudience.Private // for use by HMaster#getTableRegionRow which is used for testing only
public static ServerName getServerName(final Result r, final int replicaId) {
byte[] serverColumn = getServerColumn(replicaId);
Cell cell = r.getColumnLatestCell(getCatalogFamily(), serverColumn);
@@ -1089,8 +1079,9 @@ public class MetaTableAccessor {
@Nullable
public static TableState getTableState(Connection conn, TableName tableName)
throws IOException {
- Preconditions.checkArgument(!tableName.equals(TableName.META_TABLE_NAME),
- "Not for hbase:meta state");
+ if (tableName.equals(TableName.META_TABLE_NAME)) {
+ return new TableState(tableName, TableState.State.ENABLED);
+ }
Table metaHTable = getMetaHTable(conn);
Get get = new Get(tableName.getName()).addColumn(getTableFamily(), getTableStateColumn());
Result result = metaHTable.get(get);
@@ -1117,8 +1108,7 @@ public class MetaTableAccessor {
}
/**
- * Updates state in META.
- * Do not use. For internal use only.
+ * Updates state in META
* @param conn connection to use
* @param tableName table to look for
*/
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegistry.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegistry.java
index 7f4e818..9537777 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegistry.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegistry.java
@@ -21,7 +21,6 @@ import java.io.Closeable;
import java.util.concurrent.CompletableFuture;
import org.apache.hadoop.hbase.RegionLocations;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
import org.apache.yetus.audience.InterfaceAudience;
/**
@@ -30,13 +29,6 @@ import org.apache.yetus.audience.InterfaceAudience;
*/
@InterfaceAudience.Private
interface AsyncRegistry extends Closeable {
- /**
- * A completed CompletableFuture to host default hbase:meta table state (ENABLED).
- */
- TableState ENABLED_META_TABLE_STATE =
- new TableState(TableName.META_TABLE_NAME, TableState.State.ENABLED);
- CompletableFuture<TableState> COMPLETED_GET_META_TABLE_STATE =
- CompletableFuture.completedFuture(ENABLED_META_TABLE_STATE);
/**
* Get the location of meta region.
@@ -44,13 +36,6 @@ interface AsyncRegistry extends Closeable {
CompletableFuture<RegionLocations> getMetaRegionLocation();
/**
- * The hbase:meta table state.
- */
- default CompletableFuture<TableState> getMetaTableState() {
- return COMPLETED_GET_META_TABLE_STATE;
- }
-
- /**
* Should only be called once.
* <p>
* The upper layer should store this value somewhere as it will not be change any more.
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
index 0bb257c..4a349d5 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
@@ -90,7 +90,6 @@ import org.apache.hadoop.hbase.security.access.GetUserPermissionsRequest;
import org.apache.hadoop.hbase.security.access.Permission;
import org.apache.hadoop.hbase.security.access.ShadedAccessControlUtil;
import org.apache.hadoop.hbase.security.access.UserPermission;
-
import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
@@ -667,38 +666,22 @@ class RawAsyncHBaseAdmin implements AsyncAdmin {
new DisableTableProcedureBiConsumer(tableName));
}
- /**
- * Utility for completing passed TableState {@link CompletableFuture} <code>future</code>
- * using passed parameters.
- */
- private static CompletableFuture<Boolean> completeCheckTableState(
- CompletableFuture<Boolean> future, TableState tableState, Throwable error,
- TableState.State targetState, TableName tableName) {
- if (error != null) {
- future.completeExceptionally(error);
- } else {
- if (tableState != null) {
- future.complete(tableState.inStates(targetState));
- } else {
- future.completeExceptionally(new TableNotFoundException(tableName));
- }
- }
- return future;
- }
-
@Override
public CompletableFuture<Boolean> isTableEnabled(TableName tableName) {
if (TableName.isMetaTableName(tableName)) {
- CompletableFuture<Boolean> future = new CompletableFuture<>();
- addListener(this.connection.registry.getMetaTableState(), (tableState, error) -> {
- completeCheckTableState(future, tableState, error, TableState.State.ENABLED, tableName);
- });
- return future;
+ return CompletableFuture.completedFuture(true);
}
CompletableFuture<Boolean> future = new CompletableFuture<>();
- addListener(AsyncMetaTableAccessor.getTableState(metaTable, tableName), (tableState, error) -> {
- completeCheckTableState(future, tableState.isPresent()? tableState.get(): null, error,
- TableState.State.ENABLED, tableName);
+ addListener(AsyncMetaTableAccessor.getTableState(metaTable, tableName), (state, error) -> {
+ if (error != null) {
+ future.completeExceptionally(error);
+ return;
+ }
+ if (state.isPresent()) {
+ future.complete(state.get().inStates(TableState.State.ENABLED));
+ } else {
+ future.completeExceptionally(new TableNotFoundException(tableName));
+ }
});
return future;
}
@@ -706,16 +689,19 @@ class RawAsyncHBaseAdmin implements AsyncAdmin {
@Override
public CompletableFuture<Boolean> isTableDisabled(TableName tableName) {
if (TableName.isMetaTableName(tableName)) {
- CompletableFuture<Boolean> future = new CompletableFuture<>();
- addListener(this.connection.registry.getMetaTableState(), (tableState, error) -> {
- completeCheckTableState(future, tableState, error, TableState.State.DISABLED, tableName);
- });
- return future;
+ return CompletableFuture.completedFuture(false);
}
CompletableFuture<Boolean> future = new CompletableFuture<>();
- addListener(AsyncMetaTableAccessor.getTableState(metaTable, tableName), (tableState, error) -> {
- completeCheckTableState(future, tableState.isPresent()? tableState.get(): null, error,
- TableState.State.DISABLED, tableName);
+ addListener(AsyncMetaTableAccessor.getTableState(metaTable, tableName), (state, error) -> {
+ if (error != null) {
+ future.completeExceptionally(error);
+ return;
+ }
+ if (state.isPresent()) {
+ future.complete(state.get().inStates(TableState.State.DISABLED));
+ } else {
+ future.completeExceptionally(new TableNotFoundException(tableName));
+ }
});
return future;
}
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZKAsyncRegistry.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZKAsyncRegistry.java
index 92fa63c..08e3846 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZKAsyncRegistry.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZKAsyncRegistry.java
@@ -27,7 +27,6 @@ import static org.apache.hadoop.hbase.zookeeper.ZKMetadata.removeMetaData;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
import java.util.stream.Collectors;
import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.hadoop.conf.Configuration;
@@ -35,18 +34,15 @@ import org.apache.hadoop.hbase.ClusterId;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.RegionLocations;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient;
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos;
@@ -62,19 +58,8 @@ class ZKAsyncRegistry implements AsyncRegistry {
private final ZNodePaths znodePaths;
- /**
- * A znode maintained by MirroringTableStateManager.
- * MirroringTableStateManager is deprecated to be removed in hbase3. It can also be disabled.
- * Make sure it is enabled if you want to alter hbase:meta table in hbase2. In hbase3,
- * TBD how metatable state will be hosted; likely on active hbase master.
- */
- private final String znodeMirroredMetaTableState;
-
-
ZKAsyncRegistry(Configuration conf) {
this.znodePaths = new ZNodePaths(conf);
- this.znodeMirroredMetaTableState =
- ZNodePaths.joinZNode(this.znodePaths.tableZNode, TableName.META_TABLE_NAME.getNameAsString());
this.zk = new ReadOnlyZKClient(conf);
}
@@ -170,8 +155,7 @@ class ZKAsyncRegistry implements AsyncRegistry {
}
Pair<RegionState.State, ServerName> stateAndServerName = getStateAndServerName(proto);
if (stateAndServerName.getFirst() != RegionState.State.OPEN) {
- LOG.warn("hbase:meta region (replicaId={}) is in state {}", replicaId,
- stateAndServerName.getFirst());
+ LOG.warn("Meta region is in state " + stateAndServerName.getFirst());
}
locs[DEFAULT_REPLICA_ID] = new HRegionLocation(
getRegionInfoForDefaultReplica(FIRST_META_REGIONINFO), stateAndServerName.getSecond());
@@ -186,7 +170,7 @@ class ZKAsyncRegistry implements AsyncRegistry {
LOG.warn("Failed to fetch " + path, error);
locs[replicaId] = null;
} else if (proto == null) {
- LOG.warn("hbase:meta znode for replica " + replicaId + " is null");
+ LOG.warn("Meta znode for replica " + replicaId + " is null");
locs[replicaId] = null;
} else {
Pair<RegionState.State, ServerName> stateAndServerName = getStateAndServerName(proto);
@@ -210,8 +194,9 @@ class ZKAsyncRegistry implements AsyncRegistry {
public CompletableFuture<RegionLocations> getMetaRegionLocation() {
CompletableFuture<RegionLocations> future = new CompletableFuture<>();
addListener(
- zk.list(znodePaths.baseZNode).thenApply(children -> children.stream().
- filter(c -> znodePaths.isMetaZNodePrefix(c)).collect(Collectors.toList())),
+ zk.list(znodePaths.baseZNode)
+ .thenApply(children -> children.stream()
+ .filter(c -> c.startsWith(znodePaths.metaZNodePrefix)).collect(Collectors.toList())),
(metaReplicaZNodes, error) -> {
if (error != null) {
future.completeExceptionally(error);
@@ -245,43 +230,6 @@ class ZKAsyncRegistry implements AsyncRegistry {
}
@Override
- public CompletableFuture<TableState> getMetaTableState() {
- return getAndConvert(this.znodeMirroredMetaTableState, ZKAsyncRegistry::getTableState).
- thenApply(state -> {
- return state == null || state.equals(ENABLED_META_TABLE_STATE.getState())?
- ENABLED_META_TABLE_STATE: new TableState(TableName.META_TABLE_NAME, state);
- }).exceptionally(e -> {
- // Handle this case where no znode... Return default ENABLED in this case:
- // Caused by: java.io.IOException: java.util.concurrent.ExecutionException:
- // java.util.concurrent.ExecutionException:
- // org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for
- // /hbase/table/hbase:meta
- // If not case of above, then rethrow but may need to wrap. See
- // https://stackoverflow.com/questions/55453961/
- // completablefutureexceptionally-rethrow-checked-exception
- if (e.getCause() instanceof KeeperException.NoNodeException) {
- return ENABLED_META_TABLE_STATE;
- }
- throw e instanceof CompletionException? (CompletionException)e:
- new CompletionException(e);
- });
- }
-
- /**
- * Get tablestate from data byte array found in the mirroring znode of table state.
- */
- private static TableState.State getTableState(byte[] data) throws DeserializationException {
- if (data == null || data.length == 0) {
- return null;
- }
- try {
- return ProtobufUtil.toTableState(ProtobufUtil.toTableState(removeMetaData(data)));
- } catch (IOException ioe) {
- throw new DeserializationException(ioe);
- }
- }
-
- @Override
public void close() {
zk.close();
}
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index df6217f..5a71917 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -87,7 +87,6 @@ import org.apache.hadoop.hbase.client.SnapshotDescription;
import org.apache.hadoop.hbase.client.SnapshotType;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
-import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.client.security.SecurityCapability;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
@@ -3369,46 +3368,4 @@ public final class ProtobufUtil {
.build();
}
-
- /**
- * Parses pb TableState from <code>data</code>
- */
- public static ZooKeeperProtos.DeprecatedTableState.State toTableState(byte [] data)
- throws DeserializationException, IOException {
- if (data == null || data.length <= 0) {
- return null;
- }
- ProtobufUtil.expectPBMagicPrefix(data);
- ZooKeeperProtos.DeprecatedTableState.Builder builder =
- ZooKeeperProtos.DeprecatedTableState.newBuilder();
- int magicLen = ProtobufUtil.lengthOfPBMagic();
- ProtobufUtil.mergeFrom(builder, data, magicLen, data.length - magicLen);
- return builder.getState();
- }
-
-
- /**
- * @return Convert from pb TableState to pojo TableState.
- */
- public static TableState.State toTableState(ZooKeeperProtos.DeprecatedTableState.State state) {
- TableState.State newState = TableState.State.ENABLED;
- if (state != null) {
- switch (state) {
- case ENABLED:
- newState = TableState.State.ENABLED;
- break;
- case DISABLED:
- newState = TableState.State.DISABLED;
- break;
- case DISABLING:
- newState = TableState.State.DISABLING;
- break;
- case ENABLING:
- newState = TableState.State.ENABLING;
- break;
- default:
- }
- }
- return newState;
- }
}
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java
index 98d7eb7..c8511d4 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -24,7 +24,6 @@ import static org.apache.hadoop.hbase.HConstants.SPLIT_LOGDIR_NAME;
import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_ZNODE_PARENT;
import static org.apache.hadoop.hbase.client.RegionInfo.DEFAULT_REPLICA_ID;
-import java.util.Collection;
import java.util.Optional;
import java.util.stream.IntStream;
import org.apache.hadoop.conf.Configuration;
@@ -41,24 +40,15 @@ public class ZNodePaths {
// TODO: Replace this with ZooKeeper constant when ZOOKEEPER-277 is resolved.
public static final char ZNODE_PATH_SEPARATOR = '/';
- private static final String META_ZNODE_PREFIX = "meta-region-server";
+ public final static String META_ZNODE_PREFIX = "meta-region-server";
private static final String DEFAULT_SNAPSHOT_CLEANUP_ZNODE = "snapshot-cleanup";
// base znode for this cluster
public final String baseZNode;
-
- /**
- * The prefix of meta znode. Does not include baseZNode.
- * Its a 'prefix' because meta replica id integer can be tagged on the end (if
- * no number present, it is 'default' replica).
- */
- private final String metaZNodePrefix;
-
- /**
- * znodes containing the locations of the servers hosting the meta replicas
- */
- private final ImmutableMap<Integer, String> metaReplicaZNodes;
-
+ // the prefix of meta znode, does not include baseZNode.
+ public final String metaZNodePrefix;
+ // znodes containing the locations of the servers hosting the meta replicas
+ public final ImmutableMap<Integer, String> metaReplicaZNodes;
// znode containing ephemeral nodes of the regionservers
public final String rsZNode;
// znode containing ephemeral nodes of the draining regionservers
@@ -164,21 +154,21 @@ public class ZNodePaths {
}
/**
- * @return true if the znode is a meta region replica
+ * Is the znode of any meta replica
+ * @param node
+ * @return true or false
*/
public boolean isAnyMetaReplicaZNode(String node) {
- return this.metaReplicaZNodes.containsValue(node);
- }
-
- /**
- * @return Meta Replica ZNodes
- */
- public Collection<String> getMetaReplicaZNodes() {
- return this.metaReplicaZNodes.values();
+ if (metaReplicaZNodes.containsValue(node)) {
+ return true;
+ }
+ return false;
}
/**
- * @return the znode string corresponding to a replicaId
+ * Get the znode string corresponding to a replicaId
+ * @param replicaId
+ * @return znode
*/
public String getZNodeForReplica(int replicaId) {
// return a newly created path but don't update the cache of paths
@@ -189,21 +179,24 @@ public class ZNodePaths {
}
/**
- * Parse the meta replicaId from the passed znode name.
+ * Parse the meta replicaId from the passed znode
* @param znode the name of the znode, does not include baseZNode
* @return replicaId
*/
public int getMetaReplicaIdFromZnode(String znode) {
- return znode.equals(metaZNodePrefix)?
- RegionInfo.DEFAULT_REPLICA_ID:
- Integer.parseInt(znode.substring(metaZNodePrefix.length() + 1));
+ if (znode.equals(metaZNodePrefix)) {
+ return RegionInfo.DEFAULT_REPLICA_ID;
+ }
+ return Integer.parseInt(znode.substring(metaZNodePrefix.length() + 1));
}
/**
- * @return True if meta znode.
+ * Is it the default meta replica's znode
+ * @param znode the name of the znode, does not include baseZNode
+ * @return true or false
*/
- public boolean isMetaZNodePrefix(String znode) {
- return znode != null && znode.startsWith(this.metaZNodePrefix);
+ public boolean isDefaultMetaReplicaZnode(String znode) {
+ return metaReplicaZNodes.get(DEFAULT_REPLICA_ID).equals(znode);
}
/**
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index fa84b7d..132d3e0 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -26,6 +26,7 @@ import java.util.Collections;
import java.util.List;
import java.util.UUID;
import java.util.regex.Pattern;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.yetus.audience.InterfaceAudience;
@@ -1188,6 +1189,12 @@ public final class HConstants {
HBCK_SIDELINEDIR_NAME, HBASE_TEMP_DIRECTORY, MIGRATION_NAME
}));
+ /** Directories that are not HBase user table directories */
+ public static final List<String> HBASE_NON_USER_TABLE_DIRS =
+ Collections.unmodifiableList(Arrays.asList((String[])ArrayUtils.addAll(
+ new String[] { TableName.META_TABLE_NAME.getNameAsString() },
+ HBASE_NON_TABLE_DIRS.toArray())));
+
/** Health script related settings. */
public static final String HEALTH_SCRIPT_LOC = "hbase.node.health.script.location";
public static final String HEALTH_SCRIPT_TIMEOUT = "hbase.node.health.script.timeout";
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
index c5b6eff..89ad897 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
@@ -703,8 +703,6 @@ public abstract class CommonFSUtils {
if (LOG.isTraceEnabled()) {
LOG.trace("{} doesn't exist", dir);
}
- } catch (IllegalArgumentException iae) {
- int x = 0;
}
if (status == null || status.length < 1) {
return null;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
index e0a9eab..2537e7f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
@@ -25,19 +25,25 @@ import org.apache.hadoop.hbase.client.TableDescriptor;
/**
* Get, remove and modify table descriptors.
+ * Used by servers to host descriptors.
*/
@InterfaceAudience.Private
public interface TableDescriptors {
/**
+ * @param tableName
* @return TableDescriptor for tablename
+ * @throws IOException
*/
- TableDescriptor get(final TableName tableName) throws IOException;
+ TableDescriptor get(final TableName tableName)
+ throws IOException;
/**
* Get Map of all NamespaceDescriptors for a given namespace.
* @return Map of all descriptors.
+ * @throws IOException
*/
- Map<String, TableDescriptor> getByNamespace(String name) throws IOException;
+ Map<String, TableDescriptor> getByNamespace(String name)
+ throws IOException;
/**
* Get Map of all TableDescriptors. Populates the descriptor cache as a
@@ -45,19 +51,25 @@ public interface TableDescriptors {
* Notice: the key of map is the table name which contains namespace. It was generated by
* {@link TableName#getNameWithNamespaceInclAsString()}.
* @return Map of all descriptors.
+ * @throws IOException
*/
Map<String, TableDescriptor> getAll() throws IOException;
/**
* Add or update descriptor
* @param htd Descriptor to set into TableDescriptors
+ * @throws IOException
*/
- void add(final TableDescriptor htd) throws IOException;
+ void add(final TableDescriptor htd)
+ throws IOException;
/**
+ * @param tablename
* @return Instance of table descriptor or null if none found.
+ * @throws IOException
*/
- TableDescriptor remove(final TableName tablename) throws IOException;
+ TableDescriptor remove(final TableName tablename)
+ throws IOException;
/**
* Enables the tabledescriptor cache
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index decd3f6..b5f19d8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1046,7 +1046,7 @@ public class HMaster extends HRegionServer implements MasterServices {
RegionState rs = this.assignmentManager.getRegionStates().
getRegionState(RegionInfoBuilder.FIRST_META_REGIONINFO);
LOG.info("hbase:meta {}", rs);
- if (rs != null && rs.isOffline()) {
+ if (rs.isOffline()) {
Optional<InitMetaProcedure> optProc = procedureExecutor.getProcedures().stream()
.filter(p -> p instanceof InitMetaProcedure).map(o -> (InitMetaProcedure) o).findAny();
initMetaProc = optProc.orElseGet(() -> {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MirroringTableStateManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MirroringTableStateManager.java
index c1364f0..e486496 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MirroringTableStateManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MirroringTableStateManager.java
@@ -38,8 +38,7 @@ import org.slf4j.LoggerFactory;
* mirroring. See in HMaster where we make the choice. The below does zk updates on a best-effort
* basis only. If we fail updating zk we keep going because only hbase1 clients suffer; we'll just
* log at WARN level.
- * @deprecated Since 2.0.0. To be removed in 3.0.0. ZKRegistry#getMetaTableState reads
- * mirrored state so add alternative mechanism before purge else cannot disable hbase:meta table
+ * @deprecated Since 2.0.0. To be removed in 3.0.0.
*/
@Deprecated
@InterfaceAudience.Private
@@ -48,7 +47,7 @@ public class MirroringTableStateManager extends TableStateManager {
/**
* Set this key to true in Configuration to enable mirroring of table state out to zookeeper so
- * hbase-1.x clients can pick-up table state. Default value is 'true'.
+ * hbase-1.x clients can pick-up table state.
*/
static final String MIRROR_TABLE_STATE_TO_ZK_KEY = "hbase.mirror.table.state.to.zookeeper";
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java
index 7cd18fb..0ebd25b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableState;
+import org.apache.hadoop.hbase.exceptions.IllegalArgumentIOException;
import org.apache.hadoop.hbase.util.IdReadWriteLock;
import org.apache.hadoop.hbase.util.IdReadWriteLockWithObjectPool;
import org.apache.hadoop.hbase.util.ZKDataMigrator;
@@ -53,20 +54,8 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
// TODO: Make this a guava Service
@InterfaceAudience.Private
public class TableStateManager {
- private static final Logger LOG = LoggerFactory.getLogger(TableStateManager.class);
-
- /**
- * All table state is kept in hbase:meta except that of hbase:meta itself.
- * hbase:meta state is kept here locally in this in-memory variable. State
- * for hbase:meta is not persistent. If this process dies, the hbase:meta
- * state reverts to enabled. State is used so we can edit hbase:meta as we
- * would any other table by disabling, altering, and then re-enabling. If this
- * process dies in the midst of an edit, the table reverts to enabled. Schema
- * is read from the filesystem. It is changed atomically so if we die midway
- * through an edit we should be good.
- */
- private TableState.State metaTableState = TableState.State.ENABLED;
+ private static final Logger LOG = LoggerFactory.getLogger(TableStateManager.class);
/**
* Set this key to false in Configuration to disable migrating table state from zookeeper so
* hbase:meta table.
@@ -80,7 +69,7 @@ public class TableStateManager {
private final ConcurrentMap<TableName, TableState.State> tableName2State =
new ConcurrentHashMap<>();
- TableStateManager(MasterServices master) {
+ public TableStateManager(MasterServices master) {
this.master = master;
}
@@ -99,6 +88,61 @@ public class TableStateManager {
}
}
+ /**
+ * Set table state to provided but only if table in specified states Caller should lock table on
+ * write.
+ * @param tableName table to change state for
+ * @param newState new state
+ * @param states states to check against
+ * @return null if succeed or table state if failed
+ */
+ public TableState setTableStateIfInStates(TableName tableName, TableState.State newState,
+ TableState.State... states) throws IOException {
+ ReadWriteLock lock = tnLock.getLock(tableName);
+ lock.writeLock().lock();
+ try {
+ TableState currentState = readMetaState(tableName);
+ if (currentState == null) {
+ throw new TableNotFoundException(tableName);
+ }
+ if (currentState.inStates(states)) {
+ updateMetaState(tableName, newState);
+ return null;
+ } else {
+ return currentState;
+ }
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ /**
+ * Set table state to provided but only if table not in specified states Caller should lock table
+ * on write.
+ * @param tableName table to change state for
+ * @param newState new state
+ * @param states states to check against
+ */
+ public boolean setTableStateIfNotInStates(TableName tableName, TableState.State newState,
+ TableState.State... states) throws IOException {
+ ReadWriteLock lock = tnLock.getLock(tableName);
+ lock.writeLock().lock();
+ try {
+ TableState currentState = readMetaState(tableName);
+ if (currentState == null) {
+ throw new TableNotFoundException(tableName);
+ }
+ if (!currentState.inStates(states)) {
+ updateMetaState(tableName, newState);
+ return true;
+ } else {
+ return false;
+ }
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
public boolean isTableState(TableName tableName, TableState.State... states) {
try {
TableState tableState = getTableState(tableName);
@@ -112,7 +156,6 @@ public class TableStateManager {
public void setDeletedTable(TableName tableName) throws IOException {
if (tableName.equals(TableName.META_TABLE_NAME)) {
- // Can't delete the hbase:meta table.
return;
}
ReadWriteLock lock = tnLock.getLock(tableName);
@@ -141,7 +184,7 @@ public class TableStateManager {
* @param states filter by states
* @return tables in given states
*/
- Set<TableName> getTablesInStates(TableState.State... states) throws IOException {
+ public Set<TableName> getTablesInStates(TableState.State... states) throws IOException {
// Only be called in region normalizer, will not use cache.
final Set<TableName> rv = Sets.newHashSet();
MetaTableAccessor.fullScanTables(master.getConnection(), new MetaTableAccessor.Visitor() {
@@ -157,6 +200,12 @@ public class TableStateManager {
return rv;
}
+ public static class TableStateNotFoundException extends TableNotFoundException {
+ TableStateNotFoundException(TableName tableName) {
+ super(tableName.getNameAsString());
+ }
+ }
+
@NonNull
public TableState getTableState(TableName tableName) throws IOException {
ReadWriteLock lock = tnLock.getLock(tableName);
@@ -164,7 +213,7 @@ public class TableStateManager {
try {
TableState currentState = readMetaState(tableName);
if (currentState == null) {
- throw new TableNotFoundException("No state found for " + tableName);
+ throw new TableStateNotFoundException(tableName);
}
return currentState;
} finally {
@@ -173,18 +222,22 @@ public class TableStateManager {
}
private void updateMetaState(TableName tableName, TableState.State newState) throws IOException {
+ if (tableName.equals(TableName.META_TABLE_NAME)) {
+ if (TableState.State.DISABLING.equals(newState) ||
+ TableState.State.DISABLED.equals(newState)) {
+ throw new IllegalArgumentIOException("Cannot disable the meta table; " + newState);
+ }
+ // Otherwise, just return; no need to set ENABLED on meta -- it is always ENABLED.
+ return;
+ }
boolean succ = false;
try {
- if (tableName.equals(TableName.META_TABLE_NAME)) {
- this.metaTableState = newState;
- } else {
- MetaTableAccessor.updateTableState(master.getConnection(), tableName, newState);
- }
- this.tableName2State.put(tableName, newState);
+ MetaTableAccessor.updateTableState(master.getConnection(), tableName, newState);
+ tableName2State.put(tableName, newState);
succ = true;
} finally {
if (!succ) {
- this.tableName2State.remove(tableName);
+ tableName2State.remove(tableName);
}
}
metaStateUpdated(tableName, newState);
@@ -203,9 +256,7 @@ public class TableStateManager {
if (state != null) {
return new TableState(tableName, state);
}
- TableState tableState = tableName.equals(TableName.META_TABLE_NAME)?
- new TableState(TableName.META_TABLE_NAME, this.metaTableState):
- MetaTableAccessor.getTableState(master.getConnection(), tableName);
+ TableState tableState = MetaTableAccessor.getTableState(master.getConnection(), tableName);
if (tableState != null) {
tableName2State.putIfAbsent(tableName, tableState.getState());
}
@@ -213,8 +264,10 @@ public class TableStateManager {
}
public void start() throws IOException {
+ TableDescriptors tableDescriptors = master.getTableDescriptors();
migrateZooKeeper();
- fixTableStates(master.getTableDescriptors(), master.getConnection());
+ Connection connection = master.getConnection();
+ fixTableStates(tableDescriptors, connection);
}
private void fixTableStates(TableDescriptors tableDescriptors, Connection connection)
@@ -283,7 +336,7 @@ public class TableStateManager {
TableState ts = null;
try {
ts = getTableState(entry.getKey());
- } catch (TableNotFoundException e) {
+ } catch (TableStateNotFoundException e) {
// This can happen; table exists but no TableState.
}
if (ts == null) {
@@ -325,4 +378,4 @@ public class TableStateManager {
LOG.warn("Failed deleting table state from zookeeper", e);
}
}
-}
\ No newline at end of file
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java
index 7d79732..60c407d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java
@@ -146,7 +146,8 @@ public class RegionStateStore {
}
}
- void updateRegionLocation(RegionStateNode regionStateNode) throws IOException {
+ public void updateRegionLocation(RegionStateNode regionStateNode)
+ throws IOException {
if (regionStateNode.getRegionInfo().isMetaRegion()) {
updateMetaLocation(regionStateNode.getRegionInfo(), regionStateNode.getRegionLocation(),
regionStateNode.getState());
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
index a6c96a3..34fde27 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
@@ -78,7 +78,9 @@ public class CreateTableProcedure
@Override
protected Flow executeFromState(final MasterProcedureEnv env, final CreateTableState state)
throws InterruptedException {
- LOG.info("{} execute state={}", this, state);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(this + " execute state=" + state);
+ }
try {
switch (state) {
case CREATE_TABLE_PRE_OPERATION:
@@ -318,7 +320,8 @@ public class CreateTableProcedure
// using a copy of descriptor, table will be created enabling first
final Path tempTableDir = FSUtils.getTableDir(tempdir, tableDescriptor.getTableName());
((FSTableDescriptors)(env.getMasterServices().getTableDescriptors()))
- .createTableDescriptorForTableDirectory(tempTableDir, tableDescriptor, false);
+ .createTableDescriptorForTableDirectory(
+ tempTableDir, tableDescriptor, false);
// 2. Create Regions
newRegions = hdfsRegionHandler.createHdfsRegions(env, tempdir,
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java
index 29c9fc7..18c194f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.BufferedMutator;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.TableState;
+import org.apache.hadoop.hbase.constraint.ConstraintException;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.master.TableStateManager;
@@ -108,8 +109,8 @@ public class DisableTableProcedure
setNextState(DisableTableState.DISABLE_TABLE_ADD_REPLICATION_BARRIER);
break;
case DISABLE_TABLE_ADD_REPLICATION_BARRIER:
- if (env.getMasterServices().getTableDescriptors().get(tableName).
- hasGlobalReplicationScope()) {
+ if (env.getMasterServices().getTableDescriptors().get(tableName)
+ .hasGlobalReplicationScope()) {
MasterFileSystem fs = env.getMasterFileSystem();
try (BufferedMutator mutator = env.getMasterServices().getConnection()
.getBufferedMutator(TableName.META_TABLE_NAME)) {
@@ -241,7 +242,10 @@ public class DisableTableProcedure
*/
private boolean prepareDisable(final MasterProcedureEnv env) throws IOException {
boolean canTableBeDisabled = true;
- if (!MetaTableAccessor.tableExists(env.getMasterServices().getConnection(), tableName)) {
+ if (tableName.equals(TableName.META_TABLE_NAME)) {
+ setFailure("master-disable-table", new ConstraintException("Cannot disable catalog table"));
+ canTableBeDisabled = false;
+ } else if (!MetaTableAccessor.tableExists(env.getMasterServices().getConnection(), tableName)) {
setFailure("master-disable-table", new TableNotFoundException(tableName));
canTableBeDisabled = false;
} else if (!skipTableStateCheck) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java
index 614e4a2..06d6a2c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -27,9 +27,11 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
@@ -97,55 +99,66 @@ public class EnableTableProcedure
setNextState(EnableTableState.ENABLE_TABLE_MARK_REGIONS_ONLINE);
break;
case ENABLE_TABLE_MARK_REGIONS_ONLINE:
- // Get the region replica count. If changed since disable, need to do
- // more work assigning.
Connection connection = env.getMasterServices().getConnection();
- TableDescriptor tableDescriptor =
+ // we will need to get the tableDescriptor here to see if there is a change in the replica
+ // count
+ TableDescriptor hTableDescriptor =
env.getMasterServices().getTableDescriptors().get(tableName);
- int configuredReplicaCount = tableDescriptor.getRegionReplication();
- // Get regions for the table from memory; get both online and offline regions ('true').
+
+ // Get the replica count
+ int regionReplicaCount = hTableDescriptor.getRegionReplication();
+
+ // Get the regions for the table from memory; get both online and offline regions
+ // ('true').
List<RegionInfo> regionsOfTable =
env.getAssignmentManager().getRegionStates().getRegionsOfTable(tableName, true);
- // How many replicas do we currently have? Check regions returned from
- // in-memory state.
- int currentMaxReplica = getMaxReplicaId(regionsOfTable);
-
- // Read the META table to know the number of replicas the table currently has.
- // If there was a table modification on region replica count then need to
- // adjust replica counts here.
- int replicasFound = TableName.isMetaTableName(this.tableName)?
- 0: // TODO: Figure better what to do here for hbase:meta replica.
- getReplicaCountInMeta(connection, configuredReplicaCount, regionsOfTable);
- LOG.info("replicasFound={} (configuredReplicaCount={} for {}", replicasFound,
- configuredReplicaCount, tableName.getNameAsString());
- if (currentMaxReplica == (configuredReplicaCount - 1)) {
+ int currentMaxReplica = 0;
+ // Check if the regions in memory have replica regions as marked in META table
+ for (RegionInfo regionInfo : regionsOfTable) {
+ if (regionInfo.getReplicaId() > currentMaxReplica) {
+ // Iterating through all the list to identify the highest replicaID region.
+ // We can stop after checking with the first set of regions??
+ currentMaxReplica = regionInfo.getReplicaId();
+ }
+ }
+
+ // read the META table to know the actual number of replicas for the table - if there
+ // was a table modification on region replica then this will reflect the new entries also
+ int replicasFound =
+ getNumberOfReplicasFromMeta(connection, regionReplicaCount, regionsOfTable);
+ assert regionReplicaCount - 1 == replicasFound;
+ LOG.info(replicasFound + " META entries added for the given regionReplicaCount "
+ + regionReplicaCount + " for the table " + tableName.getNameAsString());
+ if (currentMaxReplica == (regionReplicaCount - 1)) {
if (LOG.isDebugEnabled()) {
- LOG.debug("No change in number of region replicas (configuredReplicaCount={});"
- + " assigning.", configuredReplicaCount);
+ LOG.debug("There is no change to the number of region replicas."
+ + " Assigning the available regions." + " Current and previous"
+ + "replica count is " + regionReplicaCount);
}
- } else if (currentMaxReplica > (configuredReplicaCount - 1)) {
- // We have additional regions as the replica count has been decreased. Delete
+ } else if (currentMaxReplica > (regionReplicaCount - 1)) {
+ // we have additional regions as the replica count has been decreased. Delete
// those regions because already the table is in the unassigned state
LOG.info("The number of replicas " + (currentMaxReplica + 1)
- + " is more than the region replica count " + configuredReplicaCount);
+ + " is more than the region replica count " + regionReplicaCount);
List<RegionInfo> copyOfRegions = new ArrayList<RegionInfo>(regionsOfTable);
for (RegionInfo regionInfo : copyOfRegions) {
- if (regionInfo.getReplicaId() > (configuredReplicaCount - 1)) {
+ if (regionInfo.getReplicaId() > (regionReplicaCount - 1)) {
// delete the region from the regionStates
env.getAssignmentManager().getRegionStates().deleteRegion(regionInfo);
// remove it from the list of regions of the table
- LOG.info("Removed replica={} of {}", regionInfo.getRegionId(), regionInfo);
+ LOG.info("The regioninfo being removed is " + regionInfo + " "
+ + regionInfo.getReplicaId());
regionsOfTable.remove(regionInfo);
}
}
} else {
// the replicasFound is less than the regionReplication
- LOG.info("Number of replicas has increased. Assigning new region replicas." +
- "The previous replica count was {}. The current replica count is {}.",
- (currentMaxReplica + 1), configuredReplicaCount);
- regionsOfTable = RegionReplicaUtil.addReplicas(tableDescriptor, regionsOfTable,
- currentMaxReplica + 1, configuredReplicaCount);
+ LOG.info("The number of replicas has been changed(increased)."
+ + " Lets assign the new region replicas. The previous replica count was "
+ + (currentMaxReplica + 1) + ". The current replica count is " + regionReplicaCount);
+ regionsOfTable = RegionReplicaUtil.addReplicas(hTableDescriptor, regionsOfTable,
+ currentMaxReplica + 1, regionReplicaCount);
}
// Assign all the table regions. (including region replicas if added).
// createAssignProcedure will try to retain old assignments if possible.
@@ -173,13 +186,9 @@ public class EnableTableProcedure
return Flow.HAS_MORE_STATE;
}
- /**
- * @return Count of replicas found reading hbase:meta Region row or zk if
- * asking about the hbase:meta table itself..
- */
- private int getReplicaCountInMeta(Connection connection, int regionReplicaCount,
+ private int getNumberOfReplicasFromMeta(Connection connection, int regionReplicaCount,
List<RegionInfo> regionsOfTable) throws IOException {
- Result r = MetaTableAccessor.getCatalogFamilyRow(connection, regionsOfTable.get(0));
+ Result r = getRegionFromMeta(connection, regionsOfTable);
int replicasFound = 0;
for (int i = 1; i < regionReplicaCount; i++) {
// Since we have already added the entries to the META we will be getting only that here
@@ -192,6 +201,16 @@ public class EnableTableProcedure
return replicasFound;
}
+ private Result getRegionFromMeta(Connection connection, List<RegionInfo> regionsOfTable)
+ throws IOException {
+ byte[] metaKeyForRegion = MetaTableAccessor.getMetaKeyForRegion(regionsOfTable.get(0));
+ Get get = new Get(metaKeyForRegion);
+ get.addFamily(HConstants.CATALOG_FAMILY);
+ Table metaTable = MetaTableAccessor.getMetaHTable(connection);
+ Result r = metaTable.get(get);
+ return r;
+ }
+
@Override
protected void rollbackState(final MasterProcedureEnv env, final EnableTableState state)
throws IOException {
@@ -389,20 +408,4 @@ public class EnableTableProcedure
}
}
}
-
- /**
- * @return Maximum region replica id found in passed list of regions.
- */
- private static int getMaxReplicaId(List<RegionInfo> regions) {
- int max = 0;
- for (RegionInfo regionInfo: regions) {
- if (regionInfo.getReplicaId() > max) {
- // Iterating through all the list to identify the highest replicaID region.
- // We can stop after checking with the first set of regions??
- max = regionInfo.getReplicaId();
- }
- }
- return max;
-
- }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
index 5ed7142..cdc0db2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
@@ -23,10 +23,10 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.master.TableStateManager;
+import org.apache.hadoop.hbase.master.TableStateManager.TableStateNotFoundException;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface;
import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
@@ -139,7 +139,7 @@ public abstract class AbstractPeerProcedure<TState> extends AbstractPeerNoLockPr
return true;
}
Thread.sleep(SLEEP_INTERVAL_MS);
- } catch (TableNotFoundException e) {
+ } catch (TableStateNotFoundException e) {
return false;
} catch (InterruptedException e) {
throw (IOException) new InterruptedIOException(e.getMessage()).initCause(e);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
index f443ed9..007a214 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
@@ -20,10 +20,10 @@ package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
import java.io.InterruptedIOException;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.master.TableStateManager;
+import org.apache.hadoop.hbase.master.TableStateManager.TableStateNotFoundException;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
import org.apache.hadoop.hbase.master.procedure.ReopenTableRegionsProcedure;
@@ -124,7 +124,7 @@ public abstract class ModifyPeerProcedure extends AbstractPeerProcedure<PeerModi
return false;
}
Thread.sleep(SLEEP_INTERVAL_MS);
- } catch (TableNotFoundException e) {
+ } catch (TableStateNotFoundException e) {
return false;
} catch (InterruptedException e) {
throw (IOException) new InterruptedIOException(e.getMessage()).initCause(e);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/zksyncer/MetaLocationSyncer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/zksyncer/MetaLocationSyncer.java
index 98d7322..eb80a2a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/zksyncer/MetaLocationSyncer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/zksyncer/MetaLocationSyncer.java
@@ -1,4 +1,4 @@
-/*
+/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -41,6 +41,6 @@ public class MetaLocationSyncer extends ClientZKSyncer {
@Override
Collection<String> getNodesToWatch() {
- return watcher.getZNodePaths().getMetaReplicaZNodes();
+ return watcher.getZNodePaths().metaReplicaZNodes.values();
}
-}
\ No newline at end of file
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RejectReplicationRequestStateChecker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RejectReplicationRequestStateChecker.java
index 9202133..9ad0af2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RejectReplicationRequestStateChecker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RejectReplicationRequestStateChecker.java
@@ -1,6 +1,4 @@
-/*
- * Copyright The Apache Software Foundation
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -24,7 +22,6 @@ import java.util.function.BiPredicate;
import org.apache.hadoop.hbase.replication.SyncReplicationState;
import org.apache.yetus.audience.InterfaceAudience;
-
/**
* Check whether we need to reject the replication request from source cluster.
*/
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
index db0224a..bf6699d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
@@ -472,10 +472,11 @@ public final class SnapshotManifest {
public void consolidate() throws IOException {
if (getSnapshotFormat(desc) == SnapshotManifestV1.DESCRIPTOR_VERSION) {
+ Path rootDir = FSUtils.getRootDir(conf);
LOG.info("Using old Snapshot Format");
// write a copy of descriptor to the snapshot directory
- FSTableDescriptors.createTableDescriptorForTableDirectory(workingDirFs, workingDir, htd,
- false);
+ new FSTableDescriptors(conf, workingDirFs, rootDir)
+ .createTableDescriptorForTableDirectory(workingDir, htd, false);
} else {
LOG.debug("Convert to Single Snapshot Manifest");
convertToV2SingleManifest();
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
index 784162c..8ae5b4b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
@@ -122,9 +122,8 @@ public class FSTableDescriptors implements TableDescriptors {
* @param fsreadonly True if we are read-only when it comes to filesystem
* operations; i.e. on remove, we do not do delete in fs.
*/
- @VisibleForTesting
public FSTableDescriptors(final Configuration conf, final FileSystem fs,
- final Path rootdir, final boolean fsreadonly, final boolean usecache) throws IOException {
+ final Path rootdir, final boolean fsreadonly, final boolean usecache) throws IOException {
this(conf, fs, rootdir, fsreadonly, usecache, null);
}
@@ -136,32 +135,16 @@ public class FSTableDescriptors implements TableDescriptors {
* TODO: This is a workaround. Should remove this ugly code...
*/
public FSTableDescriptors(final Configuration conf, final FileSystem fs,
- final Path rootdir, final boolean fsreadonly, final boolean usecache,
- Function<TableDescriptorBuilder, TableDescriptorBuilder> metaObserver) throws IOException {
+ final Path rootdir, final boolean fsreadonly, final boolean usecache,
+ Function<TableDescriptorBuilder, TableDescriptorBuilder> metaObserver) throws IOException {
this.fs = fs;
this.rootdir = rootdir;
this.fsreadonly = fsreadonly;
this.usecache = usecache;
- TableDescriptor td = null;
- try {
- td = getTableDescriptorFromFs(fs, rootdir, TableName.META_TABLE_NAME);
- } catch (TableInfoMissingException e) {
- td = metaObserver == null? createMetaTableDescriptor(conf):
- metaObserver.apply(createMetaTableDescriptorBuilder(conf)).build();
- if (!fsreadonly) {
- LOG.info("Creating new hbase:meta table default descriptor/schema {}", td);
- updateTableDescriptor(td);
- }
- }
- this.metaTableDescriptor = td;
+ this.metaTableDescriptor = metaObserver == null ? createMetaTableDescriptor(conf)
+ : metaObserver.apply(createMetaTableDescriptorBuilder(conf)).build();
}
- /**
- *
- * Should be private
- * @deprecated Since 2.3.0. Should be for internal use only. Used by testing.
- */
- @Deprecated
@VisibleForTesting
public static TableDescriptorBuilder createMetaTableDescriptorBuilder(final Configuration conf) throws IOException {
// TODO We used to set CacheDataInL1 for META table. When we have BucketCache in file mode, now
@@ -245,6 +228,16 @@ public class FSTableDescriptors implements TableDescriptors {
public TableDescriptor get(final TableName tablename)
throws IOException {
invocations++;
+ if (TableName.META_TABLE_NAME.equals(tablename)) {
+ cachehits++;
+ return metaTableDescriptor;
+ }
+ // hbase:meta is already handled. If some one tries to get the descriptor for
+ // .logs, .oldlogs or .corrupt throw an exception.
+ if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(tablename.getNameAsString())) {
+ throw new IOException("No descriptor found for non table = " + tablename);
+ }
+
if (usecache) {
// Look in cache of descriptors.
TableDescriptor cachedtdm = this.cache.get(tablename);
@@ -280,6 +273,7 @@ public class FSTableDescriptors implements TableDescriptors {
public Map<String, TableDescriptor> getAll()
throws IOException {
Map<String, TableDescriptor> tds = new TreeMap<>();
+
if (fsvisited && usecache) {
for (Map.Entry<TableName, TableDescriptor> entry: this.cache.entrySet()) {
tds.put(entry.getKey().getNameWithNamespaceInclAsString(), entry.getValue());
@@ -342,6 +336,15 @@ public class FSTableDescriptors implements TableDescriptors {
if (fsreadonly) {
throw new NotImplementedException("Cannot add a table descriptor - in read only mode");
}
+ TableName tableName = htd.getTableName();
+ if (TableName.META_TABLE_NAME.equals(tableName)) {
+ throw new NotImplementedException(HConstants.NOT_IMPLEMENTED);
+ }
+ if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(tableName.getNameAsString())) {
+ throw new NotImplementedException(
+ "Cannot add a table descriptor for a reserved subdirectory name: "
+ + htd.getTableName().getNameAsString());
+ }
updateTableDescriptor(htd);
}
@@ -366,6 +369,26 @@ public class FSTableDescriptors implements TableDescriptors {
return descriptor;
}
+ /**
+ * Checks if a current table info file exists for the given table
+ *
+ * @param tableName name of table
+ * @return true if exists
+ * @throws IOException
+ */
+ public boolean isTableInfoExists(TableName tableName) throws IOException {
+ return getTableInfoPath(tableName) != null;
+ }
+
+ /**
+ * Find the most current table info file for the given table in the hbase root directory.
+ * @return The file status of the current table info file or null if it does not exist
+ */
+ private FileStatus getTableInfoPath(final TableName tableName) throws IOException {
+ Path tableDir = getTableDir(tableName);
+ return getTableInfoPath(tableDir);
+ }
+
private FileStatus getTableInfoPath(Path tableDir)
throws IOException {
return getTableInfoPath(fs, tableDir, !fsreadonly);
@@ -380,6 +403,7 @@ public class FSTableDescriptors implements TableDescriptors {
* were sequence numbers).
*
* @return The file status of the current table info file or null if it does not exist
+ * @throws IOException
*/
public static FileStatus getTableInfoPath(FileSystem fs, Path tableDir)
throws IOException {
@@ -397,6 +421,7 @@ public class FSTableDescriptors implements TableDescriptors {
* older files.
*
* @return The file status of the current table info file or null if none exist
+ * @throws IOException
*/
private static FileStatus getTableInfoPath(FileSystem fs, Path tableDir, boolean removeOldFiles)
throws IOException {
@@ -585,6 +610,21 @@ public class FSTableDescriptors implements TableDescriptors {
}
/**
+ * Deletes all the table descriptor files from the file system.
+ * Used in unit tests only.
+ * @throws NotImplementedException if in read only mode
+ */
+ public void deleteTableDescriptorIfExists(TableName tableName) throws IOException {
+ if (fsreadonly) {
+ throw new NotImplementedException("Cannot delete a table descriptor - in read only mode");
+ }
+
+ Path tableDir = getTableDir(tableName);
+ Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR);
+ deleteTableDescriptorFiles(fs, tableInfoDir, Integer.MAX_VALUE);
+ }
+
+ /**
* Deletes files matching the table info file pattern within the given directory
* whose sequenceId is at most the given max sequenceId.
*/
@@ -706,8 +746,7 @@ public class FSTableDescriptors implements TableDescriptors {
/**
* Create a new TableDescriptor in HDFS in the specified table directory. Happens when we create
- * a new table during cluster start or in Clone and Create Table Procedures. Checks readOnly flag
- * passed on construction.
+ * a new table or snapshot a table.
* @param tableDir table directory under which we should write the file
* @param htd description of the table to write
* @param forceCreation if <tt>true</tt>,then even if previous table descriptor is present it will
@@ -716,28 +755,11 @@ public class FSTableDescriptors implements TableDescriptors {
* already exists and we weren't forcing the descriptor creation.
* @throws IOException if a filesystem error occurs
*/
- public boolean createTableDescriptorForTableDirectory(Path tableDir, TableDescriptor htd,
- boolean forceCreation) throws IOException {
- if (this.fsreadonly) {
+ public boolean createTableDescriptorForTableDirectory(Path tableDir,
+ TableDescriptor htd, boolean forceCreation) throws IOException {
+ if (fsreadonly) {
throw new NotImplementedException("Cannot create a table descriptor - in read only mode");
}
- return createTableDescriptorForTableDirectory(this.fs, tableDir, htd, forceCreation);
- }
-
- /**
- * Create a new TableDescriptor in HDFS in the specified table directory. Happens when we create
- * a new table snapshoting. Does not enforce read-only. That is for caller to determine.
- * @param fs Filesystem to use.
- * @param tableDir table directory under which we should write the file
- * @param htd description of the table to write
- * @param forceCreation if <tt>true</tt>,then even if previous table descriptor is present it will
- * be overwritten
- * @return <tt>true</tt> if the we successfully created the file, <tt>false</tt> if the file
- * already exists and we weren't forcing the descriptor creation.
- * @throws IOException if a filesystem error occurs
- */
- public static boolean createTableDescriptorForTableDirectory(FileSystem fs,
- Path tableDir, TableDescriptor htd, boolean forceCreation) throws IOException {
FileStatus status = getTableInfoPath(fs, tableDir);
if (status != null) {
LOG.debug("Current path=" + status.getPath());
@@ -750,7 +772,9 @@ public class FSTableDescriptors implements TableDescriptors {
}
}
}
- return writeTableDescriptor(fs, htd, tableDir, status) != null;
+ Path p = writeTableDescriptor(fs, htd, tableDir, status);
+ return p != null;
}
+
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ZKDataMigrator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ZKDataMigrator.java
index 338f3cd..78ef55c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ZKDataMigrator.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ZKDataMigrator.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -37,7 +37,6 @@ import org.slf4j.LoggerFactory;
/**
* Utlity method to migrate zookeeper data across HBase versions.
- * Used by Master mirroring table state to zk for hbase-1 clients.
* @deprecated Since 2.0.0. To be removed in hbase-3.0.0.
*/
@Deprecated
@@ -66,7 +65,25 @@ public class ZKDataMigrator {
return rv;
for (String child: children) {
TableName tableName = TableName.valueOf(child);
- TableState.State newState = ProtobufUtil.toTableState(getTableState(zkw, tableName));
+ ZooKeeperProtos.DeprecatedTableState.State state = getTableState(zkw, tableName);
+ TableState.State newState = TableState.State.ENABLED;
+ if (state != null) {
+ switch (state) {
+ case ENABLED:
+ newState = TableState.State.ENABLED;
+ break;
+ case DISABLED:
+ newState = TableState.State.DISABLED;
+ break;
+ case DISABLING:
+ newState = TableState.State.DISABLING;
+ break;
+ case ENABLING:
+ newState = TableState.State.ENABLING;
+ break;
+ default:
+ }
+ }
rv.put(tableName, newState);
}
return rv;
@@ -83,13 +100,20 @@ public class ZKDataMigrator {
* @deprecated Since 2.0.0. To be removed in hbase-3.0.0.
*/
@Deprecated
- private static ZooKeeperProtos.DeprecatedTableState.State getTableState(
+ private static ZooKeeperProtos.DeprecatedTableState.State getTableState(
final ZKWatcher zkw, final TableName tableName)
throws KeeperException, InterruptedException {
String znode = ZNodePaths.joinZNode(zkw.getZNodePaths().tableZNode,
tableName.getNameAsString());
+ byte [] data = ZKUtil.getData(zkw, znode);
+ if (data == null || data.length <= 0) return null;
try {
- return ProtobufUtil.toTableState(ZKUtil.getData(zkw, znode));
+ ProtobufUtil.expectPBMagicPrefix(data);
+ ZooKeeperProtos.DeprecatedTableState.Builder builder =
+ ZooKeeperProtos.DeprecatedTableState.newBuilder();
+ int magicLen = ProtobufUtil.lengthOfPBMagic();
+ ProtobufUtil.mergeFrom(builder, data, magicLen, data.length - magicLen);
+ return builder.getState();
} catch (IOException e) {
KeeperException ke = new KeeperException.DataInconsistencyException();
ke.initCause(e);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 3545183..f1e91de 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -151,7 +151,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.impl.Log4jLoggerAdapter;
-import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
@@ -494,11 +493,8 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
}
/**
- * @return META table descriptor builder
- * @Deprecated
+ * @return META table descriptor
*/
- @Deprecated
- @VisibleForTesting
public TableDescriptorBuilder getMetaTableDescriptorBuilder() {
try {
return FSTableDescriptors.createMetaTableDescriptorBuilder(conf);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseMetaEdit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseMetaEdit.java
deleted file mode 100644
index 7bdcd79..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseMetaEdit.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
-import org.apache.hadoop.hbase.client.RegionInfoBuilder;
-import org.apache.hadoop.hbase.client.TableDescriptor;
-import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
-import org.apache.hadoop.hbase.regionserver.Region;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.testclassification.MiscTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
-
-/**
- * Test being able to edit hbase:meta.
- */
-@Category({MiscTests.class, LargeTests.class})
-public class TestHBaseMetaEdit {
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestHBaseMetaEdit.class);
- @Rule
- public TestName name = new TestName();
- private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
-
- @Before
- public void before() throws Exception {
- UTIL.startMiniCluster();
- }
-
- @After
- public void after() throws Exception {
- UTIL.shutdownMiniCluster();
- }
-
- /**
- * Set versions, set HBASE-16213 indexed block encoding, and add a column family.
- * Verify they are all in place by looking at TableDescriptor AND by checking
- * what the RegionServer sees after opening Region.
- */
- @Test
- public void testEditMeta() throws IOException {
- Admin admin = UTIL.getAdmin();
- admin.tableExists(TableName.META_TABLE_NAME);
- admin.disableTable(TableName.META_TABLE_NAME);
- assertTrue(admin.isTableDisabled(TableName.META_TABLE_NAME));
- TableDescriptor descriptor = admin.getDescriptor(TableName.META_TABLE_NAME);
- ColumnFamilyDescriptor cfd = descriptor.getColumnFamily(HConstants.CATALOG_FAMILY);
- byte [] extraColumnFamilyName = Bytes.toBytes("xtra");
- ColumnFamilyDescriptor newCfd =
- ColumnFamilyDescriptorBuilder.newBuilder(extraColumnFamilyName).build();
- int oldVersions = cfd.getMaxVersions();
- // Add '1' to current versions count.
- cfd = ColumnFamilyDescriptorBuilder.newBuilder(cfd).setMaxVersions(oldVersions + 1).
- setConfiguration(ColumnFamilyDescriptorBuilder.DATA_BLOCK_ENCODING,
- DataBlockEncoding.ROW_INDEX_V1.toString()).build();
- admin.modifyColumnFamily(TableName.META_TABLE_NAME, cfd);
- admin.addColumnFamily(TableName.META_TABLE_NAME, newCfd);
- descriptor = admin.getDescriptor(TableName.META_TABLE_NAME);
- // Assert new max versions is == old versions plus 1.
- assertEquals(oldVersions + 1,
- descriptor.getColumnFamily(HConstants.CATALOG_FAMILY).getMaxVersions());
- admin.enableTable(TableName.META_TABLE_NAME);
- descriptor = admin.getDescriptor(TableName.META_TABLE_NAME);
- // Assert new max versions is == old versions plus 1.
- assertEquals(oldVersions + 1,
- descriptor.getColumnFamily(HConstants.CATALOG_FAMILY).getMaxVersions());
- assertTrue(descriptor.getColumnFamily(newCfd.getName()) != null);
- String encoding = descriptor.getColumnFamily(HConstants.CATALOG_FAMILY).getConfiguration().
- get(ColumnFamilyDescriptorBuilder.DATA_BLOCK_ENCODING);
- assertEquals(encoding, DataBlockEncoding.ROW_INDEX_V1.toString());
- Region r = UTIL.getHBaseCluster().getRegionServer(0).
- getRegion(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());
- assertEquals(oldVersions + 1,
- r.getStore(HConstants.CATALOG_FAMILY).getColumnFamilyDescriptor().getMaxVersions());
- encoding = r.getStore(HConstants.CATALOG_FAMILY).getColumnFamilyDescriptor().
- getConfigurationValue(ColumnFamilyDescriptorBuilder.DATA_BLOCK_ENCODING);
- assertEquals(encoding, DataBlockEncoding.ROW_INDEX_V1.toString());
- assertTrue(r.getStore(extraColumnFamilyName) != null);
- }
-}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java
index 2a0edc6..4fc6569 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.TableNotEnabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.UnknownRegionException;
import org.apache.hadoop.hbase.Waiter.Predicate;
+import org.apache.hadoop.hbase.constraint.ConstraintException;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -509,6 +510,22 @@ public class TestAdmin2 extends TestAdminBase {
}
@Test
+ public void testDisableCatalogTable() throws Exception {
+ try {
+ ADMIN.disableTable(TableName.META_TABLE_NAME);
+ fail("Expected to throw ConstraintException");
+ } catch (ConstraintException e) {
+ }
+ // Before the fix for HBASE-6146, the below table creation was failing as the hbase:meta table
+ // actually getting disabled by the disableTable() call.
+ HTableDescriptor htd =
+ new HTableDescriptor(TableName.valueOf(Bytes.toBytes(name.getMethodName())));
+ HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("cf1"));
+ htd.addFamily(hcd);
+ TEST_UTIL.getAdmin().createTable(htd);
+ }
+
+ @Test
public void testIsEnabledOrDisabledOnUnknownTable() throws Exception {
try {
ADMIN.isTableEnabled(TableName.valueOf(name.getMethodName()));
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAccessControlAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAccessControlAdminApi.java
index 33c8c10..9182e6f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAccessControlAdminApi.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAccessControlAdminApi.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi2.java
index cd20e10..4f5cbf2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi2.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi2.java
@@ -39,6 +39,7 @@ import java.util.Set;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/**
* Class to test asynchronous table admin operations
@@ -54,6 +55,18 @@ public class TestAsyncTableAdminApi2 extends TestAsyncAdminBase {
HBaseClassTestRule.forClass(TestAsyncTableAdminApi2.class);
@Test
+ public void testDisableCatalogTable() throws Exception {
+ try {
+ this.admin.disableTable(TableName.META_TABLE_NAME).join();
+ fail("Expected to throw ConstraintException");
+ } catch (Exception e) {
+ }
+ // Before the fix for HBASE-6146, the below table creation was failing as the hbase:meta table
+ // actually getting disabled by the disableTable() call.
+ createTableWithDefaultConf(tableName);
+ }
+
+ @Test
public void testAddColumnFamily() throws Exception {
// Create a table with two families
TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java
index 20e2bd6..344beac 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java
@@ -18,9 +18,12 @@
package org.apache.hadoop.hbase.client;
import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME;
+import static org.hamcrest.CoreMatchers.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.util.ArrayList;
import java.util.Collections;
@@ -28,6 +31,7 @@ import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.regex.Pattern;
import org.apache.hadoop.hbase.AsyncMetaTableAccessor;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.TableName;
@@ -196,6 +200,14 @@ public class TestAsyncTableAdminApi3 extends TestAsyncAdminBase {
ok = false;
}
assertTrue(ok);
+ // meta table can not be disabled.
+ try {
+ admin.disableTable(TableName.META_TABLE_NAME).get();
+ fail("meta table can not be disabled");
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
+ assertThat(cause, instanceOf(DoNotRetryIOException.class));
+ }
}
@Test
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKAsyncRegistry.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKAsyncRegistry.java
index a9b722a..3e4ca94 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKAsyncRegistry.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKAsyncRegistry.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hbase.client;
-import static junit.framework.TestCase.assertTrue;
import static org.apache.hadoop.hbase.HConstants.META_REPLICAS_NUM;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.junit.Assert.assertEquals;
@@ -129,31 +128,4 @@ public class TestZKAsyncRegistry {
}
}
}
-
- /**
- * Test meta tablestate implementation.
- * Test is a bit involved because meta has replicas... Replica assign lags so check
- * between steps all assigned.
- */
- @Test
- public void testMetaTableState() throws IOException, ExecutionException, InterruptedException {
- assertTrue(TEST_UTIL.getMiniHBaseCluster().getMaster().isActiveMaster());
- int ritTimeout = 10000;
- TEST_UTIL.waitUntilNoRegionsInTransition(ritTimeout);
- LOG.info("MASTER INITIALIZED");
- try (ZKAsyncRegistry registry = new ZKAsyncRegistry(TEST_UTIL.getConfiguration())) {
- assertEquals(TableState.State.ENABLED, registry.getMetaTableState().get().getState());
- LOG.info("META ENABLED");
- try (Admin admin = TEST_UTIL.getConnection().getAdmin()) {
- admin.disableTable(TableName.META_TABLE_NAME);
- assertEquals(TableState.State.DISABLED, registry.getMetaTableState().get().getState());
- TEST_UTIL.waitUntilNoRegionsInTransition(ritTimeout);
- LOG.info("META DISABLED");
- admin.enableTable(TableName.META_TABLE_NAME);
- assertEquals(TableState.State.ENABLED, registry.getMetaTableState().get().getState());
- TEST_UTIL.waitUntilNoRegionsInTransition(ritTimeout);
- LOG.info("META ENABLED");
- }
- }
- }
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java
index 2cc47bb..a0c37f2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java
@@ -322,13 +322,8 @@ public class TestFSTableDescriptors {
}
Map<String, TableDescriptor> tables = tds.getAll();
- // Remove hbase:meta from list. It shows up now since we made it dynamic. The schema
- // is written into the fs by the FSTableDescriptors constructor now where before it
- // didn't.
- tables.remove(TableName.META_TABLE_NAME.getNameAsString());
assertEquals(4, tables.size());
-
String[] tableNamesOrdered =
new String[] { "bar:foo", "default:bar", "default:foo", "foo:bar" };
int i = 0;
@@ -364,13 +359,12 @@ public class TestFSTableDescriptors {
assertTrue(nonchtds.getAll().size() == chtds.getAll().size());
- // add a new entry for random table name.
- TableName random = TableName.valueOf("random");
- TableDescriptor htd = TableDescriptorBuilder.newBuilder(random).build();
+ // add a new entry for hbase:meta
+ TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.META_TABLE_NAME).build();
nonchtds.createTableDescriptor(htd);
- // random will only increase the cachehit by 1
- assertEquals(nonchtds.getAll().size(), chtds.getAll().size() + 1);
+ // hbase:meta will only increase the cachehit by 1
+ assertTrue(nonchtds.getAll().size() == chtds.getAll().size());
for (Map.Entry<String, TableDescriptor> entry: nonchtds.getAll().entrySet()) {
String t = (String) entry.getKey();
diff --git a/hbase-shell/src/main/ruby/shell/commands/describe.rb b/hbase-shell/src/main/ruby/shell/commands/describe.rb
index 0553755..5ef02a0 100644
--- a/hbase-shell/src/main/ruby/shell/commands/describe.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/describe.rb
@@ -44,15 +44,12 @@ EOF
puts
end
formatter.footer
- if table.to_s != 'hbase:meta'
- # No QUOTAS if hbase:meta table
- puts
- formatter.header(%w[QUOTAS])
- count = quotas_admin.list_quotas(TABLE => table.to_s) do |_, quota|
- formatter.row([quota])
- end
- formatter.footer(count)
+ puts
+ formatter.header(%w[QUOTAS])
+ count = quotas_admin.list_quotas(TABLE => table.to_s) do |_, quota|
+ formatter.row([quota])
end
+ formatter.footer(count)
end
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
end
diff --git a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java
index 9105d7e..10fd165 100644
--- a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java
+++ b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -62,6 +62,14 @@ public final class MetaTableLocator {
}
/**
+ * Checks if the meta region location is available.
+ * @return true if meta region location is available, false if not
+ */
+ public static boolean isLocationAvailable(ZKWatcher zkw) {
+ return getMetaRegionLocation(zkw) != null;
+ }
+
+ /**
* @param zkw ZooKeeper watcher to be used
* @return meta table regions and their locations.
*/
@@ -258,7 +266,7 @@ public final class MetaTableLocator {
}
/**
- * Load the meta region state from the meta region server ZNode.
+ * Load the meta region state from the meta server ZNode.
*
* @param zkw reference to the {@link ZKWatcher} which also contains configuration and operation
* @param replicaId the ID of the replica
@@ -298,8 +306,10 @@ public final class MetaTableLocator {
if (serverName == null) {
state = RegionState.State.OFFLINE;
}
- return new RegionState(RegionReplicaUtil.getRegionInfoForReplica(
- RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId), state, serverName);
+ return new RegionState(
+ RegionReplicaUtil.getRegionInfoForReplica(
+ RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId),
+ state, serverName);
}
/**
diff --git a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index 5c9fcab..5f2185e 100644
--- a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -2056,7 +2056,7 @@ public final class ZKUtil {
" byte(s) of data from znode " + znode +
(watcherSet? " and set watcher; ": "; data=") +
(data == null? "null": data.length == 0? "empty": (
- zkw.getZNodePaths().isMetaZNodePrefix(znode)?
+ znode.startsWith(zkw.getZNodePaths().metaZNodePrefix)?
getServerNameOrEmptyString(data):
znode.startsWith(zkw.getZNodePaths().backupMasterAddressesZNode)?
getServerNameOrEmptyString(data):