You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nd...@apache.org on 2020/08/18 16:46:47 UTC
[hbase] branch master updated: HBASE-24627 Normalize one table at a
time
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new ecc0c67 HBASE-24627 Normalize one table at a time
ecc0c67 is described below
commit ecc0c6707784da442d22b92569308240c6cc1723
Author: Nick Dimiduk <nd...@apache.org>
AuthorDate: Thu Aug 6 19:07:46 2020 -0700
HBASE-24627 Normalize one table at a time
Introduce an additional method to our Admin interface that allow an
operator to selectivly run the normalizer. The IPC protocol supports
general table name select via compound filter.
Signed-off-by: Sean Busbey <bu...@apache.org>
Signed-off-by: Viraj Jasani <vj...@apache.org>
---
.../java/org/apache/hadoop/hbase/client/Admin.java | 18 ++-
.../hadoop/hbase/client/AdminOverAsyncAdmin.java | 4 +-
.../org/apache/hadoop/hbase/client/AsyncAdmin.java | 14 +-
.../hadoop/hbase/client/AsyncHBaseAdmin.java | 6 +-
.../hbase/client/NormalizeTableFilterParams.java | 107 ++++++++++++++
.../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 18 ++-
.../hadoop/hbase/shaded/protobuf/ProtobufUtil.java | 7 +
.../hbase/shaded/protobuf/RequestConverter.java | 17 ++-
.../src/main/protobuf/server/master/Master.proto | 3 +
.../org/apache/hadoop/hbase/master/HMaster.java | 36 +++--
.../hadoop/hbase/master/MasterRpcServices.java | 10 +-
.../TestSimpleRegionNormalizerOnCluster.java | 160 ++++++++++++++++-----
.../hbase/rsgroup/VerifyingRSGroupAdmin.java | 8 +-
hbase-shell/src/main/ruby/hbase/admin.rb | 51 ++++++-
hbase-shell/src/main/ruby/hbase_constants.rb | 3 +
.../src/main/ruby/shell/commands/normalize.rb | 20 ++-
.../hadoop/hbase/thrift2/client/ThriftAdmin.java | 6 +-
17 files changed, 407 insertions(+), 81 deletions(-)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index 155f024..40db1c1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -856,10 +856,24 @@ public interface Admin extends Abortable, Closeable {
* the request was submitted successfully. We need to check logs for the details of which regions
* were split/merged.
*
- * @return <code>true</code> if region normalizer ran, <code>false</code> otherwise.
+ * @return {@code true} if region normalizer ran, {@code false} otherwise.
* @throws IOException if a remote or network exception occurs
*/
- boolean normalize() throws IOException;
+ default boolean normalize() throws IOException {
+ return normalize(new NormalizeTableFilterParams.Builder().build());
+ }
+
+ /**
+ * Invoke region normalizer. Can NOT run for various reasons. Check logs.
+ * This is a non-blocking invocation to region normalizer. If return value is true, it means
+ * the request was submitted successfully. We need to check logs for the details of which regions
+ * were split/merged.
+ *
+ * @param ntfp limit to tables matching the specified filter.
+ * @return {@code true} if region normalizer ran, {@code false} otherwise.
+ * @throws IOException if a remote or network exception occurs
+ */
+ boolean normalize(NormalizeTableFilterParams ntfp) throws IOException;
/**
* Query the current state of the region normalizer.
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java
index 1255753..1b7a24b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java
@@ -395,8 +395,8 @@ class AdminOverAsyncAdmin implements Admin {
}
@Override
- public boolean normalize() throws IOException {
- return get(admin.normalize());
+ public boolean normalize(NormalizeTableFilterParams ntfp) throws IOException {
+ return get(admin.normalize(ntfp));
}
@Override
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
index 336903d..8c877e9 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -1279,7 +1279,17 @@ public interface AsyncAdmin {
* @return true if region normalizer ran, false otherwise. The return value will be wrapped by a
* {@link CompletableFuture}
*/
- CompletableFuture<Boolean> normalize();
+ default CompletableFuture<Boolean> normalize() {
+ return normalize(new NormalizeTableFilterParams.Builder().build());
+ }
+
+ /**
+ * Invoke region normalizer. Can NOT run for various reasons. Check logs.
+ * @param ntfp limit to tables matching the specified filter.
+ * @return true if region normalizer ran, false otherwise. The return value will be wrapped by a
+ * {@link CompletableFuture}
+ */
+ CompletableFuture<Boolean> normalize(NormalizeTableFilterParams ntfp);
/**
* Turn the cleaner chore on/off.
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
index c29fe71..2301d4a 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -709,8 +709,8 @@ class AsyncHBaseAdmin implements AsyncAdmin {
}
@Override
- public CompletableFuture<Boolean> normalize() {
- return wrap(rawAdmin.normalize());
+ public CompletableFuture<Boolean> normalize(NormalizeTableFilterParams ntfp) {
+ return wrap(rawAdmin.normalize(ntfp));
}
@Override
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NormalizeTableFilterParams.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NormalizeTableFilterParams.java
new file mode 100644
index 0000000..982ec5b
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NormalizeTableFilterParams.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import java.util.List;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * A collection of criteria used for table selection. The logic of table selection is as follows:
+ * <ul>
+ * <li>
+ * When no parameter values are provided, an unfiltered list of all user tables is returned.
+ * </li>
+ * <li>
+ * When a list of {@link TableName TableNames} are provided, the filter starts with any of
+ * these tables that exist.
+ * </li>
+ * <li>
+ * When a {@code namespace} name is provided, the filter starts with all the tables present in
+ * that namespace.
+ * </li>
+ * <li>
+ * If both a list of {@link TableName TableNames} and a {@code namespace} name are provided,
+ * the {@link TableName} list is honored and the {@code namespace} name is ignored.
+ * </li>
+ * <li>
+ * If a {@code regex} is provided, this subset of {@link TableName TableNames} is further
+ * reduced to those that match the provided regular expression.
+ * </li>
+ * </ul>
+ */
+@InterfaceAudience.Public
+public final class NormalizeTableFilterParams {
+ private final List<TableName> tableNames;
+ private final String regex;
+ private final String namespace;
+
+ private NormalizeTableFilterParams(final List<TableName> tableNames, final String regex,
+ final String namespace) {
+ this.tableNames = tableNames;
+ this.regex = regex;
+ this.namespace = namespace;
+ }
+
+ public List<TableName> getTableNames() {
+ return tableNames;
+ }
+
+ public String getRegex() {
+ return regex;
+ }
+
+ public String getNamespace() {
+ return namespace;
+ }
+
+ /**
+ * Used to instantiate an instance of {@link NormalizeTableFilterParams}.
+ */
+ public static class Builder {
+ private List<TableName> tableNames;
+ private String regex;
+ private String namespace;
+
+ public Builder tableFilterParams(final NormalizeTableFilterParams ntfp) {
+ this.tableNames = ntfp.getTableNames();
+ this.regex = ntfp.getRegex();
+ this.namespace = ntfp.getNamespace();
+ return this;
+ }
+
+ public Builder tableNames(final List<TableName> tableNames) {
+ this.tableNames = tableNames;
+ return this;
+ }
+
+ public Builder regex(final String regex) {
+ this.regex = regex;
+ return this;
+ }
+
+ public Builder namespace(final String namespace) {
+ this.namespace = namespace;
+ return this;
+ }
+
+ public NormalizeTableFilterParams build() {
+ return new NormalizeTableFilterParams(tableNames, regex, namespace);
+ }
+ }
+}
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
index 1330b2a..d740a3a 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
@@ -3286,14 +3286,18 @@ class RawAsyncHBaseAdmin implements AsyncAdmin {
}
@Override
- public CompletableFuture<Boolean> normalize() {
+ public CompletableFuture<Boolean> normalize(NormalizeTableFilterParams ntfp) {
+ return normalize(RequestConverter.buildNormalizeRequest(ntfp));
+ }
+
+ private CompletableFuture<Boolean> normalize(NormalizeRequest request) {
return this
- .<Boolean> newMasterCaller()
- .action(
- (controller, stub) -> this.<NormalizeRequest, NormalizeResponse, Boolean> call(
- controller, stub, RequestConverter.buildNormalizeRequest(),
- (s, c, req, done) -> s.normalize(c, req, done), (resp) -> resp.getNormalizerRan()))
- .call();
+ .<Boolean> newMasterCaller()
+ .action(
+ (controller, stub) -> this.call(
+ controller, stub, request, MasterService.Interface::normalize,
+ NormalizeResponse::getNormalizerRan))
+ .call();
}
@Override
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index 4a6adb1..ff20291 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -2293,6 +2293,13 @@ public final class ProtobufUtil {
.setQualifier(UnsafeByteOperations.unsafeWrap(tableName.getQualifier())).build();
}
+ public static List<HBaseProtos.TableName> toProtoTableNameList(List<TableName> tableNameList) {
+ if (tableNameList == null) {
+ return new ArrayList<>();
+ }
+ return tableNameList.stream().map(ProtobufUtil::toProtoTableName).collect(Collectors.toList());
+ }
+
public static List<TableName> toTableNameList(List<HBaseProtos.TableName> tableNamesList) {
if (tableNamesList == null) {
return new ArrayList<>();
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
index 7b0282a..1352b77 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.LogQueryFilter;
import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.NormalizeTableFilterParams;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionCoprocessorServiceExec;
import org.apache.hadoop.hbase.client.RegionInfo;
@@ -1476,8 +1477,18 @@ public final class RequestConverter {
*
* @return a NormalizeRequest
*/
- public static NormalizeRequest buildNormalizeRequest() {
- return NormalizeRequest.newBuilder().build();
+ public static NormalizeRequest buildNormalizeRequest(NormalizeTableFilterParams ntfp) {
+ final NormalizeRequest.Builder builder = NormalizeRequest.newBuilder();
+ if (ntfp.getTableNames() != null) {
+ builder.addAllTableNames(ProtobufUtil.toProtoTableNameList(ntfp.getTableNames()));
+ }
+ if (ntfp.getRegex() != null) {
+ builder.setRegex(ntfp.getRegex());
+ }
+ if (ntfp.getNamespace() != null) {
+ builder.setNamespace(ntfp.getNamespace());
+ }
+ return builder.build();
}
/**
diff --git a/hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto b/hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto
index 286c96f..b70ddef 100644
--- a/hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto
@@ -354,6 +354,9 @@ message IsSplitOrMergeEnabledResponse {
}
message NormalizeRequest {
+ repeated TableName table_names = 1;
+ optional string regex = 2;
+ optional string namespace = 3;
}
message NormalizeResponse {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index af99aab..1eb6525 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -21,7 +21,6 @@ import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_SPLIT_COORDINATED
import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS;
import static org.apache.hadoop.hbase.HConstants.HBASE_SPLIT_WAL_COORDINATED_BY_ZK;
import static org.apache.hadoop.hbase.util.DNS.MASTER_HOSTNAME_KEY;
-
import java.io.IOException;
import java.io.InterruptedIOException;
import java.lang.reflect.Constructor;
@@ -38,6 +37,7 @@ import java.util.Comparator;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -80,9 +80,9 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.UnknownRegionException;
-import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.MasterSwitchType;
+import org.apache.hadoop.hbase.client.NormalizeTableFilterParams;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.client.RegionStatesCount;
@@ -226,14 +226,13 @@ import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.webapp.WebAppContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
+import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors;
import org.apache.hbase.thirdparty.com.google.protobuf.Service;
import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils;
-
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
@@ -1902,14 +1901,18 @@ public class HMaster extends HRegionServer implements MasterServices {
return this.normalizer;
}
+ public boolean normalizeRegions() throws IOException {
+ return normalizeRegions(new NormalizeTableFilterParams.Builder().build());
+ }
+
/**
- * Perform normalization of cluster (invoked by {@link RegionNormalizerChore}).
+ * Perform normalization of cluster.
*
* @return true if an existing normalization was already in progress, or if a new normalization
* was performed successfully; false otherwise (specifically, if HMaster finished initializing
* or normalization is globally disabled).
*/
- public boolean normalizeRegions() throws IOException {
+ public boolean normalizeRegions(final NormalizeTableFilterParams ntfp) throws IOException {
final long startTime = EnvironmentEdgeManager.currentTime();
if (regionNormalizerTracker == null || !regionNormalizerTracker.isNormalizerOn()) {
LOG.debug("Region normalization is disabled, don't run region normalizer.");
@@ -1930,12 +1933,19 @@ public class HMaster extends HRegionServer implements MasterServices {
int affectedTables = 0;
try {
- final List<TableName> allEnabledTables =
- new ArrayList<>(tableStateManager.getTablesInStates(TableState.State.ENABLED));
- Collections.shuffle(allEnabledTables);
+ final Set<TableName> matchingTables = getTableDescriptors(new LinkedList<>(),
+ ntfp.getNamespace(), ntfp.getRegex(), ntfp.getTableNames(), false)
+ .stream()
+ .map(TableDescriptor::getTableName)
+ .collect(Collectors.toSet());
+ final Set<TableName> allEnabledTables =
+ tableStateManager.getTablesInStates(TableState.State.ENABLED);
+ final List<TableName> targetTables =
+ new ArrayList<>(Sets.intersection(matchingTables, allEnabledTables));
+ Collections.shuffle(targetTables);
final List<Long> submittedPlanProcIds = new ArrayList<>();
- for (TableName table : allEnabledTables) {
+ for (TableName table : targetTables) {
if (table.isSystemTable()) {
continue;
}
@@ -3399,9 +3409,9 @@ public class HMaster extends HRegionServer implements MasterServices {
}
/**
- * @return list of table table descriptors after filtering by regex and whether to include system
- * tables, etc.
- * @throws IOException
+ * Return a list of table table descriptors after applying any provided filter parameters. Note
+ * that the user-facing description of this filter logic is presented on the class-level javadoc
+ * of {@link NormalizeTableFilterParams}.
*/
private List<TableDescriptor> getTableDescriptors(final List<TableDescriptor> htds,
final String namespace, final String regex, final List<TableName> tableNameList,
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index c470acd..72040ae 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.UnknownRegionException;
import org.apache.hadoop.hbase.client.MasterSwitchType;
+import org.apache.hadoop.hbase.client.NormalizeTableFilterParams;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
@@ -1920,7 +1921,14 @@ public class MasterRpcServices extends RSRpcServices implements
NormalizeRequest request) throws ServiceException {
rpcPreCheck("normalize");
try {
- return NormalizeResponse.newBuilder().setNormalizerRan(master.normalizeRegions()).build();
+ final NormalizeTableFilterParams ntfp = new NormalizeTableFilterParams.Builder()
+ .tableNames(ProtobufUtil.toTableNameList(request.getTableNamesList()))
+ .regex(request.hasRegex() ? request.getRegex() : null)
+ .namespace(request.hasNamespace() ? request.getNamespace() : null)
+ .build();
+ return NormalizeResponse.newBuilder()
+ .setNormalizerRan(master.normalizeRegions(ntfp))
+ .build();
} catch (IOException ex) {
throw new ServiceException(ex);
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java
index da4c52e..ee9a160 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.TimeUnit;
@@ -35,7 +36,8 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.Size;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
-import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.AsyncAdmin;
+import org.apache.hadoop.hbase.client.NormalizeTableFilterParams;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.Table;
@@ -79,7 +81,7 @@ public class TestSimpleRegionNormalizerOnCluster {
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static final byte[] FAMILY_NAME = Bytes.toBytes("fam");
- private static Admin admin;
+ private static AsyncAdmin admin;
private static HMaster master;
@Rule
@@ -94,9 +96,12 @@ public class TestSimpleRegionNormalizerOnCluster {
// no way for the test to set the regionId on a created region, so disable this feature.
TEST_UTIL.getConfiguration().setInt("hbase.normalizer.merge.min_region_age.days", 0);
+ // disable the normalizer coming along and running via Chore
+ TEST_UTIL.getConfiguration().setInt("hbase.normalizer.period", Integer.MAX_VALUE);
+
TEST_UTIL.startMiniCluster(1);
TestNamespaceAuditor.waitForQuotaInitialize(TEST_UTIL);
- admin = TEST_UTIL.getAdmin();
+ admin = TEST_UTIL.getAsyncConnection().getAdmin();
master = TEST_UTIL.getHBaseCluster().getMaster();
assertNotNull(master);
}
@@ -107,17 +112,17 @@ public class TestSimpleRegionNormalizerOnCluster {
}
@Before
- public void before() throws IOException {
+ public void before() throws Exception {
// disable the normalizer ahead of time, let the test enable it when its ready.
- admin.normalizerSwitch(false);
+ admin.normalizerSwitch(false).get();
}
@Test
- public void testHonorsNormalizerSwitch() throws IOException {
- assertFalse(admin.isNormalizerEnabled());
- assertFalse(admin.normalize());
- assertFalse(admin.normalizerSwitch(true));
- assertTrue(admin.normalize());
+ public void testHonorsNormalizerSwitch() throws Exception {
+ assertFalse(admin.isNormalizerEnabled().get());
+ assertFalse(admin.normalize().get());
+ assertFalse(admin.normalizerSwitch(true).get());
+ assertTrue(admin.normalize().get());
}
/**
@@ -137,8 +142,8 @@ public class TestSimpleRegionNormalizerOnCluster {
final int tn2RegionCount = createTableBegsSplit(tn2, false, false);
final int tn3RegionCount = createTableBegsSplit(tn3, true, true);
- assertFalse(admin.normalizerSwitch(true));
- assertTrue(admin.normalize());
+ assertFalse(admin.normalizerSwitch(true).get());
+ assertTrue(admin.normalize().get());
waitForTableSplit(tn1, tn1RegionCount + 1);
// confirm that tn1 has (tn1RegionCount + 1) number of regions.
@@ -183,8 +188,8 @@ public class TestSimpleRegionNormalizerOnCluster {
final int currentRegionCount = createTableBegsSplit(tableName, true, false);
final long existingSkippedSplitCount = master.getRegionNormalizer()
.getSkippedCount(PlanType.SPLIT);
- assertFalse(admin.normalizerSwitch(true));
- assertTrue(admin.normalize());
+ assertFalse(admin.normalizerSwitch(true).get());
+ assertTrue(admin.normalize().get());
if (limitedByQuota) {
waitForSkippedSplits(master, existingSkippedSplitCount);
assertEquals(
@@ -208,8 +213,8 @@ public class TestSimpleRegionNormalizerOnCluster {
final TableName tableName = TableName.valueOf(name.getMethodName());
try {
final int currentRegionCount = createTableBegsMerge(tableName);
- assertFalse(admin.normalizerSwitch(true));
- assertTrue(admin.normalize());
+ assertFalse(admin.normalizerSwitch(true).get());
+ assertTrue(admin.normalize().get());
waitForTableMerge(tableName, currentRegionCount - 1);
assertEquals(
tableName + " should have merged.",
@@ -220,14 +225,103 @@ public class TestSimpleRegionNormalizerOnCluster {
}
}
- private static TableName buildTableNameForQuotaTest(final String methodName) throws IOException {
+ @Test
+ public void testHonorsNamespaceFilter() throws Exception {
+ final NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("ns").build();
+ final TableName tn1 = TableName.valueOf("ns", name.getMethodName());
+ final TableName tn2 = TableName.valueOf(name.getMethodName());
+
+ try {
+ admin.createNamespace(namespaceDescriptor).get();
+ final int tn1RegionCount = createTableBegsSplit(tn1, true, false);
+ final int tn2RegionCount = createTableBegsSplit(tn2, true, false);
+ final NormalizeTableFilterParams ntfp = new NormalizeTableFilterParams.Builder()
+ .namespace("ns")
+ .build();
+
+ assertFalse(admin.normalizerSwitch(true).get());
+ assertTrue(admin.normalize(ntfp).get());
+ waitForTableSplit(tn1, tn1RegionCount + 1);
+
+ // confirm that tn1 has (tn1RegionCount + 1) number of regions.
+ // tn2 has tn2RegionCount number of regions because it's not a member of the target namespace.
+ assertEquals(
+ tn1 + " should have split.",
+ tn1RegionCount + 1,
+ MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tn1));
+ waitForTableRegionCount(tn2, tn2RegionCount);
+ } finally {
+ dropIfExists(tn1);
+ dropIfExists(tn2);
+ }
+ }
+
+ @Test
+ public void testHonorsPatternFilter() throws Exception {
+ final TableName tn1 = TableName.valueOf(name.getMethodName() + "1");
+ final TableName tn2 = TableName.valueOf(name.getMethodName() + "2");
+
+ try {
+ final int tn1RegionCount = createTableBegsSplit(tn1, true, false);
+ final int tn2RegionCount = createTableBegsSplit(tn2, true, false);
+ final NormalizeTableFilterParams ntfp = new NormalizeTableFilterParams.Builder()
+ .regex(".*[1]")
+ .build();
+
+ assertFalse(admin.normalizerSwitch(true).get());
+ assertTrue(admin.normalize(ntfp).get());
+ waitForTableSplit(tn1, tn1RegionCount + 1);
+
+ // confirm that tn1 has (tn1RegionCount + 1) number of regions.
+ // tn2 has tn2RegionCount number of regions because it fails filter.
+ assertEquals(
+ tn1 + " should have split.",
+ tn1RegionCount + 1,
+ MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tn1));
+ waitForTableRegionCount(tn2, tn2RegionCount);
+ } finally {
+ dropIfExists(tn1);
+ dropIfExists(tn2);
+ }
+ }
+
+ @Test
+ public void testHonorsNameFilter() throws Exception {
+ final TableName tn1 = TableName.valueOf(name.getMethodName() + "1");
+ final TableName tn2 = TableName.valueOf(name.getMethodName() + "2");
+
+ try {
+ final int tn1RegionCount = createTableBegsSplit(tn1, true, false);
+ final int tn2RegionCount = createTableBegsSplit(tn2, true, false);
+ final NormalizeTableFilterParams ntfp = new NormalizeTableFilterParams.Builder()
+ .tableNames(Collections.singletonList(tn1))
+ .build();
+
+ assertFalse(admin.normalizerSwitch(true).get());
+ assertTrue(admin.normalize(ntfp).get());
+ waitForTableSplit(tn1, tn1RegionCount + 1);
+
+ // confirm that tn1 has (tn1RegionCount + 1) number of regions.
+ // tn2 has tn3RegionCount number of regions because it fails filter:
+ assertEquals(
+ tn1 + " should have split.",
+ tn1RegionCount + 1,
+ MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tn1));
+ waitForTableRegionCount(tn2, tn2RegionCount);
+ } finally {
+ dropIfExists(tn1);
+ dropIfExists(tn2);
+ }
+ }
+
+ private static TableName buildTableNameForQuotaTest(final String methodName) throws Exception {
String nsp = "np2";
NamespaceDescriptor nspDesc =
NamespaceDescriptor.create(nsp)
.addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
.addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
- admin.createNamespace(nspDesc);
- return TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + methodName);
+ admin.createNamespace(nspDesc).get();
+ return TableName.valueOf(nsp, methodName);
}
private static void waitForSkippedSplits(final HMaster master,
@@ -347,16 +441,17 @@ public class TestSimpleRegionNormalizerOnCluster {
*/
private static int createTableBegsSplit(final TableName tableName,
final boolean normalizerEnabled, final boolean isMergeEnabled)
- throws IOException {
+ throws Exception {
final List<HRegion> generatedRegions = generateTestData(tableName, 1, 1, 2, 3, 5);
assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName));
- admin.flush(tableName);
+ admin.flush(tableName).get();
- final TableDescriptor td = TableDescriptorBuilder.newBuilder(admin.getDescriptor(tableName))
+ final TableDescriptor td = TableDescriptorBuilder
+ .newBuilder(admin.getDescriptor(tableName).get())
.setNormalizationEnabled(normalizerEnabled)
.setMergeEnabled(isMergeEnabled)
.build();
- admin.modifyTable(td);
+ admin.modifyTable(td).get();
// make sure relatively accurate region statistics are available for the test table. use
// the last/largest region as clue.
@@ -383,16 +478,17 @@ public class TestSimpleRegionNormalizerOnCluster {
* <li>sum of sizes of first two regions < average</li>
* </ul>
*/
- private static int createTableBegsMerge(final TableName tableName) throws IOException {
+ private static int createTableBegsMerge(final TableName tableName) throws Exception {
// create 5 regions with sizes to trigger merge of small regions
final List<HRegion> generatedRegions = generateTestData(tableName, 1, 1, 3, 3, 5);
assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName));
- admin.flush(tableName);
+ admin.flush(tableName).get();
- final TableDescriptor td = TableDescriptorBuilder.newBuilder(admin.getDescriptor(tableName))
+ final TableDescriptor td = TableDescriptorBuilder
+ .newBuilder(admin.getDescriptor(tableName).get())
.setNormalizationEnabled(true)
.build();
- admin.modifyTable(td);
+ admin.modifyTable(td).get();
// make sure relatively accurate region statistics are available for the test table. use
// the last/largest region as clue.
@@ -411,12 +507,12 @@ public class TestSimpleRegionNormalizerOnCluster {
return 5;
}
- private static void dropIfExists(final TableName tableName) throws IOException {
- if (tableName != null && admin.tableExists(tableName)) {
- if (admin.isTableEnabled(tableName)) {
- admin.disableTable(tableName);
+ private static void dropIfExists(final TableName tableName) throws Exception {
+ if (tableName != null && admin.tableExists(tableName).get()) {
+ if (admin.isTableEnabled(tableName).get()) {
+ admin.disableTable(tableName).get();
}
- admin.deleteTable(tableName);
+ admin.deleteTable(tableName).get();
}
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java
index aad0d41..16aa12f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.client.CompactType;
import org.apache.hadoop.hbase.client.CompactionState;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.NormalizeTableFilterParams;
import org.apache.hadoop.hbase.client.OnlineLogRecord;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.Result;
@@ -341,8 +342,9 @@ public class VerifyingRSGroupAdmin implements Admin, Closeable {
return admin.clearBlockCache(tableName);
}
- public boolean normalize() throws IOException {
- return admin.normalize();
+ @Override
+ public boolean normalize(NormalizeTableFilterParams ntfp) throws IOException {
+ return admin.normalize(ntfp);
}
public boolean isNormalizerEnabled() throws IOException {
diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb
index f298a12..5392cdf 100644
--- a/hbase-shell/src/main/ruby/hbase/admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -258,9 +258,54 @@ module Hbase
#----------------------------------------------------------------------------------------------
# Requests region normalization for all configured tables in the cluster
- # Returns true if normalizer ran successfully
- def normalize
- @admin.normalize
+ # Returns true if normalize request was successfully submitted
+ def normalize(*args)
+ builder = org.apache.hadoop.hbase.client.NormalizeTableFilterParams::Builder.new
+ args.each do |arg|
+ unless arg.is_a?(String) || arg.is_a?(Hash)
+ raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash or String type")
+ end
+
+ if arg.key?(TABLE_NAME)
+ table_name = arg.delete(TABLE_NAME)
+ unless table_name.is_a?(String)
+ raise(ArgumentError, "#{TABLE_NAME} must be of type String")
+ end
+
+ builder.tableNames(java.util.Collections.singletonList(TableName.valueOf(table_name)))
+ elsif arg.key?(TABLE_NAMES)
+ table_names = arg.delete(TABLE_NAMES)
+ unless table_names.is_a?(Array)
+ raise(ArgumentError, "#{TABLE_NAMES} must be of type Array")
+ end
+
+ table_name_list = java.util.LinkedList.new
+ table_names.each do |tn|
+ unless tn.is_a?(String)
+ raise(ArgumentError, "#{TABLE_NAMES} value #{tn} must be of type String")
+ end
+
+ table_name_list.add(TableName.valueOf(tn))
+ end
+ builder.tableNames(table_name_list)
+ elsif arg.key?(REGEX)
+ regex = arg.delete(REGEX)
+ raise(ArgumentError, "#{REGEX} must be of type String") unless regex.is_a?(String)
+
+ builder.regex(regex)
+ elsif arg.key?(NAMESPACE)
+ namespace = arg.delete(NAMESPACE)
+ unless namespace.is_a?(String)
+ raise(ArgumentError, "#{NAMESPACE} must be of type String")
+ end
+
+ builder.namespace(namespace)
+ else
+ raise(ArgumentError, "Unrecognized argument #{arg}")
+ end
+ end
+ ntfp = builder.build
+ @admin.normalize(ntfp)
end
#----------------------------------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/hbase_constants.rb b/hbase-shell/src/main/ruby/hbase_constants.rb
index 6db91c7..b1b0eae 100644
--- a/hbase-shell/src/main/ruby/hbase_constants.rb
+++ b/hbase-shell/src/main/ruby/hbase_constants.rb
@@ -71,6 +71,7 @@ module HBaseConstants
POLICY = 'POLICY'.freeze
RAW = 'RAW'.freeze
READ_TYPE = 'READ_TYPE'.freeze
+ REGEX = 'REGEX'.freeze
REGIONSERVER = 'REGIONSERVER'.freeze
REGION_REPLICATION = 'REGION_REPLICATION'.freeze
REGION_REPLICA_ID = 'REGION_REPLICA_ID'.freeze
@@ -91,6 +92,8 @@ module HBaseConstants
STOPROW = 'STOPROW'.freeze
TABLE = 'TABLE'.freeze
TABLE_CFS = 'TABLE_CFS'.freeze
+ TABLE_NAME = 'TABLE_NAME'.freeze
+ TABLE_NAMES = 'TABLE_NAMES'.freeze
TIMERANGE = 'TIMERANGE'.freeze
TIMESTAMP = 'TIMESTAMP'.freeze
TYPE = 'TYPE'.freeze
diff --git a/hbase-shell/src/main/ruby/shell/commands/normalize.rb b/hbase-shell/src/main/ruby/shell/commands/normalize.rb
index 2840e84..70e524a 100644
--- a/hbase-shell/src/main/ruby/shell/commands/normalize.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/normalize.rb
@@ -22,18 +22,24 @@ module Shell
class Normalize < Command
def help
<<-EOF
-Trigger region normalizer for all tables which have NORMALIZATION_ENABLED flag set. Returns true
- if normalizer ran successfully, false otherwise. Note that this command has no effect
- if region normalizer is disabled (make sure it's turned on using 'normalizer_switch' command).
+Trigger the region normalizer. Without arguments, invokes the normalizer without a table filter.
+Any arguments are used to limit table selection. Returns true if the normalize request was
+submitted successfully, false otherwise. Note that this command has no effect if region normalizer
+is disabled (make sure it's turned on using 'normalizer_switch' command).
- Examples:
+Examples:
- hbase> normalize
+ hbase> normalize
+ hbase> normalize TABLE_NAME => 'my_table'
+ hbase> normalize TABLE_NAMES => ['foo', 'bar', 'baz']
+ hbase> normalize REGEX => 'my_.*'
+ hbase> normalize NAMESPACE => 'ns1'
+ hbase> normalize NAMESPACE => 'ns', REGEX => '*._BIG_.*'
EOF
end
- def command
- did_normalize_run = !!admin.normalize
+ def command(*args)
+ did_normalize_run = !!admin.normalize(*args)
formatter.row([did_normalize_run.to_s])
did_normalize_run
end
diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java
index 9758d08..633238b 100644
--- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java
+++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java
@@ -1,5 +1,4 @@
-/**
- *
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -44,6 +43,7 @@ import org.apache.hadoop.hbase.client.CompactType;
import org.apache.hadoop.hbase.client.CompactionState;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.LogQueryFilter;
+import org.apache.hadoop.hbase.client.NormalizeTableFilterParams;
import org.apache.hadoop.hbase.client.OnlineLogRecord;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.SnapshotDescription;
@@ -639,7 +639,7 @@ public class ThriftAdmin implements Admin {
}
@Override
- public boolean normalize() {
+ public boolean normalize(NormalizeTableFilterParams ntfp) {
throw new NotImplementedException("normalize not supported in ThriftAdmin");
}