You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ch...@apache.org on 2017/11/08 20:09:57 UTC
hbase git commit: HBASE-18624 Added support for clearing BlockCache
based on tablename
Repository: hbase
Updated Branches:
refs/heads/branch-2 0d4f33ca2 -> 77e7c5ff2
HBASE-18624 Added support for clearing BlockCache based on tablename
Signed-off-by: Chia-Ping Tsai <ch...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/77e7c5ff
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/77e7c5ff
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/77e7c5ff
Branch: refs/heads/branch-2
Commit: 77e7c5ff274dabdd197719d25b999241188db49b
Parents: 0d4f33c
Author: Zach York <zy...@amazon.com>
Authored: Wed Oct 11 16:31:42 2017 -0700
Committer: Chia-Ping Tsai <ch...@gmail.com>
Committed: Thu Nov 9 04:03:35 2017 +0800
----------------------------------------------------------------------
.../apache/hadoop/hbase/CacheEvictionStats.java | 54 ++++++++
.../hadoop/hbase/CacheEvictionStatsBuilder.java | 50 ++++++++
.../org/apache/hadoop/hbase/client/Admin.java | 13 ++
.../apache/hadoop/hbase/client/HBaseAdmin.java | 48 +++++++
.../hadoop/hbase/protobuf/ProtobufUtil.java | 1 +
.../hbase/shaded/protobuf/ProtobufUtil.java | 15 +++
.../hbase/shaded/protobuf/RequestConverter.java | 13 ++
.../shaded/protobuf/ResponseConverter.java | 11 ++
.../src/main/protobuf/Admin.proto | 11 ++
.../src/main/protobuf/HBase.proto | 6 +
.../hbase/regionserver/HRegionServer.java | 20 ++-
.../hbase/regionserver/RSRpcServices.java | 68 +++++++---
.../hadoop/hbase/master/MockRegionServer.java | 9 ++
.../regionserver/TestClearRegionBlockCache.java | 125 +++++++++++++++++++
.../regionserver/TestReplicator.java | 6 +
hbase-shell/src/main/ruby/hbase/admin.rb | 6 +
hbase-shell/src/main/ruby/shell.rb | 1 +
.../ruby/shell/commands/clear_block_cache.rb | 39 ++++++
18 files changed, 475 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/77e7c5ff/hbase-client/src/main/java/org/apache/hadoop/hbase/CacheEvictionStats.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/CacheEvictionStats.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/CacheEvictionStats.java
new file mode 100644
index 0000000..2f118c6
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/CacheEvictionStats.java
@@ -0,0 +1,54 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+@InterfaceAudience.Public
+public final class CacheEvictionStats {
+
+ private final long evictedBlocks;
+ private final long maxCacheSize;
+
+ CacheEvictionStats(CacheEvictionStatsBuilder builder) {
+ this.evictedBlocks = builder.evictedBlocks;
+ this.maxCacheSize = builder.maxCacheSize;
+ }
+
+ public long getEvictedBlocks() {
+ return evictedBlocks;
+ }
+
+ public long getMaxCacheSize() {
+ return maxCacheSize;
+ }
+
+ @InterfaceAudience.Private
+ public static CacheEvictionStatsBuilder builder() {
+ return new CacheEvictionStatsBuilder();
+ }
+
+ @Override
+ public String toString() {
+ return "CacheEvictionStats{" +
+ "evictedBlocks=" + evictedBlocks +
+ ", maxCacheSize=" + maxCacheSize +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/77e7c5ff/hbase-client/src/main/java/org/apache/hadoop/hbase/CacheEvictionStatsBuilder.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/CacheEvictionStatsBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/CacheEvictionStatsBuilder.java
new file mode 100644
index 0000000..60606ae
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/CacheEvictionStatsBuilder.java
@@ -0,0 +1,50 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+@InterfaceAudience.Private
+public final class CacheEvictionStatsBuilder {
+ long evictedBlocks = 0;
+ long maxCacheSize = 0;
+
+ CacheEvictionStatsBuilder() {
+ }
+
+ public CacheEvictionStatsBuilder withEvictedBlocks(long evictedBlocks) {
+ this.evictedBlocks = evictedBlocks;
+ return this;
+ }
+
+ public CacheEvictionStatsBuilder withMaxCacheSize(long maxCacheSize) {
+ this.maxCacheSize = maxCacheSize;
+ return this;
+ }
+
+ public CacheEvictionStatsBuilder append(CacheEvictionStats stats) {
+ this.evictedBlocks += stats.getEvictedBlocks();
+ this.maxCacheSize += stats.getMaxCacheSize();
+ return this;
+ }
+
+ public CacheEvictionStats build() {
+ return new CacheEvictionStats(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/77e7c5ff/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index b1e46c6..d22d9e1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -31,6 +31,7 @@ import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.CacheEvictionStats;
import org.apache.hadoop.hbase.ClusterStatus;
import org.apache.hadoop.hbase.ClusterStatus.Option;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -1048,6 +1049,18 @@ public interface Admin extends Abortable, Closeable {
boolean isBalancerEnabled() throws IOException;
/**
+ * Clear all the blocks corresponding to this table from BlockCache. For expert-admins.
+ * Calling this API will drop all the cached blocks specific to a table from BlockCache.
+ * This can significantly impact the query performance as the subsequent queries will
+ * have to retrieve the blocks from underlying filesystem.
+ *
+ * @param tableName table to clear block cache
+ * @return CacheEvictionStats related to the eviction
+ * @throws IOException if a remote or network exception occurs
+ */
+ CacheEvictionStats clearBlockCache(final TableName tableName) throws IOException;
+
+ /**
* Invoke region normalizer. Can NOT run for various reasons. Check logs.
*
* @return <code>true</code> if region normalizer ran, <code>false</code> otherwise.
http://git-wip-us.apache.org/repos/asf/hbase/blob/77e7c5ff/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 556e564..0c66faf 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -46,6 +46,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.CacheEvictionStats;
+import org.apache.hadoop.hbase.CacheEvictionStatsBuilder;
import org.apache.hadoop.hbase.ClusterStatus;
import org.apache.hadoop.hbase.ClusterStatus.Option;
import org.apache.hadoop.hbase.CompoundConfiguration;
@@ -111,6 +113,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest;
@@ -1475,6 +1478,51 @@ public class HBaseAdmin implements Admin {
});
}
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public CacheEvictionStats clearBlockCache(final TableName tableName) throws IOException {
+ checkTableExists(tableName);
+ CacheEvictionStatsBuilder cacheEvictionStats = CacheEvictionStats.builder();
+ List<Pair<RegionInfo, ServerName>> pairs =
+ MetaTableAccessor.getTableRegionsAndLocations(connection, tableName);
+ for (Pair<RegionInfo, ServerName> pair: pairs) {
+ if (pair.getFirst().isOffline() || pair.getSecond() == null) {
+ continue;
+ }
+ try {
+ cacheEvictionStats = cacheEvictionStats.append(
+ clearBlockCache(pair.getSecond(), pair.getFirst()));
+ } catch (NotServingRegionException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Failed to clear block cache for " + pair.getFirst() + " on " +
+ pair.getSecond() + ": " + StringUtils.stringifyException(e));
+ }
+ }
+ }
+ return cacheEvictionStats.build();
+ }
+
+ private CacheEvictionStats clearBlockCache(final ServerName sn, final RegionInfo hri)
+ throws IOException {
+ HBaseRpcController controller = rpcControllerFactory.newController();
+ AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
+ ClearRegionBlockCacheRequest request =
+ RequestConverter.buildClearRegionBlockCacheRequest(hri.getRegionName());
+ try {
+ return ProtobufUtil.toCacheEvictionStats(
+ admin.clearRegionBlockCache(controller, request).getStats());
+ } catch (ServiceException se) {
+ throw ProtobufUtil.getRemoteException(se);
+ }
+ }
+
+ /**
+ * Invoke region normalizer. Can NOT run for various reasons. Check logs.
+ *
+ * @return True if region normalizer ran, false otherwise.
+ */
@Override
public boolean normalize() throws IOException {
return executeCallable(new MasterCallable<Boolean>(getConnection(), getRpcControllerFactory()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/77e7c5ff/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index a45629f..d154177 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -30,6 +30,7 @@ import java.util.NavigableSet;
import java.util.function.Function;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CacheEvictionStats;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellBuilderType;
import org.apache.hadoop.hbase.CellScanner;
http://git-wip-us.apache.org/repos/asf/hbase/blob/77e7c5ff/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index 96b0b61..bc6e427 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -41,6 +41,7 @@ import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ByteBufferCell;
+import org.apache.hadoop.hbase.CacheEvictionStats;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellBuilderType;
import org.apache.hadoop.hbase.CellScanner;
@@ -3385,4 +3386,18 @@ public final class ProtobufUtil {
.filter(snap -> pattern != null ? pattern.matcher(snap.getName()).matches() : true)
.collect(Collectors.toList());
}
+
+ public static CacheEvictionStats toCacheEvictionStats(HBaseProtos.CacheEvictionStats cacheEvictionStats) {
+ return CacheEvictionStats.builder()
+ .withEvictedBlocks(cacheEvictionStats.getEvictedBlocks())
+ .withMaxCacheSize(cacheEvictionStats.getMaxCacheSize())
+ .build();
+ }
+
+ public static HBaseProtos.CacheEvictionStats toCacheEvictionStats(CacheEvictionStats cacheEvictionStats) {
+ return HBaseProtos.CacheEvictionStats.newBuilder()
+ .setEvictedBlocks(cacheEvictionStats.getEvictedBlocks())
+ .setMaxCacheSize(cacheEvictionStats.getMaxCacheSize())
+ .build();
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/77e7c5ff/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
index 2da9c07..4558deb 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.hbase.client.replication.ReplicationSerDeHelper;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.filter.ByteArrayComparable;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.Pair;
@@ -60,6 +61,7 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
@@ -1423,6 +1425,17 @@ public final class RequestConverter {
}
/**
+ * Creates a protocol buffer ClearRegionBlockCacheRequest
+ *
+ * @return a ClearRegionBlockCacheRequest
+ */
+ public static ClearRegionBlockCacheRequest buildClearRegionBlockCacheRequest(final byte[]
+ regionName) {
+ RegionSpecifier region = buildRegionSpecifier(RegionSpecifierType.REGION_NAME, regionName);
+ return ClearRegionBlockCacheRequest.newBuilder().addAllRegion(Lists.newArrayList(region)).build();
+ }
+
+ /**
* Creates a protocol buffer GetClusterStatusRequest
*
* @return A GetClusterStatusRequest
http://git-wip-us.apache.org/repos/asf/hbase/blob/77e7c5ff/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java
index 611eaae..1d938c2 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.SingleResponse;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
import org.apache.hadoop.util.StringUtils;
import org.apache.yetus.audience.InterfaceAudience;
@@ -435,4 +436,14 @@ public final class ResponseConverter {
return metricMap;
}
+
+ /**
+ * Creates a protocol buffer ClearRegionBlockCacheResponse
+ *
+ * @return a ClearRegionBlockCacheResponse
+ */
+ public static AdminProtos.ClearRegionBlockCacheResponse buildClearRegionBlockCacheResponse(final HBaseProtos.CacheEvictionStats
+ cacheEvictionStats) {
+ return AdminProtos.ClearRegionBlockCacheResponse.newBuilder().setStats(cacheEvictionStats).build();
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/77e7c5ff/hbase-protocol-shaded/src/main/protobuf/Admin.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/src/main/protobuf/Admin.proto b/hbase-protocol-shaded/src/main/protobuf/Admin.proto
index 10732fd..118c79b 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Admin.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Admin.proto
@@ -248,6 +248,14 @@ message ClearCompactionQueuesRequest {
message ClearCompactionQueuesResponse {
}
+message ClearRegionBlockCacheRequest {
+ repeated RegionSpecifier region = 1;
+}
+
+message ClearRegionBlockCacheResponse {
+ required CacheEvictionStats stats = 1;
+}
+
message ExecuteProceduresRequest {
repeated OpenRegionRequest open_region = 1;
repeated CloseRegionRequest close_region = 2;
@@ -310,6 +318,9 @@ service AdminService {
rpc ClearCompactionQueues(ClearCompactionQueuesRequest)
returns(ClearCompactionQueuesResponse);
+ rpc ClearRegionBlockCache(ClearRegionBlockCacheRequest)
+ returns(ClearRegionBlockCacheResponse);
+
/** Fetches the RegionServer's view of space quotas */
rpc GetSpaceQuotaSnapshots(GetSpaceQuotaSnapshotsRequest)
returns(GetSpaceQuotaSnapshotsResponse);
http://git-wip-us.apache.org/repos/asf/hbase/blob/77e7c5ff/hbase-protocol-shaded/src/main/protobuf/HBase.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/src/main/protobuf/HBase.proto b/hbase-protocol-shaded/src/main/protobuf/HBase.proto
index cc1ae8f..7dd7680 100644
--- a/hbase-protocol-shaded/src/main/protobuf/HBase.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/HBase.proto
@@ -241,3 +241,9 @@ message RegionServerInfo {
optional int32 infoPort = 1;
optional VersionInfo version_info = 2;
}
+
+message CacheEvictionStats {
+ optional int64 evicted_blocks = 1;
+ optional int64 bytes_evicted = 2;
+ optional int64 max_cache_size = 3;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/77e7c5ff/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index e8b0a9a..48189be 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.CacheEvictionStats;
import org.apache.hadoop.hbase.ChoreService;
import org.apache.hadoop.hbase.ClockOutOfSyncException;
import org.apache.hadoop.hbase.CoordinatedStateManager;
@@ -99,6 +100,7 @@ import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.executor.ExecutorType;
import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.http.InfoServer;
+import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.util.MemorySizeUtil;
@@ -3269,7 +3271,7 @@ public class HRegionServer extends HasThread implements
}
/**
- * Protected utility method for safely obtaining an HRegion handle.
+ * Protected Utility method for safely obtaining an HRegion handle.
*
* @param regionName
* Name of online {@link HRegion} to return
@@ -3613,6 +3615,22 @@ public class HRegionServer extends HasThread implements
configurationManager.notifyAllObservers(conf);
}
+ public CacheEvictionStats clearRegionBlockCache(Region region) {
+ BlockCache blockCache = this.getCacheConfig().getBlockCache();
+ long evictedBlocks = 0;
+
+ for(Store store : region.getStores()) {
+ for(StoreFile hFile : store.getStorefiles()) {
+ evictedBlocks += blockCache.evictBlocksByHfileName(hFile.getPath().getName());
+ }
+ }
+
+ return CacheEvictionStats.builder()
+ .withEvictedBlocks(evictedBlocks)
+ .withMaxCacheSize(blockCache.getMaxSize())
+ .build();
+ }
+
@Override
public double getCompactionPressure() {
double max = 0;
http://git-wip-us.apache.org/repos/asf/hbase/blob/77e7c5ff/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 7419252..e104936 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -49,6 +49,8 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ByteBufferCell;
+import org.apache.hadoop.hbase.CacheEvictionStats;
+import org.apache.hadoop.hbase.CacheEvictionStatsBuilder;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScannable;
import org.apache.hadoop.hbase.CellScanner;
@@ -130,6 +132,7 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hbase.shaded.com.google.common.cache.Cache;
import org.apache.hadoop.hbase.shaded.com.google.common.cache.CacheBuilder;
+import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString;
import org.apache.hadoop.hbase.shaded.com.google.protobuf.Message;
import org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController;
@@ -142,6 +145,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.ResponseConverter;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionRequest;
@@ -1361,19 +1366,24 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
@VisibleForTesting
public HRegion getRegion(
final RegionSpecifier regionSpecifier) throws IOException {
- ByteString value = regionSpecifier.getValue();
- RegionSpecifierType type = regionSpecifier.getType();
- switch (type) {
- case REGION_NAME:
- byte[] regionName = value.toByteArray();
- String encodedRegionName = RegionInfo.encodeRegionName(regionName);
- return regionServer.getRegionByEncodedName(regionName, encodedRegionName);
- case ENCODED_REGION_NAME:
- return regionServer.getRegionByEncodedName(value.toStringUtf8());
- default:
- throw new DoNotRetryIOException(
- "Unsupported region specifier type: " + type);
+ return regionServer.getRegion(regionSpecifier.getValue().toByteArray());
+ }
+
+ /**
+ * Find the List of HRegions based on a list of region specifiers
+ *
+ * @param regionSpecifiers the list of region specifiers
+ * @return the corresponding list of regions
+ * @throws IOException if any of the specifiers is not null,
+ * but failed to find the region
+ */
+ private List<HRegion> getRegions(
+ final List<RegionSpecifier> regionSpecifiers) throws IOException {
+ List<HRegion> regions = Lists.newArrayListWithCapacity(regionSpecifiers.size());
+ for (RegionSpecifier regionSpecifier: regionSpecifiers) {
+ regions.add(regionServer.getRegion(regionSpecifier.getValue().toByteArray()));
}
+ return regions;
}
@VisibleForTesting
@@ -3442,15 +3452,33 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
ExecuteProceduresRequest request) throws ServiceException {
ExecuteProceduresResponse.Builder builder = ExecuteProceduresResponse.newBuilder();
if (request.getOpenRegionCount() > 0) {
- for (OpenRegionRequest req: request.getOpenRegionList()) {
+ for (OpenRegionRequest req : request.getOpenRegionList()) {
builder.addOpenRegion(openRegion(controller, req));
}
- }
- if (request.getCloseRegionCount() > 0) {
- for (CloseRegionRequest req: request.getCloseRegionList()) {
- builder.addCloseRegion(closeRegion(controller, req));
- }
- }
- return builder.build();
+ }
+ if (request.getCloseRegionCount() > 0) {
+ for (CloseRegionRequest req : request.getCloseRegionList()) {
+ builder.addCloseRegion(closeRegion(controller, req));
+ }
+ }
+ return builder.build();
+ }
+
+ @Override
+ public ClearRegionBlockCacheResponse clearRegionBlockCache(RpcController controller,
+ ClearRegionBlockCacheRequest request)
+ throws ServiceException {
+ CacheEvictionStatsBuilder stats = CacheEvictionStats.builder();
+ try {
+ List<HRegion> regions = getRegions(request.getRegionList());
+ for (HRegion region : regions) {
+ stats = stats.append(this.regionServer.clearRegionBlockCache(region));
+ }
+ } catch (Exception e) {
+ throw new ServiceException(e);
+ }
+ return ClearRegionBlockCacheResponse.newBuilder()
+ .setStats(ProtobufUtil.toCacheEvictionStats(stats.build()))
+ .build();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/77e7c5ff/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
index 964bb26..25cfbd9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
@@ -74,6 +74,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionRequest;
@@ -591,6 +593,13 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices {
}
@Override
+ public ClearRegionBlockCacheResponse clearRegionBlockCache(RpcController controller,
+ ClearRegionBlockCacheRequest request)
+ throws ServiceException {
+ return null;
+ }
+
+ @Override
public HeapMemoryManager getHeapMemoryManager() {
return null;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/77e7c5ff/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClearRegionBlockCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClearRegionBlockCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClearRegionBlockCache.java
new file mode 100644
index 0000000..b9d38f1
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClearRegionBlockCache.java
@@ -0,0 +1,125 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.io.hfile.BlockCache;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.IOException;
+import java.util.ArrayList;
+
+import static org.junit.Assert.assertEquals;
+
+@Category(MediumTests.class)
+@RunWith(Parameterized.class)
+public class TestClearRegionBlockCache {
+ private static final TableName TABLE_NAME = TableName.valueOf("testClearRegionBlockCache");
+ private static final byte[] FAMILY = Bytes.toBytes("family");
+ private static final byte[][] SPLIT_KEY = new byte[][] { Bytes.toBytes("5") };
+ private static final int NUM_MASTERS = 1;
+ private static final int NUM_RS = 2;
+
+ private final HBaseTestingUtility HTU = new HBaseTestingUtility();
+
+ private Configuration CONF = HTU.getConfiguration();
+ private Table table;
+ private HRegionServer rs1, rs2;
+ private MiniHBaseCluster cluster;
+
+ @Parameterized.Parameter public String cacheType;
+
+ @Parameterized.Parameters(name = "{index}: {0}")
+ public static Object[] data() {
+ return new Object[] { "lru", "bucket" };
+ }
+
+ @Before
+ public void setup() throws Exception {
+ if (cacheType.equals("bucket")) {
+ CONF.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
+ CONF.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 30);
+ }
+
+ cluster = HTU.startMiniCluster(NUM_MASTERS, NUM_RS);
+ rs1 = cluster.getRegionServer(0);
+ rs2 = cluster.getRegionServer(1);
+
+ // Create table
+ table = HTU.createTable(TABLE_NAME, FAMILY, SPLIT_KEY);
+ }
+
+ @After
+ public void teardown() throws Exception {
+ HTU.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testClearBlockCache() throws Exception {
+ HTU.loadNumericRows(table, FAMILY, 1, 10);
+ HTU.flush(TABLE_NAME);
+
+ BlockCache blockCache1 = rs1.getCacheConfig().getBlockCache();
+ BlockCache blockCache2 = rs2.getCacheConfig().getBlockCache();
+
+ long initialBlockCount1 = blockCache1.getBlockCount();
+ long initialBlockCount2 = blockCache2.getBlockCount();
+
+ // scan will cause blocks to be added in BlockCache
+ scanAllRegionsForRS(rs1);
+ assertEquals(blockCache1.getBlockCount() - initialBlockCount1,
+ HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY));
+ clearRegionBlockCache(rs1);
+
+ scanAllRegionsForRS(rs2);
+ assertEquals(blockCache2.getBlockCount() - initialBlockCount2,
+ HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY));
+ clearRegionBlockCache(rs2);
+
+ assertEquals(initialBlockCount1, blockCache1.getBlockCount());
+ assertEquals(initialBlockCount2, blockCache2.getBlockCount());
+ }
+
+ private void scanAllRegionsForRS(HRegionServer rs) throws IOException {
+ for (Region region : rs.getRegions(TABLE_NAME)) {
+ RegionScanner scanner = region.getScanner(new Scan());
+ while (scanner.next(new ArrayList<Cell>()));
+ }
+ }
+
+ private void clearRegionBlockCache(HRegionServer rs) {
+ for (Region region : rs.getRegions(TABLE_NAME)) {
+ rs.clearRegionBlockCache(region);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/77e7c5ff/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicator.java
index 4b0e353..6149721 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicator.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicator.java
@@ -399,6 +399,12 @@ public class TestReplicator extends TestReplicationBase {
throws ServiceException {
return null;
}
+
+ @Override
+ public ClearRegionBlockCacheResponse clearRegionBlockCache(RpcController controller,
+ ClearRegionBlockCacheRequest request) throws ServiceException {
+ return delegate.clearRegionBlockCache(controller, request);
+ }
}
public class FailureInjectingReplicatorForTest extends ReplicatorForTest {
http://git-wip-us.apache.org/repos/asf/hbase/blob/77e7c5ff/hbase-shell/src/main/ruby/hbase/admin.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb
index 581ccb3..0ce38cc 100644
--- a/hbase-shell/src/main/ruby/hbase/admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -197,6 +197,12 @@ module Hbase
end
#----------------------------------------------------------------------------------------------
+ # Requests clear block cache for table
+ def clear_block_cache(table_name)
+ @admin.clearBlockCache(org.apache.hadoop.hbase.TableName.valueOf(table_name)).toString
+ end
+
+ #----------------------------------------------------------------------------------------------
# Requests region normalization for all configured tables in the cluster
# Returns true if normalizer ran successfully
def normalize
http://git-wip-us.apache.org/repos/asf/hbase/blob/77e7c5ff/hbase-shell/src/main/ruby/shell.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb
index 759898b..687af12 100644
--- a/hbase-shell/src/main/ruby/shell.rb
+++ b/hbase-shell/src/main/ruby/shell.rb
@@ -360,6 +360,7 @@ Shell.load_command_group(
clear_compaction_queues
list_deadservers
clear_deadservers
+ clear_block_cache
],
# TODO: remove older hlog_roll command
aliases: {
http://git-wip-us.apache.org/repos/asf/hbase/blob/77e7c5ff/hbase-shell/src/main/ruby/shell/commands/clear_block_cache.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell/commands/clear_block_cache.rb b/hbase-shell/src/main/ruby/shell/commands/clear_block_cache.rb
new file mode 100644
index 0000000..ef46cc5
--- /dev/null
+++ b/hbase-shell/src/main/ruby/shell/commands/clear_block_cache.rb
@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+ module Commands
+ class ClearBlockCache < Command
+ def help
+ <<-EOF
+Clear all the blocks corresponding to this table from BlockCache. For expert-admins.
+Calling this API will drop all the cached blocks specific to a table from BlockCache.
+This can significantly impact the query performance as the subsequent queries will
+have to retrieve the blocks from underlying filesystem.
+For example:
+
+ hbase> clear_block_cache 'TABLENAME'
+EOF
+ end
+
+ def command(table_name)
+ formatter.row([admin.clear_block_cache(table_name)])
+ end
+ end
+ end
+end