You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by en...@apache.org on 2015/12/01 01:49:13 UTC
hbase git commit: HBASE-14223 Meta WALs are not cleared if meta
region was closed and RS aborts
Repository: hbase
Updated Branches:
refs/heads/master 999ae6915 -> 756e6de2a
HBASE-14223 Meta WALs are not cleared if meta region was closed and RS aborts
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/756e6de2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/756e6de2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/756e6de2
Branch: refs/heads/master
Commit: 756e6de2a0948c39a4fb3ab880dd9d182d645dbc
Parents: 999ae69
Author: Enis Soztutar <en...@apache.org>
Authored: Mon Nov 30 16:48:52 2015 -0800
Committer: Enis Soztutar <en...@apache.org>
Committed: Mon Nov 30 16:48:52 2015 -0800
----------------------------------------------------------------------
.../hbase/regionserver/HRegionServer.java | 20 ++++++++++++++++++++
.../hadoop/hbase/regionserver/LogRoller.java | 4 ++++
.../hbase/regionserver/RSRpcServices.java | 2 +-
.../regionserver/RegionServerServices.java | 6 ++++++
.../regionserver/handler/CloseMetaHandler.java | 8 ++++++++
.../handler/CloseRegionHandler.java | 15 ++++++++++++---
.../org/apache/hadoop/hbase/wal/WALFactory.java | 15 +++++++++++++++
.../hadoop/hbase/MockRegionServerServices.java | 4 ++++
.../hadoop/hbase/master/MockRegionServer.java | 10 +++++++---
9 files changed, 77 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/756e6de2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 2ce2193..5be240d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1867,6 +1867,26 @@ public class HRegionServer extends HasThread implements
}
@Override
+ public void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException {
+ if (regionInfo != null && regionInfo.isMetaTable() &&
+ regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
+
+ walFactory.closeMetaWAL(regionInfo.getEncodedNameAsBytes());
+
+ LogRoller roller;
+ if (regionInfo != null && regionInfo.isMetaTable() &&
+ regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
+ roller = metawalRoller.get();
+ if (roller != null) {
+ roller.removeWAL(wal); // only do this for meta WAL
+ }
+
+ // TODO: meta wal roller is left running. Should be fine.
+ }
+ }
+ }
+
+ @Override
public ClusterConnection getConnection() {
return this.clusterConnection;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/756e6de2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
index 40edc05..12cd060 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
@@ -78,6 +78,10 @@ public class LogRoller extends HasThread {
}
}
+ public void removeWAL(final WAL wal) {
+ walNeedsRoll.remove(wal);
+ }
+
public void requestRollAll() {
for (WAL wal : walNeedsRoll.keySet()) {
walNeedsRoll.put(wal, Boolean.TRUE);
http://git-wip-us.apache.org/repos/asf/hbase/blob/756e6de2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index d94e11c..6fadd0c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -1669,7 +1669,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
return response;
}
- HRegion.warmupHRegion(region, htd, regionServer.getWAL(region),
+ HRegion.warmupHRegion(region, htd, null,
regionServer.getConfiguration(), regionServer, null);
} catch (IOException ie) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/756e6de2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
index cd4816c..29e6a95 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
@@ -55,6 +55,12 @@ public interface RegionServerServices extends OnlineRegions, FavoredNodesForRegi
WAL getWAL(HRegionInfo regionInfo) throws IOException;
/**
+ * Releases the dependency of this region to the WAL previously obtained from
+ * {@link #getWAL(HRegionInfo)}.
+ */
+ void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException;
+
+ /**
* @return Implementation of {@link CompactionRequestor} or null.
*/
CompactionRequestor getCompactionRequester();
http://git-wip-us.apache.org/repos/asf/hbase/blob/756e6de2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java
index 70e5283..237eb5a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java
@@ -19,6 +19,9 @@
package org.apache.hadoop.hbase.regionserver.handler;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+import java.io.IOException;
+
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.executor.EventType;
@@ -37,4 +40,9 @@ public class CloseMetaHandler extends CloseRegionHandler {
final boolean abort) {
super(server, rsServices, regionInfo, abort, EventType.M_RS_CLOSE_META, null);
}
+
+ @Override
+ protected void releaseWALIfNeeded() throws IOException {
+ rsServices.releaseWAL(regionInfo, region.getWAL());
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/756e6de2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java
index 26e824d..b5bab27 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java
@@ -44,8 +44,9 @@ public class CloseRegionHandler extends EventHandler {
// have a running queue of user regions to close?
private static final Log LOG = LogFactory.getLog(CloseRegionHandler.class);
- private final RegionServerServices rsServices;
- private final HRegionInfo regionInfo;
+ protected final RegionServerServices rsServices;
+ protected final HRegionInfo regionInfo;
+ protected HRegion region;
// If true, the hosting server is aborting. Region close process is different
// when we are aborting.
@@ -90,7 +91,7 @@ public class CloseRegionHandler extends EventHandler {
LOG.debug("Processing close of " + name);
String encodedRegionName = regionInfo.getEncodedName();
// Check that this region is being served here
- HRegion region = (HRegion)rsServices.getFromOnlineRegions(encodedRegionName);
+ region = (HRegion)rsServices.getFromOnlineRegions(encodedRegionName);
if (region == null) {
LOG.warn("Received CLOSE for region " + name + " but currently not serving - ignoring");
// TODO: do better than a simple warning
@@ -106,6 +107,10 @@ public class CloseRegionHandler extends EventHandler {
regionInfo.getRegionNameAsString());
return;
}
+
+ if (!abort) {
+ releaseWALIfNeeded();
+ }
} catch (IOException ioe) {
// An IOException here indicates that we couldn't successfully flush the
// memstore before closing. So, we need to abort the server and allow
@@ -125,4 +130,8 @@ public class CloseRegionHandler extends EventHandler {
remove(this.regionInfo.getEncodedNameAsBytes(), Boolean.FALSE);
}
}
+
+ protected void releaseWALIfNeeded() throws IOException {
+ // release the WAL if needed. Only meta does this for now.
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/756e6de2/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
index 0317b66..7426979 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
@@ -250,6 +250,21 @@ public class WALFactory {
return metaProvider.getWAL(identifier, null);
}
+ /**
+ * Closes the meta WAL and meta WALProvider
+ */
+ public void closeMetaWAL(final byte[] identifier) throws IOException {
+ // NOTE: this assumes single META region. The close of WAL does not do ref-counting for the
+ // number of regions depending on the meta WAL
+ WALProvider metaProvider = this.metaProvider.get();
+ if (metaProvider != null) {
+ if (this.metaProvider.compareAndSet(metaProvider, null)) {
+ // close the metaProvider
+ metaProvider.close();
+ }
+ }
+ }
+
public Reader createReader(final FileSystem fs, final Path path) throws IOException {
return createReader(fs, path, (CancelableProgressable)null);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/756e6de2/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
index c126b19..250d3c1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
@@ -306,4 +306,8 @@ public class MockRegionServerServices implements RegionServerServices {
public double getCompactionPressure() {
return 0;
}
+
+ @Override
+ public void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException {
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/756e6de2/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
index eb8f803..e89ce00 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
@@ -66,8 +66,6 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsReques
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsResponse;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
-import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionRequest;
-import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionResponse;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryResponse;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest;
@@ -80,6 +78,8 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateConfiguratio
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateConfigurationResponse;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesResponse;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionRequest;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionResponse;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileResponse;
@@ -651,4 +651,8 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices {
public double getCompactionPressure() {
return 0;
}
-}
\ No newline at end of file
+
+ @Override
+ public void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException {
+ }
+}