You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2016/09/26 20:49:54 UTC
[1/6] hbase git commit: HBASE-16649 Truncate table with splits
preserved can cause both data loss and truncated data appeared again
Repository: hbase
Updated Branches:
refs/heads/0.98 88bba5169 -> 1bdf05b1f
refs/heads/branch-1 d6f9eab4b -> 4566e4df5
refs/heads/branch-1.1 cdc799e9b -> 88512be52
refs/heads/branch-1.2 f224e09ad -> 2733e24d3
refs/heads/branch-1.3 49a4980e6 -> 1441b7c79
refs/heads/master 43f47a8e7 -> f06c0060a
HBASE-16649 Truncate table with splits preserved can cause both data loss and truncated data appeared again
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f06c0060
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f06c0060
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f06c0060
Branch: refs/heads/master
Commit: f06c0060aa13a2b5b18edeb66b7479bdd3c6fdc8
Parents: 43f47a8
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Mon Sep 26 12:58:04 2016 -0700
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Mon Sep 26 12:58:04 2016 -0700
----------------------------------------------------------------------
.../hadoop/hbase/master/CatalogJanitor.java | 3 +++
.../apache/hadoop/hbase/master/ServerManager.java | 18 ++++++++++++++++++
.../master/procedure/DeleteTableProcedure.java | 3 +++
.../master/procedure/TruncateTableProcedure.java | 10 ++++++++++
.../hadoop/hbase/master/TestCatalogJanitor.java | 7 +++++++
.../procedure/TestTruncateTableProcedure.java | 8 ++++----
6 files changed, 45 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f06c0060/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index d14a925..e748c3b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -213,6 +213,8 @@ public class CatalogJanitor extends ScheduledChore {
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);
MetaTableAccessor.deleteMergeQualifiers(services.getConnection(), mergedRegion);
+ services.getServerManager().removeRegion(regionA);
+ services.getServerManager().removeRegion(regionB);
return true;
}
return false;
@@ -357,6 +359,7 @@ public class CatalogJanitor extends ScheduledChore {
if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
MetaTableAccessor.deleteRegion(this.connection, parent);
+ services.getServerManager().removeRegion(parent);
result = true;
}
return result;
http://git-wip-us.apache.org/repos/asf/hbase/blob/f06c0060/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index f97dfb4..3d8715a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -1184,4 +1184,22 @@ public class ServerManager {
deadservers.cleanAllPreviousInstances(serverName);
}
}
+
+ /**
+ * Called by delete table and similar to notify the ServerManager that a region was removed.
+ */
+ public void removeRegion(final HRegionInfo regionInfo) {
+ final byte[] encodedName = regionInfo.getEncodedNameAsBytes();
+ storeFlushedSequenceIdsByRegion.remove(encodedName);
+ flushedSequenceIdByRegion.remove(encodedName);
+ }
+
+ /**
+ * Called by delete table and similar to notify the ServerManager that a region was removed.
+ */
+ public void removeRegions(final List<HRegionInfo> regions) {
+ for (HRegionInfo hri: regions) {
+ removeRegion(hri);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f06c0060/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
index f8dd0b6..fdf0b82 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
@@ -363,6 +363,9 @@ public class DeleteTableProcedure
// Clean any remaining rows for this table.
cleanAnyRemainingRows(env, tableName);
+
+ // clean region references from the server manager
+ env.getMasterServices().getServerManager().removeRegions(regions);
}
protected static void deleteAssignmentState(final MasterProcedureEnv env,
http://git-wip-us.apache.org/repos/asf/hbase/blob/f06c0060/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
index 701eaf7..95c3192 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
@@ -107,6 +107,8 @@ public class TruncateTableProcedure
if (!preserveSplits) {
// if we are not preserving splits, generate a new single region
regions = Arrays.asList(ModifyRegionUtils.createHRegionInfos(hTableDescriptor, null));
+ } else {
+ regions = recreateRegionInfo(regions);
}
setNextState(TruncateTableState.TRUNCATE_TABLE_CREATE_FS_LAYOUT);
break;
@@ -256,6 +258,14 @@ public class TruncateTableProcedure
}
}
+ private static List<HRegionInfo> recreateRegionInfo(final List<HRegionInfo> regions) {
+ ArrayList<HRegionInfo> newRegions = new ArrayList<HRegionInfo>(regions.size());
+ for (HRegionInfo hri: regions) {
+ newRegions.add(new HRegionInfo(hri.getTable(), hri.getStartKey(), hri.getEndKey()));
+ }
+ return newRegions;
+ }
+
private boolean prepareTruncate(final MasterProcedureEnv env) throws IOException {
try {
env.getMasterServices().checkTableModifiable(getTableName());
http://git-wip-us.apache.org/repos/asf/hbase/blob/f06c0060/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 67ea5f6..140dcd9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -91,6 +91,7 @@ public class TestCatalogJanitor {
private final ClusterConnection connection;
private final MasterFileSystem mfs;
private final AssignmentManager asm;
+ private final ServerManager sm;
MockMasterServices(final HBaseTestingUtility htu) throws IOException {
super(htu.getConfiguration());
@@ -134,6 +135,7 @@ public class TestCatalogJanitor {
this.mfs = new MasterFileSystem(this);
this.asm = Mockito.mock(AssignmentManager.class);
+ this.sm = Mockito.mock(ServerManager.class);
}
@Override
@@ -157,6 +159,11 @@ public class TestCatalogJanitor {
}
@Override
+ public ServerManager getServerManager() {
+ return this.sm;
+ }
+
+ @Override
public CoordinatedStateManager getCoordinatedStateManager() {
BaseCoordinatedStateManager m = Mockito.mock(BaseCoordinatedStateManager.class);
SplitLogManagerCoordination c = Mockito.mock(SplitLogManagerCoordination.class);
http://git-wip-us.apache.org/repos/asf/hbase/blob/f06c0060/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
index a7bfe18..c72e3f3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
@@ -165,10 +165,10 @@ public class TestTruncateTableProcedure {
UTIL.waitUntilAllRegionsAssigned(tableName);
// validate the table regions and layout
+ regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[0]);
if (preserveSplits) {
- assertEquals(1 + splitKeys.length, UTIL.getHBaseAdmin().getTableRegions(tableName).size());
+ assertEquals(1 + splitKeys.length, regions.length);
} else {
- regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[1]);
assertEquals(1, regions.length);
}
MasterProcedureTestingUtility.validateTableCreation(
@@ -231,10 +231,10 @@ public class TestTruncateTableProcedure {
UTIL.waitUntilAllRegionsAssigned(tableName);
// validate the table regions and layout
+ regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[0]);
if (preserveSplits) {
- assertEquals(1 + splitKeys.length, UTIL.getHBaseAdmin().getTableRegions(tableName).size());
+ assertEquals(1 + splitKeys.length, regions.length);
} else {
- regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[1]);
assertEquals(1, regions.length);
}
MasterProcedureTestingUtility.validateTableCreation(
[3/6] hbase git commit: HBASE-16649 Truncate table with splits
preserved can cause both data loss and truncated data appeared again
Posted by mb...@apache.org.
HBASE-16649 Truncate table with splits preserved can cause both data loss and truncated data appeared again
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1441b7c7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1441b7c7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1441b7c7
Branch: refs/heads/branch-1.3
Commit: 1441b7c795292ce5b056ff9e3b5b2443ecd8e8cb
Parents: 49a4980
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Mon Sep 26 12:58:04 2016 -0700
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Mon Sep 26 13:09:23 2016 -0700
----------------------------------------------------------------------
.../hadoop/hbase/master/CatalogJanitor.java | 8 +++++---
.../apache/hadoop/hbase/master/ServerManager.java | 18 ++++++++++++++++++
.../master/procedure/DeleteTableProcedure.java | 3 +++
.../master/procedure/TruncateTableProcedure.java | 10 ++++++++++
.../hadoop/hbase/master/TestCatalogJanitor.java | 4 +++-
.../procedure/TestTruncateTableProcedure.java | 8 ++++----
6 files changed, 43 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/1441b7c7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index d686d7c..48105c8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -215,8 +215,9 @@ public class CatalogJanitor extends ScheduledChore {
+ " from fs because merged region no longer holds references");
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);
- MetaTableAccessor.deleteMergeQualifiers(server.getConnection(),
- mergedRegion);
+ MetaTableAccessor.deleteMergeQualifiers(services.getConnection(), mergedRegion);
+ services.getServerManager().removeRegion(regionA);
+ services.getServerManager().removeRegion(regionB);
return true;
}
return false;
@@ -349,6 +350,7 @@ public class CatalogJanitor extends ScheduledChore {
if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
MetaTableAccessor.deleteRegion(this.connection, parent);
+ services.getServerManager().removeRegion(parent);
result = true;
}
return result;
@@ -450,4 +452,4 @@ public class CatalogJanitor extends ScheduledChore {
return cleanMergeRegion(region, mergeRegions.getFirst(),
mergeRegions.getSecond());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/1441b7c7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index fb98300..305687f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -1206,4 +1206,22 @@ public class ServerManager {
deadservers.cleanAllPreviousInstances(serverName);
}
}
+
+ /**
+ * Called by delete table and similar to notify the ServerManager that a region was removed.
+ */
+ public void removeRegion(final HRegionInfo regionInfo) {
+ final byte[] encodedName = regionInfo.getEncodedNameAsBytes();
+ storeFlushedSequenceIdsByRegion.remove(encodedName);
+ flushedSequenceIdByRegion.remove(encodedName);
+ }
+
+ /**
+ * Called by delete table and similar to notify the ServerManager that a region was removed.
+ */
+ public void removeRegions(final List<HRegionInfo> regions) {
+ for (HRegionInfo hri: regions) {
+ removeRegion(hri);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/1441b7c7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
index 44dde9e..20466d1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
@@ -379,6 +379,9 @@ public class DeleteTableProcedure
// Clean any remaining rows for this table.
cleanAnyRemainingRows(env, tableName);
+
+ // clean region references from the server manager
+ env.getMasterServices().getServerManager().removeRegions(regions);
}
protected static void deleteAssignmentState(final MasterProcedureEnv env,
http://git-wip-us.apache.org/repos/asf/hbase/blob/1441b7c7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
index 3a3739d..9dccef6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
@@ -105,6 +105,8 @@ public class TruncateTableProcedure
if (!preserveSplits) {
// if we are not preserving splits, generate a new single region
regions = Arrays.asList(ModifyRegionUtils.createHRegionInfos(hTableDescriptor, null));
+ } else {
+ regions = recreateRegionInfo(regions);
}
setNextState(TruncateTableState.TRUNCATE_TABLE_CREATE_FS_LAYOUT);
break;
@@ -245,6 +247,14 @@ public class TruncateTableProcedure
}
}
+ private static List<HRegionInfo> recreateRegionInfo(final List<HRegionInfo> regions) {
+ ArrayList<HRegionInfo> newRegions = new ArrayList<HRegionInfo>(regions.size());
+ for (HRegionInfo hri: regions) {
+ newRegions.add(new HRegionInfo(hri.getTable(), hri.getStartKey(), hri.getEndKey()));
+ }
+ return newRegions;
+ }
+
private boolean prepareTruncate(final MasterProcedureEnv env) throws IOException {
try {
env.getMasterServices().checkTableModifiable(getTableName());
http://git-wip-us.apache.org/repos/asf/hbase/blob/1441b7c7/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 9f9b0a8..34948de 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -215,10 +215,12 @@ public class TestCatalogJanitor {
class MockMasterServices implements MasterServices {
private final MasterFileSystem mfs;
private final AssignmentManager asm;
+ private final ServerManager sm;
MockMasterServices(final Server server) throws IOException {
this.mfs = new MasterFileSystem(server, this);
this.asm = Mockito.mock(AssignmentManager.class);
+ this.sm = Mockito.mock(ServerManager.class);
}
@Override
@@ -283,7 +285,7 @@ public class TestCatalogJanitor {
@Override
public ServerManager getServerManager() {
- return null;
+ return sm;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/1441b7c7/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
index 08442ce..941dd2f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
@@ -162,10 +162,10 @@ public class TestTruncateTableProcedure {
UTIL.waitUntilAllRegionsAssigned(tableName);
// validate the table regions and layout
+ regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[0]);
if (preserveSplits) {
- assertEquals(1 + splitKeys.length, UTIL.getHBaseAdmin().getTableRegions(tableName).size());
+ assertEquals(1 + splitKeys.length, regions.length);
} else {
- regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[1]);
assertEquals(1, regions.length);
}
MasterProcedureTestingUtility.validateTableCreation(
@@ -229,10 +229,10 @@ public class TestTruncateTableProcedure {
UTIL.waitUntilAllRegionsAssigned(tableName);
// validate the table regions and layout
+ regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[0]);
if (preserveSplits) {
- assertEquals(1 + splitKeys.length, UTIL.getHBaseAdmin().getTableRegions(tableName).size());
+ assertEquals(1 + splitKeys.length, regions.length);
} else {
- regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[1]);
assertEquals(1, regions.length);
}
MasterProcedureTestingUtility.validateTableCreation(
[5/6] hbase git commit: HBASE-16649 Truncate table with splits
preserved can cause both data loss and truncated data appeared again
Posted by mb...@apache.org.
HBASE-16649 Truncate table with splits preserved can cause both data loss and truncated data appeared again
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/88512be5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/88512be5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/88512be5
Branch: refs/heads/branch-1.1
Commit: 88512be52b8707fb87ab2c5979fd71664a417a90
Parents: cdc799e
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Mon Sep 26 12:58:04 2016 -0700
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Mon Sep 26 13:21:24 2016 -0700
----------------------------------------------------------------------
.../hadoop/hbase/master/CatalogJanitor.java | 8 +++++---
.../apache/hadoop/hbase/master/ServerManager.java | 18 ++++++++++++++++++
.../master/handler/TruncateTableHandler.java | 11 ++++++++++-
.../master/procedure/DeleteTableProcedure.java | 3 +++
.../master/procedure/TruncateTableProcedure.java | 10 ++++++++++
.../hadoop/hbase/master/TestCatalogJanitor.java | 4 +++-
.../procedure/TestTruncateTableProcedure.java | 8 ++++----
7 files changed, 53 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/88512be5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index d686d7c..48105c8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -215,8 +215,9 @@ public class CatalogJanitor extends ScheduledChore {
+ " from fs because merged region no longer holds references");
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);
- MetaTableAccessor.deleteMergeQualifiers(server.getConnection(),
- mergedRegion);
+ MetaTableAccessor.deleteMergeQualifiers(services.getConnection(), mergedRegion);
+ services.getServerManager().removeRegion(regionA);
+ services.getServerManager().removeRegion(regionB);
return true;
}
return false;
@@ -349,6 +350,7 @@ public class CatalogJanitor extends ScheduledChore {
if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
MetaTableAccessor.deleteRegion(this.connection, parent);
+ services.getServerManager().removeRegion(parent);
result = true;
}
return result;
@@ -450,4 +452,4 @@ public class CatalogJanitor extends ScheduledChore {
return cleanMergeRegion(region, mergeRegions.getFirst(),
mergeRegions.getSecond());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/88512be5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 11ca461..02ed231 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -1192,4 +1192,22 @@ public class ServerManager {
deadservers.cleanAllPreviousInstances(serverName);
}
}
+
+ /**
+ * Called by delete table and similar to notify the ServerManager that a region was removed.
+ */
+ public void removeRegion(final HRegionInfo regionInfo) {
+ final byte[] encodedName = regionInfo.getEncodedNameAsBytes();
+ storeFlushedSequenceIdsByRegion.remove(encodedName);
+ flushedSequenceIdByRegion.remove(encodedName);
+ }
+
+ /**
+ * Called by delete table and similar to notify the ServerManager that a region was removed.
+ */
+ public void removeRegions(final List<HRegionInfo> regions) {
+ for (HRegionInfo hri: regions) {
+ removeRegion(hri);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/88512be5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java
index 67bf343..600ae44 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java
@@ -104,7 +104,7 @@ public class TruncateTableHandler extends DeleteTableHandler {
HRegionInfo[] newRegions;
if (this.preserveSplits) {
- newRegions = regions.toArray(new HRegionInfo[regions.size()]);
+ newRegions = recreateRegionInfo(regions);
LOG.info("Truncate will preserve " + newRegions.length + " regions");
} else {
newRegions = new HRegionInfo[1];
@@ -143,4 +143,13 @@ public class TruncateTableHandler extends DeleteTableHandler {
throw e;
}
}
+
+ private static HRegionInfo[] recreateRegionInfo(final List<HRegionInfo> regions) {
+ HRegionInfo[] newRegions = new HRegionInfo[regions.size()];
+ int index = 0;
+ for (HRegionInfo hri: regions) {
+ newRegions[index++] = new HRegionInfo(hri.getTable(), hri.getStartKey(), hri.getEndKey());
+ }
+ return newRegions;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/88512be5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
index 45cb2eb..734fa13 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
@@ -381,6 +381,9 @@ public class DeleteTableProcedure
// Clean any remaining rows for this table.
cleanAnyRemainingRows(env, tableName);
+
+ // clean region references from the server manager
+ env.getMasterServices().getServerManager().removeRegions(regions);
}
protected static void deleteAssignmentState(final MasterProcedureEnv env,
http://git-wip-us.apache.org/repos/asf/hbase/blob/88512be5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
index 1df18f4..d065d2f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
@@ -104,6 +104,8 @@ public class TruncateTableProcedure
if (!preserveSplits) {
// if we are not preserving splits, generate a new single region
regions = Arrays.asList(ModifyRegionUtils.createHRegionInfos(hTableDescriptor, null));
+ } else {
+ regions = recreateRegionInfo(regions);
}
setNextState(TruncateTableState.TRUNCATE_TABLE_CREATE_FS_LAYOUT);
break;
@@ -247,6 +249,14 @@ public class TruncateTableProcedure
}
}
+ private static List<HRegionInfo> recreateRegionInfo(final List<HRegionInfo> regions) {
+ ArrayList<HRegionInfo> newRegions = new ArrayList<HRegionInfo>(regions.size());
+ for (HRegionInfo hri: regions) {
+ newRegions.add(new HRegionInfo(hri.getTable(), hri.getStartKey(), hri.getEndKey()));
+ }
+ return newRegions;
+ }
+
private boolean prepareTruncate(final MasterProcedureEnv env) throws IOException {
try {
env.getMasterServices().checkTableModifiable(getTableName());
http://git-wip-us.apache.org/repos/asf/hbase/blob/88512be5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 1f9c087..ae175b9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -213,10 +213,12 @@ public class TestCatalogJanitor {
class MockMasterServices implements MasterServices {
private final MasterFileSystem mfs;
private final AssignmentManager asm;
+ private final ServerManager sm;
MockMasterServices(final Server server) throws IOException {
this.mfs = new MasterFileSystem(server, this);
this.asm = Mockito.mock(AssignmentManager.class);
+ this.sm = Mockito.mock(ServerManager.class);
}
@Override
@@ -276,7 +278,7 @@ public class TestCatalogJanitor {
@Override
public ServerManager getServerManager() {
- return null;
+ return sm;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/88512be5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
index 08442ce..941dd2f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
@@ -162,10 +162,10 @@ public class TestTruncateTableProcedure {
UTIL.waitUntilAllRegionsAssigned(tableName);
// validate the table regions and layout
+ regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[0]);
if (preserveSplits) {
- assertEquals(1 + splitKeys.length, UTIL.getHBaseAdmin().getTableRegions(tableName).size());
+ assertEquals(1 + splitKeys.length, regions.length);
} else {
- regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[1]);
assertEquals(1, regions.length);
}
MasterProcedureTestingUtility.validateTableCreation(
@@ -229,10 +229,10 @@ public class TestTruncateTableProcedure {
UTIL.waitUntilAllRegionsAssigned(tableName);
// validate the table regions and layout
+ regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[0]);
if (preserveSplits) {
- assertEquals(1 + splitKeys.length, UTIL.getHBaseAdmin().getTableRegions(tableName).size());
+ assertEquals(1 + splitKeys.length, regions.length);
} else {
- regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[1]);
assertEquals(1, regions.length);
}
MasterProcedureTestingUtility.validateTableCreation(
[4/6] hbase git commit: HBASE-16649 Truncate table with splits
preserved can cause both data loss and truncated data appeared again
Posted by mb...@apache.org.
HBASE-16649 Truncate table with splits preserved can cause both data loss and truncated data appeared again
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2733e24d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2733e24d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2733e24d
Branch: refs/heads/branch-1.2
Commit: 2733e24d3f2f110ac98d8876ee1de1fd9740b51e
Parents: f224e09
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Mon Sep 26 12:58:04 2016 -0700
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Mon Sep 26 13:12:27 2016 -0700
----------------------------------------------------------------------
.../hadoop/hbase/master/CatalogJanitor.java | 8 +++++---
.../apache/hadoop/hbase/master/ServerManager.java | 18 ++++++++++++++++++
.../master/procedure/DeleteTableProcedure.java | 3 +++
.../master/procedure/TruncateTableProcedure.java | 10 ++++++++++
.../hadoop/hbase/master/TestCatalogJanitor.java | 4 +++-
.../procedure/TestTruncateTableProcedure.java | 8 ++++----
6 files changed, 43 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2733e24d/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index d686d7c..48105c8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -215,8 +215,9 @@ public class CatalogJanitor extends ScheduledChore {
+ " from fs because merged region no longer holds references");
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);
- MetaTableAccessor.deleteMergeQualifiers(server.getConnection(),
- mergedRegion);
+ MetaTableAccessor.deleteMergeQualifiers(services.getConnection(), mergedRegion);
+ services.getServerManager().removeRegion(regionA);
+ services.getServerManager().removeRegion(regionB);
return true;
}
return false;
@@ -349,6 +350,7 @@ public class CatalogJanitor extends ScheduledChore {
if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
MetaTableAccessor.deleteRegion(this.connection, parent);
+ services.getServerManager().removeRegion(parent);
result = true;
}
return result;
@@ -450,4 +452,4 @@ public class CatalogJanitor extends ScheduledChore {
return cleanMergeRegion(region, mergeRegions.getFirst(),
mergeRegions.getSecond());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2733e24d/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 2a23025..f6f2d03 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -1205,4 +1205,22 @@ public class ServerManager {
deadservers.cleanAllPreviousInstances(serverName);
}
}
+
+ /**
+ * Called by delete table and similar to notify the ServerManager that a region was removed.
+ */
+ public void removeRegion(final HRegionInfo regionInfo) {
+ final byte[] encodedName = regionInfo.getEncodedNameAsBytes();
+ storeFlushedSequenceIdsByRegion.remove(encodedName);
+ flushedSequenceIdByRegion.remove(encodedName);
+ }
+
+ /**
+ * Called by delete table and similar to notify the ServerManager that a region was removed.
+ */
+ public void removeRegions(final List<HRegionInfo> regions) {
+ for (HRegionInfo hri: regions) {
+ removeRegion(hri);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2733e24d/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
index 44dde9e..20466d1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
@@ -379,6 +379,9 @@ public class DeleteTableProcedure
// Clean any remaining rows for this table.
cleanAnyRemainingRows(env, tableName);
+
+ // clean region references from the server manager
+ env.getMasterServices().getServerManager().removeRegions(regions);
}
protected static void deleteAssignmentState(final MasterProcedureEnv env,
http://git-wip-us.apache.org/repos/asf/hbase/blob/2733e24d/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
index 3a3739d..9dccef6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
@@ -105,6 +105,8 @@ public class TruncateTableProcedure
if (!preserveSplits) {
// if we are not preserving splits, generate a new single region
regions = Arrays.asList(ModifyRegionUtils.createHRegionInfos(hTableDescriptor, null));
+ } else {
+ regions = recreateRegionInfo(regions);
}
setNextState(TruncateTableState.TRUNCATE_TABLE_CREATE_FS_LAYOUT);
break;
@@ -245,6 +247,14 @@ public class TruncateTableProcedure
}
}
+ private static List<HRegionInfo> recreateRegionInfo(final List<HRegionInfo> regions) {
+ ArrayList<HRegionInfo> newRegions = new ArrayList<HRegionInfo>(regions.size());
+ for (HRegionInfo hri: regions) {
+ newRegions.add(new HRegionInfo(hri.getTable(), hri.getStartKey(), hri.getEndKey()));
+ }
+ return newRegions;
+ }
+
private boolean prepareTruncate(final MasterProcedureEnv env) throws IOException {
try {
env.getMasterServices().checkTableModifiable(getTableName());
http://git-wip-us.apache.org/repos/asf/hbase/blob/2733e24d/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 8c05fab..4bc13a6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -213,10 +213,12 @@ public class TestCatalogJanitor {
class MockMasterServices implements MasterServices {
private final MasterFileSystem mfs;
private final AssignmentManager asm;
+ private final ServerManager sm;
MockMasterServices(final Server server) throws IOException {
this.mfs = new MasterFileSystem(server, this);
this.asm = Mockito.mock(AssignmentManager.class);
+ this.sm = Mockito.mock(ServerManager.class);
}
@Override
@@ -271,7 +273,7 @@ public class TestCatalogJanitor {
@Override
public ServerManager getServerManager() {
- return null;
+ return sm;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/2733e24d/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
index 08442ce..941dd2f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
@@ -162,10 +162,10 @@ public class TestTruncateTableProcedure {
UTIL.waitUntilAllRegionsAssigned(tableName);
// validate the table regions and layout
+ regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[0]);
if (preserveSplits) {
- assertEquals(1 + splitKeys.length, UTIL.getHBaseAdmin().getTableRegions(tableName).size());
+ assertEquals(1 + splitKeys.length, regions.length);
} else {
- regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[1]);
assertEquals(1, regions.length);
}
MasterProcedureTestingUtility.validateTableCreation(
@@ -229,10 +229,10 @@ public class TestTruncateTableProcedure {
UTIL.waitUntilAllRegionsAssigned(tableName);
// validate the table regions and layout
+ regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[0]);
if (preserveSplits) {
- assertEquals(1 + splitKeys.length, UTIL.getHBaseAdmin().getTableRegions(tableName).size());
+ assertEquals(1 + splitKeys.length, regions.length);
} else {
- regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[1]);
assertEquals(1, regions.length);
}
MasterProcedureTestingUtility.validateTableCreation(
[2/6] hbase git commit: HBASE-16649 Truncate table with splits
preserved can cause both data loss and truncated data appeared again
Posted by mb...@apache.org.
HBASE-16649 Truncate table with splits preserved can cause both data loss and truncated data appeared again
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4566e4df
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4566e4df
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4566e4df
Branch: refs/heads/branch-1
Commit: 4566e4df58bdd176228aab2cd3cfd80dd983072f
Parents: d6f9eab
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Mon Sep 26 12:58:04 2016 -0700
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Mon Sep 26 13:09:06 2016 -0700
----------------------------------------------------------------------
.../hadoop/hbase/master/CatalogJanitor.java | 8 +++++---
.../apache/hadoop/hbase/master/ServerManager.java | 18 ++++++++++++++++++
.../master/procedure/DeleteTableProcedure.java | 3 +++
.../master/procedure/TruncateTableProcedure.java | 10 ++++++++++
.../hadoop/hbase/master/TestCatalogJanitor.java | 4 +++-
.../procedure/TestTruncateTableProcedure.java | 8 ++++----
6 files changed, 43 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/4566e4df/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index a921112..17644eb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -216,8 +216,9 @@ public class CatalogJanitor extends ScheduledChore {
+ " from fs because merged region no longer holds references");
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);
- MetaTableAccessor.deleteMergeQualifiers(server.getConnection(),
- mergedRegion);
+ MetaTableAccessor.deleteMergeQualifiers(services.getConnection(), mergedRegion);
+ services.getServerManager().removeRegion(regionA);
+ services.getServerManager().removeRegion(regionB);
return true;
}
return false;
@@ -360,6 +361,7 @@ public class CatalogJanitor extends ScheduledChore {
if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
MetaTableAccessor.deleteRegion(this.connection, parent);
+ services.getServerManager().removeRegion(parent);
result = true;
}
return result;
@@ -461,4 +463,4 @@ public class CatalogJanitor extends ScheduledChore {
return cleanMergeRegion(region, mergeRegions.getFirst(),
mergeRegions.getSecond());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/4566e4df/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 5eb569e..bcaa4d0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -1212,4 +1212,22 @@ public class ServerManager {
deadservers.cleanAllPreviousInstances(serverName);
}
}
+
+ /**
+ * Called by delete table and similar to notify the ServerManager that a region was removed.
+ */
+ public void removeRegion(final HRegionInfo regionInfo) {
+ final byte[] encodedName = regionInfo.getEncodedNameAsBytes();
+ storeFlushedSequenceIdsByRegion.remove(encodedName);
+ flushedSequenceIdByRegion.remove(encodedName);
+ }
+
+ /**
+ * Called by delete table and similar to notify the ServerManager that a region was removed.
+ */
+ public void removeRegions(final List<HRegionInfo> regions) {
+ for (HRegionInfo hri: regions) {
+ removeRegion(hri);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/4566e4df/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
index 44dde9e..20466d1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
@@ -379,6 +379,9 @@ public class DeleteTableProcedure
// Clean any remaining rows for this table.
cleanAnyRemainingRows(env, tableName);
+
+ // clean region references from the server manager
+ env.getMasterServices().getServerManager().removeRegions(regions);
}
protected static void deleteAssignmentState(final MasterProcedureEnv env,
http://git-wip-us.apache.org/repos/asf/hbase/blob/4566e4df/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
index 3a3739d..9dccef6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
@@ -105,6 +105,8 @@ public class TruncateTableProcedure
if (!preserveSplits) {
// if we are not preserving splits, generate a new single region
regions = Arrays.asList(ModifyRegionUtils.createHRegionInfos(hTableDescriptor, null));
+ } else {
+ regions = recreateRegionInfo(regions);
}
setNextState(TruncateTableState.TRUNCATE_TABLE_CREATE_FS_LAYOUT);
break;
@@ -245,6 +247,14 @@ public class TruncateTableProcedure
}
}
+ private static List<HRegionInfo> recreateRegionInfo(final List<HRegionInfo> regions) {
+ ArrayList<HRegionInfo> newRegions = new ArrayList<HRegionInfo>(regions.size());
+ for (HRegionInfo hri: regions) {
+ newRegions.add(new HRegionInfo(hri.getTable(), hri.getStartKey(), hri.getEndKey()));
+ }
+ return newRegions;
+ }
+
private boolean prepareTruncate(final MasterProcedureEnv env) throws IOException {
try {
env.getMasterServices().checkTableModifiable(getTableName());
http://git-wip-us.apache.org/repos/asf/hbase/blob/4566e4df/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 1fc74bc..13c0cb1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -216,10 +216,12 @@ public class TestCatalogJanitor {
class MockMasterServices implements MasterServices {
private final MasterFileSystem mfs;
private final AssignmentManager asm;
+ private final ServerManager sm;
MockMasterServices(final Server server) throws IOException {
this.mfs = new MasterFileSystem(server, this);
this.asm = Mockito.mock(AssignmentManager.class);
+ this.sm = Mockito.mock(ServerManager.class);
}
@Override
@@ -289,7 +291,7 @@ public class TestCatalogJanitor {
@Override
public ServerManager getServerManager() {
- return null;
+ return sm;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/4566e4df/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
index 08442ce..941dd2f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
@@ -162,10 +162,10 @@ public class TestTruncateTableProcedure {
UTIL.waitUntilAllRegionsAssigned(tableName);
// validate the table regions and layout
+ regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[0]);
if (preserveSplits) {
- assertEquals(1 + splitKeys.length, UTIL.getHBaseAdmin().getTableRegions(tableName).size());
+ assertEquals(1 + splitKeys.length, regions.length);
} else {
- regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[1]);
assertEquals(1, regions.length);
}
MasterProcedureTestingUtility.validateTableCreation(
@@ -229,10 +229,10 @@ public class TestTruncateTableProcedure {
UTIL.waitUntilAllRegionsAssigned(tableName);
// validate the table regions and layout
+ regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[0]);
if (preserveSplits) {
- assertEquals(1 + splitKeys.length, UTIL.getHBaseAdmin().getTableRegions(tableName).size());
+ assertEquals(1 + splitKeys.length, regions.length);
} else {
- regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[1]);
assertEquals(1, regions.length);
}
MasterProcedureTestingUtility.validateTableCreation(
[6/6] hbase git commit: HBASE-16649 Truncate table with splits
preserved can cause both data loss and truncated data appeared again
Posted by mb...@apache.org.
HBASE-16649 Truncate table with splits preserved can cause both data loss and truncated data appeared again
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1bdf05b1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1bdf05b1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1bdf05b1
Branch: refs/heads/0.98
Commit: 1bdf05b1fa1c3d09b9e3c4fb43957c5ff32ac701
Parents: 88bba51
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Mon Sep 26 13:48:20 2016 -0700
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Mon Sep 26 13:48:20 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hbase/master/CatalogJanitor.java | 3 +++
.../apache/hadoop/hbase/master/ServerManager.java | 17 +++++++++++++++++
.../hbase/master/handler/DeleteTableHandler.java | 3 +++
.../hbase/master/handler/TruncateTableHandler.java | 14 +++++++++++++-
.../hadoop/hbase/master/TestCatalogJanitor.java | 4 +++-
5 files changed, 39 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/1bdf05b1/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index 8e6c675..2edc480 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -204,6 +204,8 @@ public class CatalogJanitor extends Chore {
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);
MetaEditor.deleteMergeQualifiers(server.getCatalogTracker(), mergedRegion);
+ services.getServerManager().removeRegion(regionA);
+ services.getServerManager().removeRegion(regionB);
return true;
}
return false;
@@ -336,6 +338,7 @@ public class CatalogJanitor extends Chore {
if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
MetaEditor.deleteRegion(this.server.getCatalogTracker(), parent);
+ services.getServerManager().removeRegion(parent);
result = true;
}
return result;
http://git-wip-us.apache.org/repos/asf/hbase/blob/1bdf05b1/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 452ec14..ca34de5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -1058,4 +1058,21 @@ public class ServerManager {
deadservers.cleanAllPreviousInstances(serverName);
}
}
+
+ /**
+ * Called by delete table and similar to notify the ServerManager that a region was removed.
+ */
+ public void removeRegion(final HRegionInfo regionInfo) {
+ final byte[] encodedName = regionInfo.getEncodedNameAsBytes();
+ flushedSequenceIdByRegion.remove(encodedName);
+ }
+
+ /**
+ * Called by delete table and similar to notify the ServerManager that a region was removed.
+ */
+ public void removeRegions(final List<HRegionInfo> regions) {
+ for (HRegionInfo hri: regions) {
+ removeRegion(hri);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/1bdf05b1/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
index 474da0a..7172e98 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
@@ -134,6 +134,9 @@ public class DeleteTableHandler extends TableEventHandler {
// 9. Clean up any remaining rows for this table
cleanAnyRemainingRows();
+
+ // clean region references from the server manager
+ this.masterServices.getServerManager().removeRegions(regions);
}
if (cpHost != null) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/1bdf05b1/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java
index 17d87bb..6acfc35 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java
@@ -109,7 +109,7 @@ public class TruncateTableHandler extends DeleteTableHandler {
HRegionInfo[] newRegions;
if (this.preserveSplits) {
- newRegions = regions.toArray(new HRegionInfo[regions.size()]);
+ newRegions = recreateRegionInfo(regions);
LOG.info("Truncate will preserve " + newRegions.length + " regions");
} else {
newRegions = new HRegionInfo[1];
@@ -190,6 +190,9 @@ public class TruncateTableHandler extends DeleteTableHandler {
LOG.debug("Deleting regions from META");
MetaEditor.deleteRegions(this.server.getCatalogTracker(), regions);
+ // clean region references from the server manager
+ this.masterServices.getServerManager().removeRegions(regions);
+
// -----------------------------------------------------------------------
// NOTE: At this point we still have data on disk, but nothing in .META.
// if the rename below fails, hbck will report an inconsistency.
@@ -244,4 +247,13 @@ public class TruncateTableHandler extends DeleteTableHandler {
}
}
}
+
+ private static HRegionInfo[] recreateRegionInfo(final List<HRegionInfo> regions) {
+ HRegionInfo[] newRegions = new HRegionInfo[regions.size()];
+ int index = 0;
+ for (HRegionInfo hri: regions) {
+ newRegions[index++] = new HRegionInfo(hri.getTable(), hri.getStartKey(), hri.getEndKey());
+ }
+ return newRegions;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/1bdf05b1/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index d39fb86..009d212 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -176,10 +176,12 @@ public class TestCatalogJanitor {
class MockMasterServices implements MasterServices {
private final MasterFileSystem mfs;
private final AssignmentManager asm;
+ private final ServerManager sm;
MockMasterServices(final Server server) throws IOException {
this.mfs = new MasterFileSystem(server, this, false);
this.asm = Mockito.mock(AssignmentManager.class);
+ this.sm = Mockito.mock(ServerManager.class);
}
@Override
@@ -215,7 +217,7 @@ public class TestCatalogJanitor {
@Override
public ServerManager getServerManager() {
- return null;
+ return sm;
}
@Override