You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2014/05/30 02:18:49 UTC
git commit: HBASE-11272 Backport HBASE-11218 (Data loss in HBase
standalone mode) to 0.98
Repository: hbase
Updated Branches:
refs/heads/0.98 111ebaaea -> 13c7ff669
HBASE-11272 Backport HBASE-11218 (Data loss in HBase standalone mode) to 0.98
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/13c7ff66
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/13c7ff66
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/13c7ff66
Branch: refs/heads/0.98
Commit: 13c7ff669532e18ebf5261734b98c51a4ad69c6b
Parents: 111ebaa
Author: Andrew Purtell <ap...@apache.org>
Authored: Thu May 29 16:46:34 2014 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Thu May 29 17:16:55 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/fs/HFileSystem.java | 22 ++++++++++++++++++++
.../hadoop/hbase/master/HMasterCommandLine.java | 2 ++
.../hbase/zookeeper/MiniZooKeeperCluster.java | 13 +++++-------
.../hadoop/hbase/HBaseTestingUtility.java | 14 +++++++++++++
.../util/hbck/TestOfflineMetaRebuildBase.java | 4 ++--
.../util/hbck/TestOfflineMetaRebuildHole.java | 3 +--
.../hbck/TestOfflineMetaRebuildOverlap.java | 3 +--
7 files changed, 47 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/13c7ff66/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
index 8178787..a6e9ef0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
@@ -81,6 +81,13 @@ public class HFileSystem extends FilterFileSystem {
this.useHBaseChecksum = useHBaseChecksum;
fs.initialize(getDefaultUri(conf), conf);
+
+ // disable checksum verification for local fileSystem, see HBASE-11218
+ if (fs instanceof LocalFileSystem) {
+ setWriteChecksum(fs, false);
+ setVerifyChecksum(fs, false);
+ }
+
addLocationsOrderInterceptor(conf);
// If hbase checksum verification is switched on, then create a new
@@ -101,6 +108,21 @@ public class HFileSystem extends FilterFileSystem {
}
}
+ private static void setWriteChecksum(FileSystem fs, boolean writeChecksum) throws IOException {
+ // Some Hadoop versions will not have this method available
+ try {
+ FileSystem.class.getMethod("setWriteChecksum", boolean.class).invoke(fs, writeChecksum);
+ } catch (NoSuchMethodException e) {
+ LOG.debug("FileSystem does not support setWriteChecksum, ignoring");
+ } catch (Throwable t) {
+ throw new IOException("setWriteChecksum failed", t);
+ }
+ }
+
+ private static void setVerifyChecksum(FileSystem fs, boolean verifyChecksum) throws IOException {
+ fs.setVerifyChecksum(verifyChecksum);
+ }
+
/**
* Wrap a FileSystem object within a HFileSystem. The noChecksumFs and
* writefs are both set to be the same specified fs.
http://git-wip-us.apache.org/repos/asf/hbase/blob/13c7ff66/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
index 886fde6..e56ab6f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
@@ -30,6 +30,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.ZNodeClearer;
import org.apache.hadoop.hbase.HConstants;
@@ -172,6 +173,7 @@ public class HMasterCommandLine extends ServerCommandLine {
}
conf.set(HConstants.ZOOKEEPER_CLIENT_PORT,
Integer.toString(clientPort));
+ conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 10 *1000);
// Need to have the zk cluster shutdown when master is shutdown.
// Run a subclass that does the zk cluster shutdown on its way out.
LocalHBaseCluster cluster = new LocalHBaseCluster(conf, conf.getInt("hbase.masters", 1),
http://git-wip-us.apache.org/repos/asf/hbase/blob/13c7ff66/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
index 3a42ea0..69a7cba 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
@@ -152,7 +152,7 @@ public class MiniZooKeeperCluster {
// running all the ZK servers
for (int i = 0; i < numZooKeeperServers; i++) {
File dir = new File(baseDir, "zookeeper_"+i).getAbsoluteFile();
- recreateDir(dir);
+ createDir(dir);
int tickTimeToUse;
if (this.tickTime > 0) {
tickTimeToUse = this.tickTime;
@@ -202,14 +202,11 @@ public class MiniZooKeeperCluster {
return clientPort;
}
- private void recreateDir(File dir) throws IOException {
- if (dir.exists()) {
- if(!FileUtil.fullyDelete(dir)) {
- throw new IOException("Could not delete zk base directory: " + dir);
- }
- }
+ private void createDir(File dir) throws IOException {
try {
- dir.mkdirs();
+ if (!dir.exists()) {
+ dir.mkdirs();
+ }
} catch (SecurityException e) {
throw new IOException("creating dir: " + dir, e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/13c7ff66/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 2984fe3..6e7a5ee 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -952,6 +952,20 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
miniClusterRunning = false;
LOG.info("Minicluster is down");
}
+
+ /**
+ * @return True if we removed the test dirs
+ * @throws IOException
+ */
+ @Override
+ public boolean cleanupTestDir() throws IOException {
+ boolean ret = super.cleanupTestDir();
+ if (deleteDir(this.clusterTestDir)) {
+ this.clusterTestDir = null;
+ return ret & true;
+ }
+ return false;
+ }
/**
* Shutdown HBase mini cluster. Does not shutdown zk or dfs if running.
http://git-wip-us.apache.org/repos/asf/hbase/blob/13c7ff66/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java
index 1a85bc5..7a075a6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java
@@ -65,9 +65,9 @@ public class TestOfflineMetaRebuildBase extends OfflineMetaRebuildTestCore {
assertTrue(fsck.rebuildMeta(false));
// bring up the minicluster
- TEST_UTIL.startMiniZKCluster(); // tables seem enabled by default
+ TEST_UTIL.startMiniZKCluster();
TEST_UTIL.restartHBaseCluster(3);
-
+ TEST_UTIL.getHBaseAdmin().enableTable(table);
ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
LOG.info("Waiting for no more RIT");
http://git-wip-us.apache.org/repos/asf/hbase/blob/13c7ff66/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java
index 9cf8516..a3225bd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java
@@ -96,8 +96,7 @@ public class TestOfflineMetaRebuildHole extends OfflineMetaRebuildTestCore {
assertErrors(doFsck(conf, false), new ERROR_CODE[] {
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
- ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
- ERROR_CODE.HOLE_IN_REGION_CHAIN});
+ ERROR_CODE.NOT_IN_META_OR_DEPLOYED});
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/13c7ff66/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java
index 34098ba..9a17948 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java
@@ -107,8 +107,7 @@ public class TestOfflineMetaRebuildOverlap extends OfflineMetaRebuildTestCore {
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
- ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
- ERROR_CODE.HOLE_IN_REGION_CHAIN});
+ ERROR_CODE.NOT_IN_META_OR_DEPLOYED});
}
}