You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2009/07/16 07:09:54 UTC
svn commit: r794530 - in /hadoop/hbase/trunk_on_hadoop-0.18.3: ./ src/java/
src/java/org/apache/hadoop/hbase/
src/java/org/apache/hadoop/hbase/regionserver/
src/java/org/apache/hadoop/hbase/util/ src/test/data/
src/test/org/apache/hadoop/hbase/regionse...
Author: apurtell
Date: Thu Jul 16 05:09:53 2009
New Revision: 794530
URL: http://svn.apache.org/viewvc?rev=794530&view=rev
Log:
HBASE-1658, HBASE-1659, HBASE-1603, HBASE-1632, HBASE-1662
Added:
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/data/
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/data/hbase-0.19-two-small-tables.zip (with props)
Removed:
hadoop/hbase/trunk_on_hadoop-0.18.3/src/testdata/
Modified:
hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HConstants.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/FSUtils.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Merge.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/MigrationTest.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/master.jsp
hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/table.jsp
hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/regionserver/regionserver.jsp
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt Thu Jul 16 05:09:53 2009
@@ -265,6 +265,8 @@
HBASE-1650 HBASE-1551 broke the ability to manage non-regionserver
start-up/shut down. ie: you cant start/stop thrift on a cluster
anymore
+ HBASE-1658 Remove UI refresh -- its annoying
+ HBASE-1659 merge tool doesnt take binary regions with \x escape format
IMPROVEMENTS
HBASE-1089 Add count of regions on filesystem to master UI; add percentage
@@ -474,9 +476,12 @@
HBASE-1640 Allow passing arguments to jruby script run when run by bin/hbase shell
HBASE-698 HLog recovery is not performed after master failure
HBASE-1643 ScanDeleteTracker takes comparator but it unused
- HBASE-1603 MR failed "RetriesExhaustedException: Trying to contact region server
- Some server for region TestTable..." -- deubugging
+ HBASE-1603 MR failed "RetriesExhaustedException: Trying to contact region
+ server Some server for region TestTable..." -- deubugging
HBASE-1470 hbase and HADOOP-4379, dhruba's flush/sync
+ HBASE-1632 Write documentation for configuring/managing ZooKeeper with HBase
+ HBASE-1662 Tool to run major compaction on catalog regions when hbase is
+ shutdown
OPTIMIZATIONS
HBASE-1412 Change values for delete column and column family in KeyValue
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HConstants.java?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HConstants.java Thu Jul 16 05:09:53 2009
@@ -47,8 +47,9 @@
* Version 4 supports only one kind of bloom filter.
* Version 5 changes versions in catalog table regions.
* Version 6 enables blockcaching on catalog tables.
+ * Version 7 introduces hfile -- hbase 0.19 to 0.20..
*/
- public static final String FILE_SYSTEM_VERSION = "6";
+ public static final String FILE_SYSTEM_VERSION = "7";
// Configuration parameters
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java Thu Jul 16 05:09:53 2009
@@ -56,6 +56,7 @@
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.io.Reference.Range;
+import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
@@ -1674,14 +1675,14 @@
private byte [] stopRow;
RegionScanner(Scan scan) {
- if(Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) {
+ if (Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) {
this.stopRow = null;
} else {
this.stopRow = scan.getStopRow();
}
List<KeyValueScanner> scanners = new ArrayList<KeyValueScanner>();
- for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
+ for (Map.Entry<byte[], NavigableSet<byte[]>> entry :
scan.getFamilyMap().entrySet()) {
Store store = stores.get(entry.getKey());
scanners.add(store.getScanner(scan, entry.getValue()));
@@ -1703,21 +1704,20 @@
throws IOException {
// This method should probably be reorganized a bit... has gotten messy
KeyValue kv = this.storeHeap.peek();
- if(kv == null) {
+ if (kv == null) {
return false;
}
byte [] currentRow = kv.getRow();
// See if we passed stopRow
- if(stopRow != null &&
- comparator.compareRows(stopRow, 0, stopRow.length,
- currentRow, 0, currentRow.length)
- <= 0) {
+ if (stopRow != null &&
+ comparator.compareRows(stopRow, 0, stopRow.length,
+ currentRow, 0, currentRow.length) <= 0) {
return false;
}
this.storeHeap.next(results);
while(true) {
kv = this.storeHeap.peek();
- if(kv == null) {
+ if (kv == null) {
return false;
}
byte [] row = kv.getRow();
@@ -2348,4 +2348,106 @@
}
return heapSize;
}
-}
+
+ /*
+ * This method calls System.exit.
+ * @param message Message to print out. May be null.
+ */
+ private static void printUsageAndExit(final String message) {
+ if (message != null && message.length() > 0) System.out.println(message);
+ System.out.println("Usage: HRegion CATLALOG_TABLE_DIR [major_compact]");
+ System.out.println("Options:");
+ System.out.println(" major_compact Pass this option to major compact " +
+ "passed region.");
+ System.out.println("Default outputs scan of passed region.");
+ System.exit(1);
+ }
+
+ /*
+ * Process table.
+ * Do major compaction or list content.
+ * @param fs
+ * @param p
+ * @param log
+ * @param c
+ * @param majorCompact
+ * @throws IOException
+ */
+ private static void processTable(final FileSystem fs, final Path p,
+ final HLog log, final HBaseConfiguration c,
+ final boolean majorCompact)
+ throws IOException {
+ HRegion region = null;
+ String rootStr = Bytes.toString(HConstants.ROOT_TABLE_NAME);
+ String metaStr = Bytes.toString(HConstants.META_TABLE_NAME);
+ // Currently expects tables have one region only.
+ if (p.getName().startsWith(rootStr)) {
+ region = new HRegion(p, log, fs, c, HRegionInfo.ROOT_REGIONINFO, null);
+ } else if (p.getName().startsWith(metaStr)) {
+ region = new HRegion(p, log, fs, c, HRegionInfo.FIRST_META_REGIONINFO,
+ null);
+ } else {
+ throw new IOException("Not a known catalog table: " + p.toString());
+ }
+ try {
+ region.initialize(null, null);
+ if (majorCompact) {
+ region.compactStores(true);
+ } else {
+ // Default behavior
+ Scan scan = new Scan();
+ InternalScanner scanner = region.getScanner(scan);
+ try {
+ List<KeyValue> kvs = new ArrayList<KeyValue>();
+ boolean done = false;
+ do {
+ kvs.clear();
+ done = scanner.next(kvs);
+ if (kvs.size() > 0) LOG.info(kvs);
+ } while (done);
+ } finally {
+ scanner.close();
+ }
+ }
+ } finally {
+ region.close();
+ }
+ }
+
+ /**
+ * Facility for dumping and compacting catalog tables.
+ * Only does catalog tables since these are only tables we for sure know
+ * schema on. For usage run:
+ * <pre>
+ * ./bin/hbase org.apache.hadoop.hbase.regionserver.HRegion
+ * </pre>
+ * @param args
+ * @throws IOException
+ */
+ public static void main(String[] args) throws IOException {
+ if (args.length < 1) {
+ printUsageAndExit(null);
+ }
+ boolean majorCompact = false;
+ if (args.length > 1) {
+ if (!args[1].toLowerCase().startsWith("major")) {
+ printUsageAndExit("ERROR: Unrecognized option <" + args[1] + ">");
+ }
+ majorCompact = true;
+
+ }
+ Path tableDir = new Path(args[0]);
+ HBaseConfiguration c = new HBaseConfiguration();
+ FileSystem fs = FileSystem.get(c);
+ Path logdir = new Path(c.get("hbase.tmp.dir"),
+ "hlog" + tableDir.getName() + System.currentTimeMillis());
+ HLog log = new HLog(fs, logdir, c, null);
+ try {
+ processTable(fs, tableDir, log, c, majorCompact);
+ } finally {
+ log.close();
+ BlockCache bc = StoreFile.getBlockCache(c);
+ if (bc != null) bc.shutdown();
+ }
+ }
+}
\ No newline at end of file
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/FSUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/FSUtils.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/FSUtils.java Thu Jul 16 05:09:53 2009
@@ -30,8 +30,10 @@
import org.apache.hadoop.dfs.DistributedFileSystem;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -265,4 +267,128 @@
HRegion.getRegionDir(rootdir, HRegionInfo.ROOT_REGIONINFO);
return fs.exists(rootRegionDir);
}
+
+ /**
+ * Runs through the hbase rootdir and checks all stores have only
+ * one file in them -- that is, they've been major compacted. Looks
+ * at root and meta tables too.
+ * @param fs
+ * @param hbaseRootDir
+ * @return True if this hbase install is major compacted.
+ * @throws IOException
+ */
+ public static boolean isMajorCompacted(final FileSystem fs,
+ final Path hbaseRootDir)
+ throws IOException {
+ // Presumes any directory under hbase.rootdir is a table.
+ FileStatus [] directories = fs.listStatus(hbaseRootDir, new DirFilter(fs));
+ for (int i = 0; i < directories.length; i++) {
+ // Skip the .log directory. All others should be tables. Inside a table,
+ // there are compaction.dir directories to skip. Otherwise, all else
+ // should be regions. Then in each region, should only be family
+ // directories. Under each of these, should be one file only.
+ Path d = directories[i].getPath();
+ if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) continue;
+ FileStatus [] tablesubdirectories = fs.listStatus(d, new DirFilter(fs));
+ for (int j = 0; j < tablesubdirectories.length; j++) {
+ Path dd = tablesubdirectories[j].getPath();
+ if (dd.equals(HConstants.HREGION_COMPACTIONDIR_NAME)) continue;
+ // Else its a region name. Now look in region for families.
+ FileStatus [] familydirectories = fs.listStatus(dd, new DirFilter(fs));
+ for (int k = 0; k < familydirectories.length; k++) {
+ Path family = familydirectories[k].getPath();
+ // Now in family make sure only one file.
+ FileStatus [] familyStatus = fs.listStatus(family);
+ if (familyStatus.length > 1) {
+ LOG.debug(family.toString() + " has " + familyStatus.length +
+ " files.");
+ return false;
+ }
+ }
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Expects to find -ROOT- directory.
+ * @param fs
+ * @param hbaseRootDir
+ * @return True if this a pre020 layout.
+ * @throws IOException
+ */
+ public static boolean isPre020FileLayout(final FileSystem fs,
+ final Path hbaseRootDir)
+ throws IOException {
+ Path mapfiles = new Path(new Path(new Path(hbaseRootDir, "-ROOT-"),
+ "70236052"), "mapfiles");
+ return fs.exists(mapfiles);
+ }
+
+ /**
+ * Runs through the hbase rootdir and checks all stores have only
+ * one file in them -- that is, they've been major compacted. Looks
+ * at root and meta tables too. This version differs from
+ * {@link #isMajorCompacted(FileSystem, Path)} in that it expects a
+ * pre-0.20.0 hbase layout on the filesystem. Used migrating.
+ * @param fs
+ * @param hbaseRootDir
+ * @return True if this hbase install is major compacted.
+ * @throws IOException
+ */
+ public static boolean isMajorCompactedPre020(final FileSystem fs,
+ final Path hbaseRootDir)
+ throws IOException {
+ // Presumes any directory under hbase.rootdir is a table.
+ FileStatus [] directories = fs.listStatus(hbaseRootDir, new DirFilter(fs));
+ for (int i = 0; i < directories.length; i++) {
+ // Inside a table, there are compaction.dir directories to skip.
+ // Otherwise, all else should be regions. Then in each region, should
+ // only be family directories. Under each of these, should be a mapfile
+ // and info directory and in these only one file.
+ Path d = directories[i].getPath();
+ if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) continue;
+ FileStatus [] tablesubdirectories = fs.listStatus(d, new DirFilter(fs));
+ for (int j = 0; j < tablesubdirectories.length; j++) {
+ Path dd = tablesubdirectories[j].getPath();
+ if (dd.equals(HConstants.HREGION_COMPACTIONDIR_NAME)) continue;
+ // Else its a region name. Now look in region for families.
+ FileStatus [] familydirectories = fs.listStatus(dd, new DirFilter(fs));
+ for (int k = 0; k < familydirectories.length; k++) {
+ Path family = familydirectories[k].getPath();
+ // Now in family, there are 'mapfile' and 'info' subdirs. Just
+ // look in the 'mapfile' subdir.
+ FileStatus [] familyStatus =
+ fs.listStatus(new Path(family, "mapfiles"));
+ if (familyStatus.length > 1) {
+ LOG.debug(family.toString() + " has " + familyStatus.length +
+ " files.");
+ return false;
+ }
+ }
+ }
+ }
+ return true;
+ }
+
+ /**
+ * A {@link PathFilter} that returns directories.
+ */
+ public static class DirFilter implements PathFilter {
+ private final FileSystem fs;
+
+ public DirFilter(final FileSystem fs) {
+ this.fs = fs;
+ }
+
+ public boolean accept(Path p) {
+ boolean isdir = false;
+ try {
+ isdir = this.fs.getFileStatus(p).isDir();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return isdir;
+ }
+ }
}
\ No newline at end of file
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Merge.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Merge.java?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Merge.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Merge.java Thu Jul 16 05:09:53 2009
@@ -345,8 +345,8 @@
tableName = Bytes.toBytes(remainingArgs[0]);
isMetaTable = Bytes.compareTo(tableName, HConstants.META_TABLE_NAME) == 0;
- region1 = Bytes.toBytes(remainingArgs[1]);
- region2 = Bytes.toBytes(remainingArgs[2]);
+ region1 = Bytes.toBytesBinary(remainingArgs[1]);
+ region2 = Bytes.toBytesBinary(remainingArgs[2]);
int status = 0;
if (notInTable(tableName, region1) || notInTable(tableName, region2)) {
status = -1;
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java Thu Jul 16 05:09:53 2009
@@ -35,7 +35,6 @@
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.Tool;
@@ -86,7 +85,7 @@
private static final float HBASE_0_1_VERSION = 0.1f;
// Filesystem version we can migrate from
- private static final int PREVIOUS_VERSION = 4;
+ private static final int PREVIOUS_VERSION = 6;
private static final String MIGRATION_LINK =
" See http://wiki.apache.org/hadoop/Hbase/HowToMigrate for more information.";
@@ -185,14 +184,14 @@
if (version == HBASE_0_1_VERSION ||
Integer.valueOf(versionStr).intValue() < PREVIOUS_VERSION) {
String msg = "Cannot upgrade from " + versionStr + " to " +
- HConstants.FILE_SYSTEM_VERSION + " you must install hbase-0.2.x, run " +
+ HConstants.FILE_SYSTEM_VERSION + " you must install an earlier hbase, run " +
"the upgrade tool, reinstall this version and run this utility again." +
MIGRATION_LINK;
System.out.println(msg);
throw new IOException(msg);
}
- migrate4To6();
+ migrate6to7();
if (!readOnly) {
// Set file system version
@@ -209,17 +208,34 @@
}
}
- // Move the fileystem version from 4 to 6.
- // In here we rewrite the catalog table regions so they keep 10 versions
- // instead of 1.
- private void migrate4To6() throws IOException {
+ // Move the fileystem version from 6 to 7.
+ private void migrate6to7() throws IOException {
if (this.readOnly && this.migrationNeeded) {
return;
}
+ // Before we start, make sure all is major compacted.
+ Path hbaseRootDir = new Path(conf.get(HConstants.HBASE_DIR));
+ boolean pre020 = FSUtils.isPre020FileLayout(fs, hbaseRootDir);
+ if (pre020) {
+ if (!FSUtils.isMajorCompactedPre020(fs, hbaseRootDir)) {
+ String msg = "All tables must be major compacted before migration." +
+ MIGRATION_LINK;
+ System.out.println(msg);
+ throw new IOException(msg);
+ }
+ // TODO: Rewrite regions.
+ }
+ // TOOD: Verify all has been brought over from old to new layout.
final MetaUtils utils = new MetaUtils(this.conf);
try {
- // These two operations are effectively useless. -ROOT- is hardcode,
- // at least until hbase 0.20.0 when we store it out in ZK.
+ // TODO: Set the .META. and -ROOT- to flush at 16k? 32k?
+ // TODO: Enable block cache on all tables
+ // TODO: Rewrite MEMCACHE_FLUSHSIZE as MEMSTORE_FLUSHSIZE â name has changed.
+ // TODO: Remove tableindexer 'index' attribute index from TableDescriptor (See HBASE-1586)
+ // TODO: TODO: Move of in-memory parameter from table to column family (from HTD to HCD).
+ // TODO: Purge isInMemory, etc., methods from HTD as part of migration.
+ // TODO: Clean up old region log files (HBASE-698)
+
updateVersions(utils.getRootRegion().getRegionInfo());
enableBlockCache(utils.getRootRegion().getRegionInfo());
// Scan the root region
@@ -235,6 +251,20 @@
return true;
}
});
+ LOG.info("TODO: Note on make sure not using old hbase-default.xml");
+ /*
+ * hbase.master / hbase.master.hostname are obsolete, that's replaced by
+hbase.cluster.distributed. This config must be set to "true" to have a
+fully-distributed cluster and the server lines in zoo.cfg must not
+point to "localhost".
+
+The clients must have a valid zoo.cfg in their classpath since we
+don't provide the master address.
+
+hbase.master.dns.interface and hbase.master.dns.nameserver should be
+set to control the master's address (not mandatory).
+ */
+ LOG.info("TODO: Note on zookeeper config. before starting:");
} finally {
utils.shutdown();
}
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html Thu Jul 16 05:09:53 2009
@@ -174,51 +174,99 @@
</p>
<p>
A distributed HBase depends on a running ZooKeeper cluster.
-The ZooKeeper configuration file for HBase is stored at <code>${HBASE_HOME}/conf/zoo.cfg</code>.
-See the ZooKeeper <a href="http://hadoop.apache.org/zookeeper/docs/current/zookeeperStarted.html"> Getting Started Guide</a>
-for information about the format and options of that file. Specifically, look at the
-<a href="http://hadoop.apache.org/zookeeper/docs/current/zookeeperStarted.html#sc_RunningReplicatedZooKeeper">Running Replicated ZooKeeper</a> section.
-</p>
-
-
-<p>
-Though not recommended, it can be convenient having HBase continue to manage
-ZooKeeper even when in distributed mode (It can be good when testing or taking
-hbase for a testdrive). Change <code>${HBASE_HOME}/conf/zoo.cfg</code> and
-set the server.0 property to the IP of the node that will be running ZooKeeper
-(Leaving the default value of "localhost" will make it impossible to start HBase).
-<pre>
- ...
-server.0=example.org:2888:3888
-<blockquote>
-</pre>
-Then on the example.org server do the following <i>before</i> running HBase.
-<pre>
-${HBASE_HOME}/bin/hbase-daemon.sh start zookeeper
-</pre>
-</blockquote>
-<p>To stop ZooKeeper, after you've shut down hbase, do:
-<blockquote>
-<pre>
-${HBASE_HOME}/bin/hbase-daemon.sh stop zookeeper
-</pre>
-</blockquote>
-Be aware that this option is only recommanded for testing purposes as a failure
-on that node would render HBase <b>unusable</b>.
-</p>
-
-<p>
-To tell HBase to stop managing a ZooKeeper instance, after configuring
-<code>zoo.cfg</code> to point at the ZooKeeper Quorum you'd like HBase to
-use, in <code>${HBASE_HOME}/conf/hbase-env.sh</code>,
-set the following to tell HBase to STOP managing its instance of ZooKeeper.
-<blockquote>
-<pre>
- ...
-# Tell HBase whether it should manage it's own instance of Zookeeper or not.
-export HBASE_MANAGES_ZK=false
-</pre>
-</blockquote>
+HBase can manage a ZooKeeper cluster for you, or you can manage it on your own
+and point HBase to it.
+To toggle this option, use the <code>HBASE_MANAGES_ZK</code> variable in <code>
+${HBASE_HOME}/conf/hbase-env.sh</code>.
+This variable, which defaults to <code>true</code>, tells HBase whether to
+start/stop the ZooKeeper quorum servers alongside the rest of the servers.
+</p>
+<p>
+To point HBase at an existing ZooKeeper cluster, add your <code>zoo.cfg</code>
+to the <code>CLASSPATH</code>.
+HBase will see this file and use it to figure out where ZooKeeper is.
+Additionally set <code>HBASE_MANAGES_ZK</code> in <code> ${HBASE_HOME}/conf/hbase-env.sh</code>
+ to <code>false</code> so that HBase doesn't mess with your ZooKeeper setup:
+<pre>
+ ...
+ # Tell HBase whether it should manage it's own instance of Zookeeper or not.
+ export HBASE_MANAGES_ZK=false
+</pre>
+For more information about setting up a ZooKeeper cluster on your own, see
+the ZooKeeper <a href="http://hadoop.apache.org/zookeeper/docs/current/zookeeperStarted.html">Getting Started Guide</a>.
+HBase currently uses ZooKeeper version 3.2.0, so any cluster setup with a 3.x.x
+version of ZooKeeper should work.
+</p>
+<p>
+To have HBase manage the ZooKeeper cluster, you can use a <code>zoo.cfg</code>
+ file as above, or edit the options directly in the <code>${HBASE_HOME}/conf/hbase-site.xml</code>.
+Every option from the <code>zoo.cfg</code> has a corresponding property in the
+XML configuration file named <code>hbase.zookeeper.property.OPTION</code>.
+For example, the <code>clientPort</code> setting in ZooKeeper can be changed by
+setting the <code>hbase.zookeeper.property.clientPort</code> property.
+For the full list of available properties, see ZooKeeper's <code>zoo.cfg</code>.
+For the default values used by HBase, see <code>${HBASE_HOME}/conf/hbase-default.xml</code>.
+</p>
+<p>
+At minimum, you should set the list of servers that you want ZooKeeper to run
+on using the <code>hbase.zookeeper.quorum</code> property.
+This property defaults to <code>localhost</code> which is not suitable for a
+fully distributed HBase.
+It is recommended to run a ZooKeeper quorum of 5 or 7 machines, and give each
+server around 1GB to ensure that they don't swap.
+It is also recommended to run the ZooKeeper servers on separate machines from
+the Region Servers with their own disks.
+If this is not easily doable for you, choose 5 of your region servers to run the
+ZooKeeper servers on.
+</p>
+<p>
+As an example, to have HBase manage a ZooKeeper quorum on nodes
+rs{1,2,3,4,5}.example.com, bound to port 2222 (the default is 2181), use:
+<pre>
+ ${HBASE_HOME}/conf/hbase-env.sh:
+
+ ...
+ # Tell HBase whether it should manage it's own instance of Zookeeper or not.
+ export HBASE_MANAGES_ZK=true
+
+ ${HBASE_HOME}/conf/hbase-site.xml:
+
+ <configuration>
+ ...
+ <property>
+ <name>hbase.zookeeper.property.clientPort</name>
+ <value>2222</value>
+ <description>Property from ZooKeeper's config zoo.cfg.
+ The port at which the clients will connect.
+ </description>
+ </property>
+ ...
+ <property>
+ <name>hbase.zookeeper.quorum</name>
+ <value>rs1.example.com,rs2.example.com,rs3.example.com,rs4.example.com,rs5.example.com</value>
+ <description>Comma separated list of servers in the ZooKeeper Quorum.
+ For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
+ By default this is set to localhost for local and pseudo-distributed modes
+ of operation. For a fully-distributed setup, this should be set to a full
+ list of ZooKeeper quorum servers. If HBASE_MANAGES_ZK is set in hbase-env.sh
+ this is the list of servers which we will start/stop ZooKeeper on.
+ </description>
+ </property>
+ ...
+ </configuration>
+</pre>
+</p>
+<p>
+When HBase manages ZooKeeper, it will start/stop the ZooKeeper servers as a part
+of the regular start/stop scripts. If you would like to run it yourself, you can
+do:
+<pre>
+ ${HBASE_HOME}/bin/hbase-daemons.sh {start,stop} zookeeper
+</pre>
+Note that you can use HBase in this manner to spin up a ZooKeeper cluster,
+unrelated to HBase. Just make sure to set <code>HBASE_MANAGES_ZK</code> to
+<code>false</code> if you want it to stay up so that when HBase shuts down it
+doesn't take ZooKeeper with it.
</p>
<p>Of note, if you have made <i>HDFS client configuration</i> on your hadoop cluster, HBase will not
Added: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/data/hbase-0.19-two-small-tables.zip
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/data/hbase-0.19-two-small-tables.zip?rev=794530&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/data/hbase-0.19-two-small-tables.zip
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java Thu Jul 16 05:09:53 2009
@@ -53,6 +53,30 @@
return cols;
}
+ public void testScanSameTimestamp() throws IOException {
+ // returns only 1 of these 2 even though same timestamp
+ KeyValue [] kvs = new KeyValue[] {
+ KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
+ };
+ KeyValueScanner [] scanners = new KeyValueScanner[] {
+ new KeyValueScanFixture(KeyValue.COMPARATOR,
+ kvs)
+ };
+
+ Scan scanSpec = new Scan(Bytes.toBytes("R1"));
+ // this only uses maxVersions (default=1) and TimeRange (default=all)
+ StoreScanner scan =
+ new StoreScanner(scanSpec, CF, Long.MAX_VALUE,
+ KeyValue.COMPARATOR, getCols("a"),
+ scanners);
+
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ assertEquals(true, scan.next(results));
+ assertEquals(1, results.size());
+ assertEquals(kvs[0], results.get(0));
+ }
+
/**
* Test test shows exactly how the matcher's return codes confuses the StoreScanner
* and prevent it from doing the right thing. Seeking once, then nexting twice
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/MigrationTest.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/MigrationTest.java?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/MigrationTest.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/MigrationTest.java Thu Jul 16 05:09:53 2009
@@ -20,14 +20,13 @@
package org.apache.hadoop.hbase.util;
-import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.InputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -42,30 +41,32 @@
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.ResultScanner;
/**
- * Runs migration of filesystem from hbase 0.x to 0.x
+ * Runs migration of filesystem from hbase 0.19 to hbase 0.20.
+ * Not part of general test suite because takes time.
*/
public class MigrationTest extends HBaseTestCase {
private static final Log LOG = LogFactory.getLog(MigrationTest.class);
-
- // This is the name of the table that is in the data file.
- private static final String TABLENAME = "TestUpgrade";
-
- // The table has two columns
- private static final byte [][] TABLENAME_COLUMNS =
- {Bytes.toBytes("column_a:"), Bytes.toBytes("column_b:")};
// Expected count of rows in migrated table.
- private static final int EXPECTED_COUNT = 17576;
+ private static final int EXPECTED_COUNT = 3;
/**
- * Test migration. To be used in future migrations
+ * Test migration.
* @throws IOException
*/
- public void testUpgrade() throws IOException {
+ public void testMigration() throws IOException {
+ Path rootdir = getUnitTestdir(getName());
+ FileSystem fs = FileSystem.get(this.conf);
+ Path hbasedir = loadTestData(fs, rootdir);
+ assertTrue(fs.exists(hbasedir));
+ listPaths(fs, hbasedir, -1);
+ Migrate migrator = new Migrate(this.conf);
+ Path qualified = fs.makeQualified(hbasedir);
+ String uri = qualified.toString();
+ this.conf.set("hbase.rootdir", uri);
+ migrator.run(new String [] {"upgrade"});
}
/*
@@ -74,24 +75,18 @@
* @param rootDir
* @throws IOException
*/
- private void loadTestData(final FileSystem dfs, final Path rootDir)
+ private Path loadTestData(final FileSystem dfs, final Path rootDir)
throws IOException {
- FileSystem localfs = FileSystem.getLocal(conf);
- // Get path for zip file. If running this test in eclipse, define
- // the system property src.testdata for your test run.
- String srcTestdata = System.getProperty("src.testdata");
- if (srcTestdata == null) {
- throw new NullPointerException("Define src.test system property");
- }
- Path data = new Path(srcTestdata, "HADOOP-2478-testdata-v0.1.zip");
- if (!localfs.exists(data)) {
- throw new FileNotFoundException(data.toString());
- }
- FSDataInputStream hs = localfs.open(data);
- ZipInputStream zip = new ZipInputStream(hs);
- unzip(zip, dfs, rootDir);
- zip.close();
- hs.close();
+ String hbasedir = "hbase-0.19-two-small-tables";
+ InputStream is = this.getClass().getClassLoader().
+ getResourceAsStream("data/" + hbasedir + ".zip");
+ ZipInputStream zip = new ZipInputStream(is);
+ try {
+ unzip(zip, dfs, rootDir);
+ } finally {
+ zip.close();
+ }
+ return new Path(rootDir, hbasedir);
}
/*
@@ -116,6 +111,7 @@
assertTrue(hb.isMasterRunning());
HTableDescriptor [] tables = hb.listTables();
boolean foundTable = false;
+ /*
for (int i = 0; i < tables.length; i++) {
if (Bytes.equals(Bytes.toBytes(TABLENAME), tables[i].getName())) {
foundTable = true;
@@ -148,6 +144,8 @@
} finally {
s.close();
}
+
+ */
} finally {
HConnectionManager.deleteConnectionInfo(conf, false);
cluster.shutdown();
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java Thu Jul 16 05:09:53 2009
@@ -25,7 +25,6 @@
import java.util.Properties;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.HConstants;
import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/master.jsp
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/master.jsp?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/master.jsp (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/master.jsp Thu Jul 16 05:09:53 2009
@@ -28,7 +28,6 @@
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="Content-Type" content="text/html;charset=UTF-8"/>
- <meta http-equiv="refresh" content="300"/>
<title>HBase Master: <%= master.getMasterAddress().getHostname()%>:<%= master.getMasterAddress().getPort() %></title>
<link rel="stylesheet" type="text/css" href="/static/hbase.css" />
<link rel="stylesheet" type="text/css" href="/static/jquery.treeview.css" />
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/table.jsp
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/table.jsp?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/table.jsp (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/table.jsp Thu Jul 16 05:09:53 2009
@@ -67,13 +67,12 @@
%> Compact request accepted. <%
}
%>
-<p>This page will refresh in 5 seconds.
+<p>Reload.
</body>
<%
} else {
%>
<head><meta http-equiv="Content-Type" content="text/html;charset=UTF-8"/>
- <meta http-equiv="refresh" content="30"/>
<title>Table: <%= tableName %></title>
<link rel="stylesheet" type="text/css" href="/static/hbase.css" />
</head>
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/regionserver/regionserver.jsp
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/regionserver/regionserver.jsp?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/regionserver/regionserver.jsp (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/regionserver/regionserver.jsp Thu Jul 16 05:09:53 2009
@@ -20,7 +20,6 @@
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="Content-Type" content="text/html;charset=UTF-8"/>
- <meta http-equiv="refresh" content="300"/>
<title>HBase Region Server: <%= serverInfo.getServerAddress().getHostname() %>:<%= serverInfo.getServerAddress().getPort() %></title>
<link rel="stylesheet" type="text/css" href="/static/hbase.css" />
</head>