You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/10/11 03:59:52 UTC
svn commit: r1181336 [2/3] - in /hbase/branches/0.89: ./ bin/ conf/
src/main/java/org/apache/hadoop/hbase/io/
src/main/java/org/apache/hadoop/hbase/master/
src/main/java/org/apache/hadoop/hbase/regionserver/
src/main/java/org/apache/hadoop/hbase/region...
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java Tue Oct 11 01:59:50 2011
@@ -36,7 +36,7 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.NavigableMap;
+import java.util.NavigableSet;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.TreeSet;
@@ -53,6 +53,7 @@ import java.util.concurrent.atomic.Atomi
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
@@ -61,6 +62,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.Syncable;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
@@ -119,9 +121,17 @@ import com.google.common.util.concurrent
*/
public class HLog implements Syncable {
static final Log LOG = LogFactory.getLog(HLog.class);
- private static final String HLOG_DATFILE = "hlog.dat.";
public static final byte [] METAFAMILY = Bytes.toBytes("METAFAMILY");
static final byte [] METAROW = Bytes.toBytes("METAROW");
+
+ /*
+ * Name of directory that holds recovered edits written by the wal log
+ * splitting code, one per region
+ */
+ private static final String RECOVERED_EDITS_DIR = "recovered.edits";
+ private static final Pattern EDITFILES_NAME_PATTERN =
+ Pattern.compile("-?[0-9]+");
+
private final FileSystem fs;
private final Path dir;
private final Configuration conf;
@@ -144,11 +154,6 @@ public class HLog implements Syncable {
private Method getNumCurrentReplicas; // refers to DFSOutputStream.getNumCurrentReplicas
final static Object [] NO_ARGS = new Object []{};
- /** Name of file that holds recovered edits written by the wal log splitting
- * code, one per region
- */
- public static final String RECOVERED_EDITS = "recovered.edits";
-
// used to indirectly tell syncFs to force the sync
private boolean forceSync = false;
@@ -1459,7 +1464,7 @@ public class HLog implements Syncable {
NamingThreadFactory f = new NamingThreadFactory(
"SplitWriter-%1$d", Executors.defaultThreadFactory());
ThreadPoolExecutor threadPool = (ThreadPoolExecutor)Executors.newFixedThreadPool(logWriterThreads, f);
- for (final byte[] region : splitLogsMap.keySet()) {
+ for (final byte [] region : splitLogsMap.keySet()) {
Callable splitter = createNewSplitter(rootDir, logWriters, splitLogsMap, region, fs, conf);
writeFutureResult.put(region, threadPool.submit(splitter));
}
@@ -1579,17 +1584,19 @@ public class HLog implements Syncable {
WriterAndPath wap = logWriters.get(region);
for (Entry logEntry: entries) {
if (wap == null) {
- Path logFile = getRegionLogPath(logEntry, rootDir);
- if (fs.exists(logFile)) {
- LOG.warn("Found existing old hlog file. It could be the result of a previous" +
- "failed split attempt. Deleting " + logFile +
- ", length=" + fs.getFileStatus(logFile).getLen());
- fs.delete(logFile, false);
+ Path regionedits = getRegionSplitEditsPath(fs, logEntry, rootDir);
+ if (fs.exists(regionedits)) {
+ LOG.warn("Found existing old edits file. It could be the " +
+ "result of a previous failed split attempt. Deleting " +
+ regionedits + ", length=" + fs.getFileStatus(regionedits).getLen());
+ if (!fs.delete(regionedits, false)) {
+ LOG.warn("Failed delete of old " + regionedits);
+ }
}
- Writer w = createWriter(fs, logFile, conf);
- wap = new WriterAndPath(logFile, w);
+ Writer w = createWriter(fs, regionedits, conf);
+ wap = new WriterAndPath(regionedits, w);
logWriters.put(region, wap);
- LOG.debug("Creating writer path=" + logFile +
+ LOG.debug("Creating writer path=" + regionedits +
" region=" + Bytes.toStringBinary(region));
}
wap.w.append(logEntry);
@@ -1643,14 +1650,101 @@ public class HLog implements Syncable {
}
}
- private static Path getRegionLogPath(Entry logEntry, Path rootDir) {
- Path tableDir =
- HTableDescriptor.getTableDir(rootDir, logEntry.getKey().getTablename());
- Path regionDir =
- HRegion.getRegionDir(tableDir, HRegionInfo.encodeRegionName(logEntry.getKey().getRegionName()));
- return new Path(regionDir, RECOVERED_EDITS);
+ /*
+ * Path to a file under RECOVERED_EDITS_DIR directory of the region found in
+ * <code>logEntry</code> named for the sequenceid in the passed
+ * <code>logEntry</code>: e.g. /hbase/some_table/2323432434/recovered.edits/2332.
+ * This method also ensures existence of RECOVERED_EDITS_DIR under the region
+ * creating it if necessary.
+ * @param fs
+ * @param logEntry
+ * @param rootDir HBase root dir.
+ * @return Path to file into which to dump split log edits.
+ * @throws IOException
+ */
+ private static Path getRegionSplitEditsPath(final FileSystem fs,
+ final Entry logEntry, final Path rootDir)
+ throws IOException {
+ Path tableDir = HTableDescriptor.getTableDir(rootDir,
+ logEntry.getKey().getTablename());
+ Path regiondir = HRegion.getRegionDir(tableDir,
+ HRegionInfo.encodeRegionName(logEntry.getKey().getRegionName()));
+ Path dir = getRegionDirRecoveredEditsDir(regiondir);
+ if (!fs.exists(dir)) {
+ if (!fs.mkdirs(dir)) LOG.warn("mkdir failed on " + dir);
+ }
+ return new Path(dir,
+ formatRecoveredEditsFileName(logEntry.getKey().getLogSeqNum()));
}
+ static String formatRecoveredEditsFileName(final long seqid) {
+ return String.format("%019d", seqid);
+ }
+
+
+ /**
+ * Returns sorted set of edit files made by wal-log splitter.
+ * @param fs
+ * @param regiondir
+ * @return Files in passed <code>regiondir</code> as a sorted set.
+ * @throws IOException
+ */
+ public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,
+ final Path regiondir)
+ throws IOException {
+ Path editsdir = getRegionDirRecoveredEditsDir(regiondir);
+ FileStatus [] files = fs.listStatus(editsdir, new PathFilter () {
+ @Override
+ public boolean accept(Path p) {
+ boolean result = false;
+ try {
+ // Return files and only files that match the editfile names pattern.
+ // There can be other files in this directory other than edit files.
+ // In particular, on error, we'll move aside the bad edit file giving
+ // it a timestamp suffix. See moveAsideBadEditsFile.
+ Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());
+ result = fs.isFile(p) && m.matches();
+ } catch (IOException e) {
+ LOG.warn("Failed isFile check on " + p);
+ }
+ return result;
+ }
+ });
+ NavigableSet<Path> filesSorted = new TreeSet<Path>();
+ if (files == null) return filesSorted;
+ for (FileStatus status: files) {
+ filesSorted.add(status.getPath());
+ }
+ return filesSorted;
+ }
+
+ /**
+ * Move aside a bad edits file.
+ * @param fs
+ * @param edits Edits file to move aside.
+ * @return The name of the moved aside file.
+ * @throws IOException
+ */
+ public static Path moveAsideBadEditsFile(final FileSystem fs,
+ final Path edits)
+ throws IOException {
+ Path moveAsideName = new Path(edits.getParent(), edits.getName() + "." +
+ System.currentTimeMillis());
+ if (!fs.rename(edits, moveAsideName)) {
+ LOG.warn("Rename failed from " + edits + " to " + moveAsideName);
+ }
+ return moveAsideName;
+ }
+
+ /**
+ * @param regiondir This regions directory in the filesystem.
+ * @return The directory that holds recovered edits files for the region
+ * <code>regiondir</code>
+ */
+ public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {
+ return new Path(regiondir, RECOVERED_EDITS_DIR);
+ }
+
/**
*
* @param visitor
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/rest/Main.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/rest/Main.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/rest/Main.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/rest/Main.java Tue Oct 11 01:59:50 2011
@@ -22,9 +22,16 @@ package org.apache.hadoop.hbase.rest;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.util.Arrays;
+import java.util.List;
+
import org.mortbay.jetty.Server;
import org.mortbay.jetty.servlet.Context;
import org.mortbay.jetty.servlet.ServletHolder;
@@ -40,17 +47,43 @@ import com.sun.jersey.spi.container.serv
* </ul>
*/
public class Main implements Constants {
+ private static final String DEFAULT_LISTEN_PORT = "8080";
- public static void main(String[] args) throws Exception {
- // process command line
+ private static void printUsageAndExit(Options options, int exitCode) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("REST", null, options,
+ "To start the REST server run 'bin/hbase-daemon.sh start rest'\n" +
+ "To shutdown the REST server run 'bin/hbase-daemon.sh stop rest' or" +
+ " send a kill signal to the rest server pid",
+ true);
+ System.exit(exitCode);
+ }
+ public static void main(String[] args) throws Exception {
+ Log LOG = LogFactory.getLog("RESTServer");
Options options = new Options();
- options.addOption("p", "port", true, "service port");
+ options.addOption("p", "port", true, "Port to bind to [default:" +
+ DEFAULT_LISTEN_PORT + "]");
CommandLineParser parser = new PosixParser();
CommandLine cmd = parser.parse(options, args);
- int port = 8080;
- if (cmd.hasOption("p")) {
- port = Integer.valueOf(cmd.getOptionValue("p"));
+ /**
+ * This is so complicated to please both bin/hbase and bin/hbase-daemon.
+ * hbase-daemon provides "start" and "stop" arguments
+ * hbase should print the help if no argument is provided
+ */
+ List<String> commandLine = Arrays.asList(args);
+ boolean stop = commandLine.contains("stop");
+ boolean start = commandLine.contains("start");
+ if (cmd.hasOption("help") || !start || stop) {
+ printUsageAndExit(options, 1);
+ }
+ // Get port to bind to
+ int port = 0;
+ try {
+ port = Integer.parseInt(cmd.getOptionValue("port", DEFAULT_LISTEN_PORT));
+ } catch (NumberFormatException e) {
+ LOG.error("Could not parse the value provided for the port option", e);
+ printUsageAndExit(options, -1);
}
// set up the Jersey servlet container for Jetty
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/Bytes.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/Bytes.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/Bytes.java Tue Oct 11 01:59:50 2011
@@ -843,27 +843,27 @@ public class Bytes {
/**
* Lexographically compare two arrays.
*
- * @param b1 left operand
- * @param b2 right operand
- * @param s1 Where to start comparing in the left buffer
- * @param s2 Where to start comparing in the right buffer
- * @param l1 How much to compare from the left buffer
- * @param l2 How much to compare from the right buffer
+ * @param buffer1 left operand
+ * @param buffer2 right operand
+ * @param offset1 Where to start comparing in the left buffer
+ * @param offset2 Where to start comparing in the right buffer
+ * @param length1 How much to compare from the left buffer
+ * @param length2 How much to compare from the right buffer
* @return 0 if equal, < 0 if left is less than right, etc.
*/
- public static int compareTo(byte[] b1, int s1, int l1,
- byte[] b2, int s2, int l2) {
+ public static int compareTo(byte[] buffer1, int offset1, int length1,
+ byte[] buffer2, int offset2, int length2) {
// Bring WritableComparator code local
- int end1 = s1 + l1;
- int end2 = s2 + l2;
- for (int i = s1, j = s2; i < end1 && j < end2; i++, j++) {
- int a = (b1[i] & 0xff);
- int b = (b2[j] & 0xff);
+ int end1 = offset1 + length1;
+ int end2 = offset2 + length2;
+ for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) {
+ int a = (buffer1[i] & 0xff);
+ int b = (buffer2[j] & 0xff);
if (a != b) {
return a - b;
}
}
- return l1 - l2;
+ return length1 - length2;
}
/**
Copied: hbase/branches/0.89/src/main/javadoc/org/apache/hadoop/hbase/replication/package.html (from r1176177, hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/replication/package.html)
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/javadoc/org/apache/hadoop/hbase/replication/package.html?p2=hbase/branches/0.89/src/main/javadoc/org/apache/hadoop/hbase/replication/package.html&p1=hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/replication/package.html&r1=1176177&r2=1181336&rev=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/replication/package.html (original)
+++ hbase/branches/0.89/src/main/javadoc/org/apache/hadoop/hbase/replication/package.html Tue Oct 11 01:59:50 2011
@@ -38,7 +38,7 @@ This package provides replication betwee
<h2>Status</h2>
</a>
<p>
-This package is alpha quality software and is only meant to be a base
+This package is experimental quality software and is only meant to be a base
for future developments. The current implementation offers the following
features:
@@ -66,8 +66,8 @@ Before trying out replication, make sure
other one on the slave cluster. That also includes the
Zookeeper clusters.</li>
<li>Both clusters should have the same HBase and Hadoop major revision.
- For example, having 0.21.1 on the master and 0.21.0 on the slave is
- correct but not 0.21.1 and 0.22.0.</li>
+ For example, having 0.90.1 on the master and 0.90.0 on the slave is
+ correct but not 0.90.1 and 0.89.20100725.</li>
<li>Every table that contains families that are scoped for replication
should exist on every cluster with the exact same name, same for those
replicated families.</li>
@@ -86,13 +86,13 @@ to another. This must be done with both
the following configurations:
<pre>
<property>
- <name>hbase.replication.enabled</name>
+ <name>hbase.replication</name>
<value>true</value>
</property></pre>
</li>
<li>Run the following command on any cluster:
<pre>
-$HBASE_HOME/bin/hbase org.jruby.Main $HBASE_HOME/replication/bin/add_peer.tb</pre>
+$HBASE_HOME/bin/hbase org.jruby.Main $HBASE_HOME/bin/replication/add_peer.tb</pre>
This will show you the help to setup the replication stream between
both clusters. If both clusters use the same Zookeeper cluster, you have
to use a different <b>zookeeper.znode.parent</b> since they can't
Modified: hbase/branches/0.89/src/main/javadoc/overview.html
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/javadoc/overview.html?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/javadoc/overview.html (original)
+++ hbase/branches/0.89/src/main/javadoc/overview.html Tue Oct 11 01:59:50 2011
@@ -56,11 +56,13 @@
<li>Java 1.6.x, preferably from <a href="http://www.java.com/download/">Sun</a>. Use the latest version available except u18 (u19 is fine).</li>
<li>This version of HBase will only run on <a href="http://hadoop.apache.org/common/releases.html">Hadoop 0.20.x</a>.
HBase will lose data unless it is running on an HDFS that has a durable sync operation.
- Currently only the <a href="http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-append/">0.20-append</a>
+ Currently only the <a href="http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-append/">branch-0.20-append</a>
branch has this attribute. No official releases have been made from this branch as of this writing
so you will have to build your own Hadoop from the tip of this branch
- (or install Cloudera's <a href="http://archive.cloudera.com/docs/">CDH3b2</a>
- when its available; it will have a durable sync).</li>
+ (or install Cloudera's <a href="http://archive.cloudera.com/docs/">CDH3</a> (as of this writing, it is in beta); it has the
+ 0.20-append patches needed to add a durable sync).
+ See <a href="http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-append/CHANGES.txt">CHANGES.txt</a>
+ in branch-0.20.-append to see list of patches involved.</li>
<li>
<em>ssh</em> must be installed and <em>sshd</em> must be running to use Hadoop's scripts to manage remote Hadoop daemons.
You must be able to ssh to all nodes, including your local node, using passwordless login
@@ -172,7 +174,7 @@ it should run with one replica only (rec
<property>
<name>dfs.replication</name>
<value>1</value>
- <description>The replication count for HLog & HFile storage. Should not be greater than HDFS datanode count.
+ <description>The replication count for HLog & HFile storage. Should not be greater than HDFS datanode count.
</description>
</property>
...
@@ -410,13 +412,21 @@ the HBase version. It does not change yo
<h2><a name="related" >Related Documentation</a></h2>
<ul>
<li><a href="http://hbase.org">HBase Home Page</a>
+ </li>
<li><a href="http://wiki.apache.org/hadoop/Hbase">HBase Wiki</a>
+ </li>
<li><a href="http://hadoop.apache.org/">Hadoop Home Page</a>
+ </li>
<li><a href="http://wiki.apache.org/hadoop/Hbase/MultipleMasters">Setting up Multiple HBase Masters</a>
+ </li>
<li><a href="http://wiki.apache.org/hadoop/Hbase/RollingRestart">Rolling Upgrades</a>
+ </li>
<li><a href="org/apache/hadoop/hbase/client/transactional/package-summary.html#package_description">Transactional HBase</a>
+ </li>
<li><a href="org/apache/hadoop/hbase/client/tableindexed/package-summary.html">Table Indexed HBase</a>
+ </li>
<li><a href="org/apache/hadoop/hbase/stargate/package-summary.html#package_description">Stargate</a> -- a RESTful Web service front end for HBase.
+ </li>
</ul>
</body>
Modified: hbase/branches/0.89/src/main/ruby/shell.rb
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/ruby/shell.rb?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/ruby/shell.rb (original)
+++ hbase/branches/0.89/src/main/ruby/shell.rb Tue Oct 11 01:59:50 2011
@@ -143,8 +143,6 @@ module Shell
puts
puts '-' * 80
puts
- puts "Here is the list of groups with their commands:"
- puts
::Shell.command_groups.each do |name, group|
puts " " + group[:full_name] + ": "
puts " group name: " + name
@@ -162,39 +160,33 @@ module Shell
end
def help_header
- return "There are few groups of commands in HBase\n\n" +
- "Use help 'group_name' (e.g. help 'general') to get help on all commands in a group\n" +
- "Use help 'command' (e.g. help 'get') to get help on a specific command"
+ return "Enter, help 'COMMAND_GROUP', (e.g. help 'general') to get help on all commands in a group\n" +
+ "Enter, help 'COMMAND', (e.g. help 'get') to get help on a specific command"
end
def help_footer
- puts "GENERAL NOTES:"
puts <<-HERE
- Quote all names in the hbase shell such as table and column names. Don't
- forget commas delimit command parameters. Type <RETURN> after entering a
- command to run it. Dictionaries of configuration used in the creation
- and alteration of tables are ruby Hashes. They look like this:
-
- {'key1' => 'value1', 'key2' => 'value2', ...}
-
- They are opened and closed with curley-braces. Key/values are delimited
- by the '=>' character combination. Usually keys are predefined constants
- such as NAME, VERSIONS, COMPRESSION, etc. Constants do not need to be
- quoted. Type 'Object.constants' to see a (messy) list of all constants
- in the environment.
-
- In case you are using binary keys or values and need to enter them into
- the shell then use double-quotes to make use of hexadecimal for example:
-
- hbase> get 't1', "key\\x03\\x3f\\xcd"
- hbase> get 't1', "key\\003\\023\\011"
- hbase> put 't1', "test\\xef\\xff", 'f1:', "\\x01\\x33\\x40"
-
- Using the double-quote notation you can directly use the values output by
- the shell for example during a "scan" call.
+Quote all names in HBase shell such as table and column names. Commas delimit
+command parameters. Type <RETURN> after entering a command to run it.
+Dictionaries of configuration used in the creation and alteration of tables are
+Ruby Hashes. They look like this:
+
+ {'key1' => 'value1', 'key2' => 'value2', ...}
+
+and are opened and closed with curley-braces. Key/values are delimited by the
+'=>' character combination. Usually keys are predefined constants such as
+NAME, VERSIONS, COMPRESSION, etc. Constants do not need to be quoted. Type
+'Object.constants' to see a (messy) list of all constants in the environment.
+
+If you are using binary keys or values and need to enter them in the shell, use
+double-quote'd hexadecimal representation. For example:
+
+ hbase> get 't1', "key\\x03\\x3f\\xcd"
+ hbase> get 't1', "key\\003\\023\\011"
+ hbase> put 't1', "test\\xef\\xff", 'f1:', "\\x01\\x33\\x40"
- This HBase shell is the JRuby IRB with the above HBase-specific commands
- added. For more on the HBase Shell, see http://wiki.apache.org/hadoop/Hbase/Shell
+The HBase shell is the (J)Ruby IRB with the above HBase-specific commands added.
+For more on the HBase Shell, see http://wiki.apache.org/hadoop/Hbase/Shell
HERE
end
end
Added: hbase/branches/0.89/src/site/resources/images/architecture.gif
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/site/resources/images/architecture.gif?rev=1181336&view=auto
==============================================================================
Files hbase/branches/0.89/src/site/resources/images/architecture.gif (added) and hbase/branches/0.89/src/site/resources/images/architecture.gif Tue Oct 11 01:59:50 2011 differ
Added: hbase/branches/0.89/src/site/resources/images/asf_logo_wide.png
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/site/resources/images/asf_logo_wide.png?rev=1181336&view=auto
==============================================================================
Files hbase/branches/0.89/src/site/resources/images/asf_logo_wide.png (added) and hbase/branches/0.89/src/site/resources/images/asf_logo_wide.png Tue Oct 11 01:59:50 2011 differ
Added: hbase/branches/0.89/src/site/resources/images/favicon.ico
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/site/resources/images/favicon.ico?rev=1181336&view=auto
==============================================================================
Files hbase/branches/0.89/src/site/resources/images/favicon.ico (added) and hbase/branches/0.89/src/site/resources/images/favicon.ico Tue Oct 11 01:59:50 2011 differ
Added: hbase/branches/0.89/src/site/resources/images/hadoop-logo.jpg
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/site/resources/images/hadoop-logo.jpg?rev=1181336&view=auto
==============================================================================
Files hbase/branches/0.89/src/site/resources/images/hadoop-logo.jpg (added) and hbase/branches/0.89/src/site/resources/images/hadoop-logo.jpg Tue Oct 11 01:59:50 2011 differ
Added: hbase/branches/0.89/src/site/resources/images/hbase_logo_med.gif
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/site/resources/images/hbase_logo_med.gif?rev=1181336&view=auto
==============================================================================
Files hbase/branches/0.89/src/site/resources/images/hbase_logo_med.gif (added) and hbase/branches/0.89/src/site/resources/images/hbase_logo_med.gif Tue Oct 11 01:59:50 2011 differ
Added: hbase/branches/0.89/src/site/resources/images/hbase_small.gif
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/site/resources/images/hbase_small.gif?rev=1181336&view=auto
==============================================================================
Files hbase/branches/0.89/src/site/resources/images/hbase_small.gif (added) and hbase/branches/0.89/src/site/resources/images/hbase_small.gif Tue Oct 11 01:59:50 2011 differ
Added: hbase/branches/0.89/src/site/resources/images/replication_overview.png
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/site/resources/images/replication_overview.png?rev=1181336&view=auto
==============================================================================
Files hbase/branches/0.89/src/site/resources/images/replication_overview.png (added) and hbase/branches/0.89/src/site/resources/images/replication_overview.png Tue Oct 11 01:59:50 2011 differ
Modified: hbase/branches/0.89/src/site/site.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/site/site.xml?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/site/site.xml (original)
+++ hbase/branches/0.89/src/site/site.xml Tue Oct 11 01:59:50 2011
@@ -35,6 +35,8 @@
<item name="Bulk Loads" href="bulk-loads.html" />
<item name="Metrics" href="metrics.html" />
<item name="HBase on Windows" href="cygwin.html" />
+ <item name="Cluster replication" href="replication.html" />
+ <item name="Pseudo-Distributed HBase" href="pseudo-distributed.html" />
</menu>
</body>
<skin>
Added: hbase/branches/0.89/src/site/xdoc/pseudo-distributed.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/site/xdoc/pseudo-distributed.xml?rev=1181336&view=auto
==============================================================================
--- hbase/branches/0.89/src/site/xdoc/pseudo-distributed.xml (added)
+++ hbase/branches/0.89/src/site/xdoc/pseudo-distributed.xml Tue Oct 11 01:59:50 2011
@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright 2010 The Apache Software Foundation
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN"
+ "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document xmlns="http://maven.apache.org/XDOC/2.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/XDOC/2.0 http://maven.apache.org/xsd/xdoc-2.0.xsd">
+ <properties>
+ <title>
+Running HBase in pseudo-distributed mode
+ </title>
+ </properties>
+
+ <body>
+ <p>This document augments what is described in the HBase 'Getting Started' in the
+ <a href="http://hbase.apache.org/docs/current/api/overview-summary.html#distributed">Distributed Operation: Pseudo- and Fully-distributed modes</a> section.
+ In particular it describes scripts that allow you start extra masters and regionservers when running in pseudo-distributed mode.
+ </p>
+
+ <ol><li>Copy the psuedo-distributed suggested configuration file (feel free to take a peek and understand what it's doing)
+ <source>% cp conf/hbase-site.xml{.psuedo-distributed.template,}</source>
+ </li>
+ <li>(Optional) Start up <a href="http://hadoop.apache.org/common/docs/r0.20.2/quickstart.html#PseudoDistributed">Pseudo-distributed HDFS</a>.
+ <ol><li>If you do, go to conf/hbase-site.xml. Uncomment the 'hbase.rootdir' property.
+ </li>
+ <li>Additionally, if you want to test HBase with high data durability enabled, also uncomment the 'dfs.support.append' property.
+ </li>
+ </ol>
+ </li>
+<li>Start up the initial HBase cluster
+ <source>% bin/start-hbase.sh</source>
+ <ol> <li>To start up an extra backup master(s) on the same server run
+ <source>% bin/local-master-backup.sh start 1</source>
+ Here the '1' means use ports 60001 & 60011, and this backup master's logfile will be at <i>logs/hbase-${USER}-1-master-${HOSTNAME}.log</i>.
+ To startup multiple backup masters run <source>% bin/local-master-backup.sh start 2 3</source> You can start up to 9 backup masters (10 total).
+ </li>
+ <li>To start up more regionservers
+ <source>% bin/local-regionservers.sh start 1</source>
+ where '1' means use ports 60201 & 60301 and its logfile will be at <i>logs/hbase-${USER}-1-regionserver-${HOSTNAME}.log</i>.
+ To add 4 more regionservers in addition to the one you just started by running <source>% bin/local-regionservers.sh start 2 3 4 5</source>
+ Supports up to 99 extra regionservers (100 total).
+ </li>
+ </ol>
+</li>
+<li>To stop the cluster
+ <ol>
+ <li>Assuming you want to stop master backup # 1, run
+ <source>% cat /tmp/hbase-${USER}-1-master.pid |xargs kill -9</source>
+ Note that bin/local-master-backup.sh stop 1 will try to stop the cluster along with the master
+ </li>
+ <li>To stop an individual regionserver, run
+ <source>% bin/local-regionservers.sh stop 1
+ </source>
+ </li>
+ </ol>
+</li>
+</ol>
+</body>
+
+</document>
Added: hbase/branches/0.89/src/site/xdoc/replication.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/site/xdoc/replication.xml?rev=1181336&view=auto
==============================================================================
--- hbase/branches/0.89/src/site/xdoc/replication.xml (added)
+++ hbase/branches/0.89/src/site/xdoc/replication.xml Tue Oct 11 01:59:50 2011
@@ -0,0 +1,429 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright 2010 The Apache Software Foundation
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN"
+ "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document xmlns="http://maven.apache.org/XDOC/2.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/XDOC/2.0 http://maven.apache.org/xsd/xdoc-2.0.xsd">
+ <properties>
+ <title>
+ HBase Replication
+ </title>
+ </properties>
+ <body>
+ <section name="Overview">
+ <p>
+ HBase replication is a way to copy data between HBase deployments. It
+ can serve as a disaster recovery solution and can contribute to provide
+ higher availability at HBase layer. It can also serve more practically;
+ for example, as a way to easily copy edits from a web-facing cluster to a "MapReduce"
+ cluster which will process old and new data and ship back the results
+ automatically.
+ </p>
+ <p>
+ The basic architecture pattern used for HBase replication is (HBase cluster) master-push;
+ it is much easier to keep track of whatâs currently being replicated since
+ each region server has its own write-ahead-log (aka WAL or HLog), just like
+ other well known solutions like MySQL master/slave replication where
+ thereâs only one bin log to keep track of. One master cluster can
+ replicate to any number of slave clusters, and each region server will
+ participate to replicate their own stream of edits.
+ </p>
+ <p>
+ The replication is done asynchronously, meaning that the clusters can
+ be geographically distant, the links between them can be offline for
+ some time, and rows inserted on the master cluster wonât be
+ available at the same time on the slave clusters (eventual consistency).
+ </p>
+ <p>
+ The replication format used in this design is conceptually the same as
+ <a href="http://dev.mysql.com/doc/refman/5.1/en/replication-formats.html">
+ MySQLâs statement-based replication </a>. Instead of SQL statements, whole
+ WALEdits (consisting of multiple cell inserts coming from the clients'
+ Put and Delete) are replicated in order to maintain atomicity.
+ </p>
+ <p>
+ The HLogs from each region server are the basis of HBase replication,
+ and must be kept in HDFS as long as they are needed to replicate data
+ to any slave cluster. Each RS reads from the oldest log it needs to
+ replicate and keeps the current position inside ZooKeeper to simplify
+ failure recovery. That position can be different for every slave
+ cluster, same for the queue of HLogs to process.
+ </p>
+ <p>
+ The clusters participating in replication can be of asymmetric sizes
+ and the master cluster will do its âbest effortâ to balance the stream
+ of replication on the slave clusters by relying on randomization.
+ </p>
+ <img src="images/replication_overview.png"/>
+ </section>
+ <section name="Life of a log edit">
+ <p>
+ The following sections describe the life of a single edit going from a
+ client that communicates with a master cluster all the way to a single
+ slave cluster.
+ </p>
+ <section name="Normal processing">
+ <p>
+ The client uses a HBase API that sends a Put, Delete or ICV to a region
+ server. The key values are transformed into a WALEdit by the region
+ server and is inspected by the replication code that, for each family
+ that is scoped for replication, adds the scope to the edit. The edit
+ is appended to the current WAL and is then applied to its MemStore.
+ </p>
+ <p>
+ In a separate thread, the edit is read from the log (as part of a batch)
+ and only the KVs that are replicable are kept (that is, that are part
+ of a family scoped GLOBAL in the family's schema and non-catalog so not
+ .META. or -ROOT-). When the buffer is filled, or the reader hits the
+ end of the file, the buffer is sent to a random region server on the
+ slave cluster.
+ </p>
+ <p>
+ Synchronously, the region server that receives the edits reads them
+ sequentially and applies them on its own cluster using the HBase client
+ (HTables managed by a HTablePool) automatically. If consecutive rows
+ belong to the same table, they are inserted together in order to
+ leverage parallel insertions.
+ </p>
+ <p>
+ Back in the master cluster's region server, the offset for the current
+ WAL that's being replicated is registered in ZooKeeper.
+ </p>
+ </section>
+ <section name="Non-responding slave clusters">
+ <p>
+ The edit is inserted in the same way.
+ </p>
+ <p>
+ In the separate thread, the region server reads, filters and buffers
+ the log edits the same way as during normal processing. The slave
+ region server that's contacted doesn't answer to the RPC, so the master
+ region server will sleep and retry up to a configured number of times.
+ If the slave RS still isn't available, the master cluster RS will select a
+ new subset of RS to replicate to and will retry sending the buffer of
+ edits.
+ </p>
+ <p>
+ In the mean time, the WALs will be rolled and stored in a queue in
+ ZooKeeper. Logs that are archived by their region server (archiving is
+ basically moving a log from the region server's logs directory to a
+ central logs archive directory) will update their paths in the in-memory
+ queue of the replicating thread.
+ </p>
+ <p>
+ When the slave cluster is finally available, the buffer will be applied
+ the same way as during normal processing. The master cluster RS will then
+ replicate the backlog of logs.
+ </p>
+ </section>
+ </section>
+ <section name="Internals">
+ <p>
+ This section describes in depth how each of replication's internal
+ features operate.
+ </p>
+ <section name="Choosing region servers to replicate to">
+ <p>
+ When a master cluster RS initiates a replication source to a slave cluster,
+ it first connects to the slave's ZooKeeper ensemble using the provided
+ cluster key (taht key is composed of the value of hbase.zookeeper.quorum,
+ zookeeper.znode.parent and hbase.zookeeper.property.clientPort). It
+ then scans the "rs" directory to discover all the available sinks
+ (region servers that are accepting incoming streams of edits to replicate)
+ and will randomly choose a subset of them using a configured
+ ratio (which has a default value of 10%). For example, if a slave
+ cluster has 150 machines, 15 will be chosen as potential recipient for
+ edits that this master cluster RS will be sending. Since this is done by all
+ master cluster RSs, the probability that all slave RSs are used is very high,
+ and this method works for clusters of any size. For example, a master cluster
+ of 10 machines replicating to a slave cluster of 5 machines with a ratio
+ of 10% means that the master cluster RSs will choose one machine each
+ at random, thus the chance of overlapping and full usage of the slave
+ cluster is higher.
+ </p>
+ </section>
+ <section name="Keeping track of logs">
+ <p>
+ Every master cluster RS has its own znode in the replication znodes hierarchy.
+ It contains one znode per peer cluster (if 5 slave clusters, 5 znodes
+ are created), and each of these contain a queue
+ of HLogs to process. Each of these queues will track the HLogs created
+ by that RS, but they can differ in size. For example, if one slave
+ cluster becomes unavailable for some time then the HLogs cannot be,
+ thus they need to stay in the queue (while the others are processed).
+ See the section named "Region server failover" for an example.
+ </p>
+ <p>
+ When a source is instantiated, it contains the current HLog that the
+ region server is writing to. During log rolling, the new file is added
+ to the queue of each slave cluster's znode just before it's made available.
+ This ensures that all the sources are aware that a new log exists
+ before HLog is able to append edits into it, but this operations is
+ now more expensive.
+ The queue items are discarded when the replication thread cannot read
+ more entries from a file (because it reached the end of the last block)
+ and that there are other files in the queue.
+ This means that if a source is up-to-date and replicates from the log
+ that the region server writes to, reading up to the "end" of the
+ current file won't delete the item in the queue.
+ </p>
+ <p>
+ When a log is archived (because it's not used anymore or because there's
+ too many of them per hbase.regionserver.maxlogs typically because insertion
+ rate is faster than region flushing), it will notify the source threads that the path
+ for that log changed. If the a particular source was already done with
+ it, it will just ignore the message. If it's in the queue, the path
+ will be updated in memory. If the log is currently being replicated,
+ the change will be done atomically so that the reader doesn't try to
+ open the file when it's already moved. Also, moving a file is a NameNode
+ operation so, if the reader is currently reading the log, it won't
+ generate any exception.
+ </p>
+ </section>
+ <section name="Reading, filtering and sending edits">
+ <p>
+ By default, a source will try to read from a log file and ship log
+ entries as fast as possible to a sink. This is first limited by the
+ filtering of log entries; only KeyValues that are scoped GLOBAL and
+ that don't belong to catalog tables will be retained. A second limit
+ is imposed on the total size of the list of edits to replicate per slave,
+ which by default is 64MB. This means that a master cluster RS with 3 slaves
+ will use at most 192MB to store data to replicate. This doesn't account
+ the data filtered that wasn't garbage collected.
+ </p>
+ <p>
+ Once the maximum size of edits was buffered or the reader hits the end
+ of the log file, the source thread will stop reading and will choose
+ at random a sink to replicate to (from the list that was generated by
+ keeping only a subset of slave RSs). It will directly issue a RPC to
+ the chosen machine and will wait for the method to return. If it's
+ successful, the source will determine if the current file is emptied
+ or if it should continue to read from it. If the former, it will delete
+ the znode in the queue. If the latter, it will register the new offset
+ in the log's znode. If the RPC threw an exception, the source will retry
+ 10 times until trying to find a different sink.
+ </p>
+ </section>
+ <section name="Applying edits">
+ <p>
+ The sink synchronously applies the edits to its local cluster using
+ the native client API. This method is also synchronized between every
+ incoming sources, which means that only one batch of log entries can be
+ replicated at a time by each slave region server.
+ </p>
+ <p>
+ The sink applies the edits one by one, unless it's able to batch
+ sequential Puts that belong to the same table in order to use the
+ parallel puts feature of HConnectionManager. The Put and Delete objects
+ are recreated by inspecting the incoming WALEdit objects and are
+ with the exact same row, family, qualifier, timestamp, and value (for
+ Put). Note that if the master and slave cluster don't have the same
+ time, time-related issues may occur.
+ </p>
+ </section>
+ <section name="Cleaning logs">
+ <p>
+ If replication isn't enabled, the master's logs cleaning thread will
+ delete old logs using a configured TTL. This doesn't work well with
+ replication since archived logs passed their TTL may still be in a
+ queue. Thus, the default behavior is augmented so that if a log is
+ passed its TTL, the cleaning thread will lookup every queue until it
+ finds the log (while caching the ones it finds). If it's not found,
+ the log will be deleted. The next time it has to look for a log,
+ it will first use its cache.
+ </p>
+ </section>
+ <section name="Region server failover">
+ <p>
+ As long as region servers don't fail, keeping track of the logs in ZK
+ doesn't add any value. Unfortunately, they do fail, so since ZooKeeper
+ is highly available we can count on it and its semantics to help us
+ managing the transfer of the queues.
+ </p>
+ <p>
+ All the master cluster RSs keep a watcher on every other one of them to be
+ notified when one dies (just like the master does). When it happens,
+ they all race to create a znode called "lock" inside the dead RS' znode
+ that contains its queues. The one that creates it successfully will
+ proceed by transferring all the queues to its own znode (one by one
+ since ZK doesn't support the rename operation) and will delete all the
+ old ones when it's done. The recovered queues' znodes will be named
+ with the id of the slave cluster appended with the name of the dead
+ server.
+ </p>
+ <p>
+ Once that is done, the master cluster RS will create one new source thread per
+ copied queue, and each of them will follow the read/filter/ship pattern.
+ The main difference is that those queues will never have new data since
+ they don't belong to their new region server, which means that when
+ the reader hits the end of the last log, the queue's znode will be
+ deleted and the master cluster RS will close that replication source.
+ </p>
+ <p>
+ For example, consider a master cluster with 3 region servers that's
+ replicating to a single slave with id '2'. The following hierarchy
+ represents what the znodes layout could be at some point in time. We
+ can see the RSs' znodes all contain a "peers" znode that contains a
+ single queue. The znode names in the queues represent the actual file
+ names on HDFS in the form "address,port.timestamp".
+ </p>
+ <pre>
+/hbase/replication/rs/
+ 1.1.1.1,60020,123456780/
+ peers/
+ 2/
+ 1.1.1.1,60020.1234 (Contains a position)
+ 1.1.1.1,60020.1265
+ 1.1.1.2,60020,123456790/
+ peers/
+ 2/
+ 1.1.1.2,60020.1214 (Contains a position)
+ 1.1.1.2,60020.1248
+ 1.1.1.2,60020.1312
+ 1.1.1.3,60020, 123456630/
+ peers/
+ 2/
+ 1.1.1.3,60020.1280 (Contains a position)
+
+ </pre>
+ <p>
+ Now let's say that 1.1.1.2 loses its ZK session. The survivors will race
+ to create a lock, and for some reasons 1.1.1.3 wins. It will then start
+ transferring all the queues to its local peers znode by appending the
+ name of the dead server. Right before 1.1.1.3 is able to clean up the
+ old znodes, the layout will look like the following:
+ </p>
+ <pre>
+/hbase/replication/rs/
+ 1.1.1.1,60020,123456780/
+ peers/
+ 2/
+ 1.1.1.1,60020.1234 (Contains a position)
+ 1.1.1.1,60020.1265
+ 1.1.1.2,60020,123456790/
+ lock
+ peers/
+ 2/
+ 1.1.1.2,60020.1214 (Contains a position)
+ 1.1.1.2,60020.1248
+ 1.1.1.2,60020.1312
+ 1.1.1.3,60020,123456630/
+ peers/
+ 2/
+ 1.1.1.3,60020.1280 (Contains a position)
+
+ 2-1.1.1.2,60020,123456790/
+ 1.1.1.2,60020.1214 (Contains a position)
+ 1.1.1.2,60020.1248
+ 1.1.1.2,60020.1312
+ </pre>
+ <p>
+ Some time later, but before 1.1.1.3 is able to finish replicating the
+ last HLog from 1.1.1.2, let's say that it dies too (also some new logs
+ were created in the normal queues). The last RS will then try to lock
+ 1.1.1.3's znode and will begin transferring all the queues. The new
+ layout will be:
+ </p>
+ <pre>
+/hbase/replication/rs/
+ 1.1.1.1,60020,123456780/
+ peers/
+ 2/
+ 1.1.1.1,60020.1378 (Contains a position)
+
+ 2-1.1.1.3,60020,123456630/
+ 1.1.1.3,60020.1325 (Contains a position)
+ 1.1.1.3,60020.1401
+
+ 2-1.1.1.2,60020,123456790-1.1.1.3,60020,123456630/
+ 1.1.1.2,60020.1312 (Contains a position)
+ 1.1.1.3,60020,123456630/
+ lock
+ peers/
+ 2/
+ 1.1.1.3,60020.1325 (Contains a position)
+ 1.1.1.3,60020.1401
+
+ 2-1.1.1.2,60020,123456790/
+ 1.1.1.2,60020.1312 (Contains a position)
+ </pre>
+ </section>
+ </section>
+ <section name="FAQ">
+ <section name="Why do all clusters need to be in the same timezone?">
+ <p>
+ Suppose an edit to cell X happens in a EST cluster, then 2 minutes
+ later a new edits happens to the same cell in a PST cluster and that
+ both clusters are in a master-master replication. The second edit is
+ considered younger, so the first will always hide it while in fact the
+ second is older.
+ </p>
+ </section>
+ <section name="GLOBAL means replicate? Any provision to replicate only to cluster X and not to cluster Y? or is that for later?">
+ <p>
+ Yes, this is for much later.
+ </p>
+ </section>
+ <section name="You need a bulk edit shipper? Something that allows you transfer 64MB of edits in one go?">
+ <p>
+ You can use the HBase-provided utility called CopyTable from the package
+ org.apache.hadoop.hbase.mapreduce in order to have a discp-like tool to
+ bulk copy data.
+ </p>
+ </section>
+ <section name="Is it a mistake that WALEdit doesn't carry Put and Delete objects, that we have to reinstantiate not only replicating but when replaying edits?">
+ <p>
+ Yes, this behavior would help a lot but it's not currently available
+ in HBase (BatchUpdate had that, but it was lost in the new API).
+ </p>
+ </section>
+ </section>
+ <section name="Known bugs/missing features">
+ <p>
+ Here's a list of all the jiras that relate to major issues or missing
+ features in the replication implementation.
+ </p>
+ <ol>
+ <li>
+ HBASE-2688, master-master replication is disabled in the code, we need
+ to enable and test it.
+ </li>
+ <li>
+ HBASE-2611, basically if a region server dies while recovering the
+ queues of another dead RS, we will miss the data from the queues
+ that weren't copied.
+ </li>
+ <li>
+ HBASE-2196, a master cluster can only support a single slave, some
+ refactoring is needed to support this.
+ </li>
+ <li>
+ HBASE-2195, edits are applied disregard their home cluster, it should
+ carry that data and check it.
+ </li>
+ <li>
+ HBASE-2200, currently all the replication operations (adding or removing
+ streams for example) are done only when the clusters are offline. This
+ should be possible at runtime.
+ </li>
+ </ol>
+ </section>
+ </body>
+</document>
\ No newline at end of file
Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Tue Oct 11 01:59:50 2011
@@ -29,6 +29,9 @@ import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Set;
import java.util.UUID;
import org.apache.commons.logging.Log;
@@ -39,6 +42,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HTable;
@@ -48,6 +52,9 @@ import org.apache.hadoop.hbase.client.Re
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.ReadWriteConsistencyControl;
+import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Threads;
@@ -344,6 +351,14 @@ public class HBaseTestingUtility {
this.hbaseCluster.flushcache();
}
+ /**
+ * Flushes all caches in the mini hbase cluster
+ * @throws IOException
+ */
+ public void flush(byte [] tableName) throws IOException {
+ this.hbaseCluster.flushcache(tableName);
+ }
+
/**
* Create a table.
@@ -954,4 +969,42 @@ public class HBaseTestingUtility {
Threads.sleep(1000);
}
}
-}
\ No newline at end of file
+
+ /**
+ * Do a small get/scan against one store. This is required because store
+ * has no actual methods of querying itself, and relies on StoreScanner.
+ */
+ public static List<KeyValue> getFromStoreFile(Store store,
+ Get get) throws IOException {
+ ReadWriteConsistencyControl.resetThreadReadPoint();
+ Scan scan = new Scan(get);
+ InternalScanner scanner = (InternalScanner) store.getScanner(scan,
+ scan.getFamilyMap().get(store.getFamily().getName()));
+
+ List<KeyValue> result = new ArrayList<KeyValue>();
+ scanner.next(result);
+ if (!result.isEmpty()) {
+ // verify that we are on the row we want:
+ KeyValue kv = result.get(0);
+ if (!Bytes.equals(kv.getRow(), get.getRow())) {
+ result.clear();
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Do a small get/scan against one store. This is required because store
+ * has no actual methods of querying itself, and relies on StoreScanner.
+ */
+ public static List<KeyValue> getFromStoreFile(Store store,
+ byte [] row,
+ NavigableSet<byte[]> columns
+ ) throws IOException {
+ Get get = new Get(row);
+ Map<byte[], NavigableSet<byte[]>> s = get.getFamilyMap();
+ s.put(store.getFamily().getName(), columns);
+
+ return getFromStoreFile(store,get);
+ }
+}
Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java Tue Oct 11 01:59:50 2011
@@ -349,6 +349,21 @@ public class MiniHBaseCluster {
}
/**
+ * Call flushCache on all regions of the specified table.
+ * @throws IOException
+ */
+ public void flushcache(byte [] tableName) throws IOException {
+ for (JVMClusterUtil.RegionServerThread t:
+ this.hbaseCluster.getRegionServers()) {
+ for(HRegion r: t.getRegionServer().getOnlineRegions()) {
+ if(Bytes.equals(r.getTableDesc().getName(), tableName)) {
+ r.flushcache();
+ }
+ }
+ }
+ }
+
+ /**
* @return List of region server threads.
*/
public List<JVMClusterUtil.RegionServerThread> getRegionServerThreads() {
Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/client/TestMultipleTimestamps.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/client/TestMultipleTimestamps.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/client/TestMultipleTimestamps.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/client/TestMultipleTimestamps.java Tue Oct 11 01:59:50 2011
@@ -79,6 +79,179 @@ public class TestMultipleTimestamps {
}
@Test
+ public void testReseeksWithOneColumnMiltipleTimestamp() throws IOException {
+ byte [] TABLE = Bytes.toBytes("testReseeksWithOne" +
+ "ColumnMiltipleTimestamps");
+ byte [] FAMILY = Bytes.toBytes("event_log");
+ byte [][] FAMILIES = new byte[][] { FAMILY };
+
+ // create table; set versions to max...
+ HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
+
+ Integer[] putRows = new Integer[] {1, 3, 5, 7};
+ Integer[] putColumns = new Integer[] { 1, 3, 5};
+ Long[] putTimestamps = new Long[] {1L, 2L, 3L, 4L, 5L};
+
+ Integer[] scanRows = new Integer[] {3, 5};
+ Integer[] scanColumns = new Integer[] {3};
+ Long[] scanTimestamps = new Long[] {3L, 4L};
+ int scanMaxVersions = 2;
+
+ put(ht, FAMILY, putRows, putColumns, putTimestamps);
+
+ flush(TABLE);
+
+ ResultScanner scanner = scan(ht, FAMILY, scanRows, scanColumns,
+ scanTimestamps, scanMaxVersions);
+
+ KeyValue[] kvs;
+
+ kvs = scanner.next().raw();
+ assertEquals(2, kvs.length);
+ checkOneCell(kvs[0], FAMILY, 3, 3, 4);
+ checkOneCell(kvs[1], FAMILY, 3, 3, 3);
+ kvs = scanner.next().raw();
+ assertEquals(2, kvs.length);
+ checkOneCell(kvs[0], FAMILY, 5, 3, 4);
+ checkOneCell(kvs[1], FAMILY, 5, 3, 3);
+ }
+
+ @Test
+ public void testReseeksWithMultipleColumnOneTimestamp() throws IOException {
+ byte [] TABLE = Bytes.toBytes("testReseeksWithMultiple" +
+ "ColumnOneTimestamps");
+ byte [] FAMILY = Bytes.toBytes("event_log");
+ byte [][] FAMILIES = new byte[][] { FAMILY };
+
+ // create table; set versions to max...
+ HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
+
+ Integer[] putRows = new Integer[] {1, 3, 5, 7};
+ Integer[] putColumns = new Integer[] { 1, 3, 5};
+ Long[] putTimestamps = new Long[] {1L, 2L, 3L, 4L, 5L};
+
+ Integer[] scanRows = new Integer[] {3, 5};
+ Integer[] scanColumns = new Integer[] {3,4};
+ Long[] scanTimestamps = new Long[] {3L};
+ int scanMaxVersions = 2;
+
+ put(ht, FAMILY, putRows, putColumns, putTimestamps);
+
+ flush(TABLE);
+
+ ResultScanner scanner = scan(ht, FAMILY, scanRows, scanColumns,
+ scanTimestamps, scanMaxVersions);
+
+ KeyValue[] kvs;
+
+ kvs = scanner.next().raw();
+ assertEquals(1, kvs.length);
+ checkOneCell(kvs[0], FAMILY, 3, 3, 3);
+ kvs = scanner.next().raw();
+ assertEquals(1, kvs.length);
+ checkOneCell(kvs[0], FAMILY, 5, 3, 3);
+ }
+
+ @Test
+ public void testReseeksWithMultipleColumnMultipleTimestamp() throws
+ IOException {
+ byte [] TABLE = Bytes.toBytes("testReseeksWithMultiple" +
+ "ColumnMiltipleTimestamps");
+ byte [] FAMILY = Bytes.toBytes("event_log");
+ byte [][] FAMILIES = new byte[][] { FAMILY };
+
+ // create table; set versions to max...
+ HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
+
+ Integer[] putRows = new Integer[] {1, 3, 5, 7};
+ Integer[] putColumns = new Integer[] { 1, 3, 5};
+ Long[] putTimestamps = new Long[] {1L, 2L, 3L, 4L, 5L};
+
+ Integer[] scanRows = new Integer[] {5, 7};
+ Integer[] scanColumns = new Integer[] {3, 4, 5};
+ Long[] scanTimestamps = new Long[] {2l, 3L};
+ int scanMaxVersions = 2;
+
+ put(ht, FAMILY, putRows, putColumns, putTimestamps);
+
+ flush(TABLE);
+
+ ResultScanner scanner = scan(ht, FAMILY, scanRows, scanColumns,
+ scanTimestamps, scanMaxVersions);
+
+ KeyValue[] kvs;
+
+ kvs = scanner.next().raw();
+ assertEquals(4, kvs.length);
+ checkOneCell(kvs[0], FAMILY, 5, 3, 3);
+ checkOneCell(kvs[1], FAMILY, 5, 3, 2);
+ checkOneCell(kvs[2], FAMILY, 5, 5, 3);
+ checkOneCell(kvs[3], FAMILY, 5, 5, 2);
+ kvs = scanner.next().raw();
+ assertEquals(4, kvs.length);
+ checkOneCell(kvs[0], FAMILY, 7, 3, 3);
+ checkOneCell(kvs[1], FAMILY, 7, 3, 2);
+ checkOneCell(kvs[2], FAMILY, 7, 5, 3);
+ checkOneCell(kvs[3], FAMILY, 7, 5, 2);
+ }
+
+ @Test
+ public void testReseeksWithMultipleFiles() throws IOException {
+ byte [] TABLE = Bytes.toBytes("testReseeksWithMultipleFiles");
+ byte [] FAMILY = Bytes.toBytes("event_log");
+ byte [][] FAMILIES = new byte[][] { FAMILY };
+
+ // create table; set versions to max...
+ HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
+
+ Integer[] putRows1 = new Integer[] {1, 2, 3};
+ Integer[] putColumns1 = new Integer[] { 2, 5, 6};
+ Long[] putTimestamps1 = new Long[] {1L, 2L, 5L};
+
+ Integer[] putRows2 = new Integer[] {6, 7};
+ Integer[] putColumns2 = new Integer[] {3, 6};
+ Long[] putTimestamps2 = new Long[] {4L, 5L};
+
+ Integer[] putRows3 = new Integer[] {2, 3, 5};
+ Integer[] putColumns3 = new Integer[] {1, 2, 3};
+ Long[] putTimestamps3 = new Long[] {4L,8L};
+
+
+ Integer[] scanRows = new Integer[] {3, 5, 7};
+ Integer[] scanColumns = new Integer[] {3, 4, 5};
+ Long[] scanTimestamps = new Long[] {2l, 4L};
+ int scanMaxVersions = 5;
+
+ put(ht, FAMILY, putRows1, putColumns1, putTimestamps1);
+ flush(TABLE);
+ put(ht, FAMILY, putRows2, putColumns2, putTimestamps2);
+ flush(TABLE);
+ put(ht, FAMILY, putRows3, putColumns3, putTimestamps3);
+
+ ResultScanner scanner = scan(ht, FAMILY, scanRows, scanColumns,
+ scanTimestamps, scanMaxVersions);
+
+ KeyValue[] kvs;
+
+ kvs = scanner.next().raw();
+ assertEquals(2, kvs.length);
+ checkOneCell(kvs[0], FAMILY, 3, 3, 4);
+ checkOneCell(kvs[1], FAMILY, 3, 5, 2);
+
+ kvs = scanner.next().raw();
+ assertEquals(1, kvs.length);
+ checkOneCell(kvs[0], FAMILY, 5, 3, 4);
+
+ kvs = scanner.next().raw();
+ assertEquals(1, kvs.length);
+ checkOneCell(kvs[0], FAMILY, 6, 3, 4);
+
+ kvs = scanner.next().raw();
+ assertEquals(1, kvs.length);
+ checkOneCell(kvs[0], FAMILY, 7, 3, 4);
+ }
+
+ @Test
public void testWithVersionDeletes() throws Exception {
// first test from memstore (without flushing).
@@ -101,7 +274,7 @@ public class TestMultipleTimestamps {
putNVersions(ht, FAMILY, 0, 0, 1, 5);
if (flushTables) {
- flush();
+ flush(TABLE);
}
// delete version 4.
@@ -109,7 +282,8 @@ public class TestMultipleTimestamps {
// request a bunch of versions including the deleted version. We should
// only get back entries for the versions that exist.
- KeyValue kvs[] = getNVersions(ht, FAMILY, 0, 0, Arrays.asList(2L, 3L, 4L, 5L));
+ KeyValue kvs[] = getNVersions(ht, FAMILY, 0, 0,
+ Arrays.asList(2L, 3L, 4L, 5L));
assertEquals(3, kvs.length);
checkOneCell(kvs[0], FAMILY, 0, 0, 5);
checkOneCell(kvs[1], FAMILY, 0, 0, 3);
@@ -128,7 +302,7 @@ public class TestMultipleTimestamps {
// For row:0, col:0: insert versions 1 through 5.
putNVersions(ht, FAMILY, 0, 0, 1, 5);
- flush();
+ flush(TABLE);
// delete all versions before 4.
deleteAllVersionsBefore(ht, FAMILY, 0, 0, 4);
@@ -151,7 +325,7 @@ public class TestMultipleTimestamps {
// For row:0, col:0: insert versions 1 through 5.
putNVersions(ht, FAMILY, 0, 0, 1, 5);
- flush();
+ flush(TABLE);
// delete all versions before 4.
deleteColumn(ht, FAMILY, 0, 0);
@@ -174,7 +348,7 @@ public class TestMultipleTimestamps {
// For row:0, col:0: insert versions 1 through 5.
putNVersions(ht, FAMILY, 0, 0, 1, 5);
- flush();
+ flush(TABLE);
// delete all versions before 4.
deleteFamily(ht, FAMILY, 0);
@@ -186,8 +360,8 @@ public class TestMultipleTimestamps {
}
// Flush tables. Since flushing is asynchronous, sleep for a bit.
- private void flush() throws IOException {
- TEST_UTIL.flush();
+ private void flush(byte [] tableName) throws IOException {
+ TEST_UTIL.flush(tableName);
try {
Thread.sleep(3000);
} catch (InterruptedException i) {
@@ -240,6 +414,44 @@ public class TestMultipleTimestamps {
return result.raw();
}
+ private ResultScanner scan(HTable ht, byte[] cf,
+ Integer[] rowIndexes, Integer[] columnIndexes,
+ Long[] versions, int maxVersions)
+ throws IOException {
+ Arrays.asList(rowIndexes);
+ byte startRow[] = Bytes.toBytes("row:" +
+ Collections.min( Arrays.asList(rowIndexes)));
+ byte endRow[] = Bytes.toBytes("row:" +
+ Collections.max( Arrays.asList(rowIndexes))+1);
+ Scan scan = new Scan(startRow, endRow);
+ for (Integer colIdx: columnIndexes) {
+ byte column[] = Bytes.toBytes("column:" + colIdx);
+ scan.addColumn(cf, column);
+ }
+ scan.setMaxVersions(maxVersions);
+ scan.setTimeRange(Collections.min(Arrays.asList(versions)),
+ Collections.max(Arrays.asList(versions))+1);
+ ResultScanner scanner = ht.getScanner(scan);
+ return scanner;
+ }
+
+ private void put(HTable ht, byte[] cf, Integer[] rowIndexes,
+ Integer[] columnIndexes, Long[] versions)
+ throws IOException {
+ for (int rowIdx: rowIndexes) {
+ byte row[] = Bytes.toBytes("row:" + rowIdx);
+ Put put = new Put(row);
+ for(int colIdx: columnIndexes) {
+ byte column[] = Bytes.toBytes("column:" + colIdx);
+ for (long version: versions) {
+ put.add(cf, column, version, Bytes.toBytes("value-version-" +
+ version));
+ }
+ }
+ ht.put(put);
+ }
+ }
+
/**
* Insert in specific row/column versions with timestamps
* versionStart..versionEnd.
Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java Tue Oct 11 01:59:50 2011
@@ -175,10 +175,11 @@ public class TestLoadIncrementalHFiles {
{
HFile.Writer writer = new HFile.Writer(fs, path, BLOCKSIZE, COMPRESSION,
KeyValue.KEY_COMPARATOR);
+ long now = System.currentTimeMillis();
try {
// subtract 2 since iterateOnSplits doesn't include boundary keys
for (byte[] key : Bytes.iterateOnSplits(startKey, endKey, numRows-2)) {
- KeyValue kv = new KeyValue(key, family, qualifier, key);
+ KeyValue kv = new KeyValue(key, family, qualifier, now, key);
writer.append(kv);
}
} finally {
Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java Tue Oct 11 01:59:50 2011
@@ -26,8 +26,7 @@ import java.util.TreeSet;
import java.util.Arrays;
import org.apache.hadoop.hbase.HBaseTestCase;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.regionserver.QueryMatcher.MatchCode;
+import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode;
import org.apache.hadoop.hbase.util.Bytes;
@@ -49,7 +48,7 @@ public class TestExplicitColumnTracker e
//Initialize result
- List<MatchCode> result = new ArrayList<MatchCode>();
+ List<ScanQueryMatcher.MatchCode> result = new ArrayList<ScanQueryMatcher.MatchCode>();
//"Match"
for(byte [] col : scannerColumns){
@@ -76,12 +75,12 @@ public class TestExplicitColumnTracker e
//Looking for every other
columns.add(col2);
columns.add(col4);
- List<MatchCode> expected = new ArrayList<MatchCode>();
- expected.add(MatchCode.SKIP);
- expected.add(MatchCode.INCLUDE);
- expected.add(MatchCode.SKIP);
- expected.add(MatchCode.INCLUDE);
- expected.add(MatchCode.DONE);
+ List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
+ expected.add(ScanQueryMatcher.MatchCode.SKIP);
+ expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
+ expected.add(ScanQueryMatcher.MatchCode.SKIP);
+ expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
+ expected.add(ScanQueryMatcher.MatchCode.DONE);
int maxVersions = 1;
//Create "Scanner"
@@ -106,26 +105,26 @@ public class TestExplicitColumnTracker e
columns.add(col2);
columns.add(col4);
- List<MatchCode> expected = new ArrayList<MatchCode>();
- expected.add(MatchCode.SKIP);
- expected.add(MatchCode.SKIP);
- expected.add(MatchCode.SKIP);
-
- expected.add(MatchCode.INCLUDE);
- expected.add(MatchCode.INCLUDE);
- expected.add(MatchCode.SKIP);
-
- expected.add(MatchCode.SKIP);
- expected.add(MatchCode.SKIP);
- expected.add(MatchCode.SKIP);
-
- expected.add(MatchCode.INCLUDE);
- expected.add(MatchCode.INCLUDE);
- expected.add(MatchCode.DONE);
-
- expected.add(MatchCode.DONE);
- expected.add(MatchCode.DONE);
- expected.add(MatchCode.DONE);
+ List<ScanQueryMatcher.MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
+ expected.add(ScanQueryMatcher.MatchCode.SKIP);
+ expected.add(ScanQueryMatcher.MatchCode.SKIP);
+ expected.add(ScanQueryMatcher.MatchCode.SKIP);
+
+ expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
+ expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
+ expected.add(ScanQueryMatcher.MatchCode.SKIP);
+
+ expected.add(ScanQueryMatcher.MatchCode.SKIP);
+ expected.add(ScanQueryMatcher.MatchCode.SKIP);
+ expected.add(ScanQueryMatcher.MatchCode.SKIP);
+
+ expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
+ expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
+ expected.add(ScanQueryMatcher.MatchCode.DONE);
+
+ expected.add(ScanQueryMatcher.MatchCode.DONE);
+ expected.add(ScanQueryMatcher.MatchCode.DONE);
+ expected.add(ScanQueryMatcher.MatchCode.DONE);
int maxVersions = 2;
//Create "Scanner"
@@ -183,10 +182,10 @@ public class TestExplicitColumnTracker e
col2, col3, col5 }));
List<byte[]> scanner = Arrays.<byte[]>asList(
new byte[][] { col1, col4 });
- List<MatchCode> expected = Arrays.<MatchCode>asList(
- new MatchCode[] {
- MatchCode.SKIP,
- MatchCode.SKIP });
+ List<ScanQueryMatcher.MatchCode> expected = Arrays.<ScanQueryMatcher.MatchCode>asList(
+ new ScanQueryMatcher.MatchCode[] {
+ ScanQueryMatcher.MatchCode.SKIP,
+ ScanQueryMatcher.MatchCode.SKIP });
runTest(1, columns, scanner, expected);
}
}
Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java Tue Oct 11 01:59:50 2011
@@ -1,4 +1,4 @@
-/**
+/*
* Copyright 2010 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java Tue Oct 11 01:59:50 2011
@@ -1367,6 +1367,7 @@ public class TestHRegion extends HBaseTe
String method = this.getName();
initHRegion(tableName, method, families);
+
//Putting data in Region
Put put = new Put(row1);
put.add(fam1, null, null);
@@ -1384,10 +1385,12 @@ public class TestHRegion extends HBaseTe
scan.addFamily(fam2);
scan.addFamily(fam4);
is = (RegionScanner) region.getScanner(scan);
+ ReadWriteConsistencyControl.resetThreadReadPoint(region.getRWCC());
assertEquals(1, ((RegionScanner)is).storeHeap.getHeap().size());
scan = new Scan();
is = (RegionScanner) region.getScanner(scan);
+ ReadWriteConsistencyControl.resetThreadReadPoint(region.getRWCC());
assertEquals(families.length -1,
((RegionScanner)is).storeHeap.getHeap().size());
}
@@ -1856,7 +1859,8 @@ public class TestHRegion extends HBaseTe
assertEquals(value+amount, result);
Store store = region.getStore(fam1);
- assertEquals(1, store.memstore.kvset.size());
+ // we will have the original Put, and also the ICV'ed Put as well.
+ assertEquals(2, store.memstore.kvset.size());
assertTrue(store.memstore.snapshot.isEmpty());
assertICV(row, fam1, qual1, value+amount);
Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java Tue Oct 11 01:59:50 2011
@@ -570,64 +570,6 @@ public class TestMemStore extends TestCa
}
}
- public void testGet_Basic_Found() throws IOException {
- byte [] row = Bytes.toBytes("testrow");
- byte [] fam = Bytes.toBytes("testfamily");
- byte [] qf1 = Bytes.toBytes("testqualifier1");
- byte [] qf2 = Bytes.toBytes("testqualifier2");
- byte [] qf3 = Bytes.toBytes("testqualifier3");
- byte [] val = Bytes.toBytes("testval");
-
- //Setting up memstore
- KeyValue add1 = new KeyValue(row, fam ,qf1, val);
- KeyValue add2 = new KeyValue(row, fam ,qf2, val);
- KeyValue add3 = new KeyValue(row, fam ,qf3, val);
- memstore.add(add1);
- memstore.add(add2);
- memstore.add(add3);
-
- //test
- Get get = new Get(row);
- NavigableSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- columns.add(qf2);
- long ttl = Long.MAX_VALUE;
-
- QueryMatcher matcher =
- new QueryMatcher(get, fam, columns, ttl, KeyValue.KEY_COMPARATOR, 1);
-
- List<KeyValue> result = new ArrayList<KeyValue>();
- boolean res = memstore.get(matcher, result);
- assertEquals(true, res);
- }
-
- public void testGet_Basic_NotFound() throws IOException {
- byte [] row = Bytes.toBytes("testrow");
- byte [] fam = Bytes.toBytes("testfamily");
- byte [] qf1 = Bytes.toBytes("testqualifier1");
- byte [] qf2 = Bytes.toBytes("testqualifier2");
- byte [] qf3 = Bytes.toBytes("testqualifier3");
- byte [] val = Bytes.toBytes("testval");
-
- //Setting up memstore
- KeyValue add1 = new KeyValue(row, fam ,qf1, val);
- KeyValue add3 = new KeyValue(row, fam ,qf3, val);
- memstore.add(add1);
- memstore.add(add3);
-
- //test
- Get get = new Get(row);
- NavigableSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- columns.add(qf2);
- long ttl = Long.MAX_VALUE;
-
- QueryMatcher matcher =
- new QueryMatcher(get, fam, columns, ttl, KeyValue.KEY_COMPARATOR, 1);
-
- List<KeyValue> result = new ArrayList<KeyValue>();
- boolean res = memstore.get(matcher, result);
- assertEquals(false, res);
- }
-
public void testGet_memstoreAndSnapShot() throws IOException {
byte [] row = Bytes.toBytes("testrow");
byte [] fam = Bytes.toBytes("testfamily");
@@ -638,16 +580,6 @@ public class TestMemStore extends TestCa
byte [] qf5 = Bytes.toBytes("testqualifier5");
byte [] val = Bytes.toBytes("testval");
- //Creating get
- Get get = new Get(row);
- NavigableSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- columns.add(qf2);
- columns.add(qf4);
- long ttl = Long.MAX_VALUE;
-
- QueryMatcher matcher =
- new QueryMatcher(get, fam, columns, ttl, KeyValue.KEY_COMPARATOR, 1);
-
//Setting up memstore
memstore.add(new KeyValue(row, fam ,qf1, val));
memstore.add(new KeyValue(row, fam ,qf2, val));
@@ -660,64 +592,6 @@ public class TestMemStore extends TestCa
memstore.add(new KeyValue(row, fam ,qf4, val));
memstore.add(new KeyValue(row, fam ,qf5, val));
assertEquals(2, memstore.kvset.size());
-
- List<KeyValue> result = new ArrayList<KeyValue>();
- boolean res = memstore.get(matcher, result);
- assertEquals(true, res);
- }
-
- public void testGet_SpecificTimeStamp() throws IOException {
- byte [] row = Bytes.toBytes("testrow");
- byte [] fam = Bytes.toBytes("testfamily");
- byte [] qf1 = Bytes.toBytes("testqualifier1");
- byte [] qf2 = Bytes.toBytes("testqualifier2");
- byte [] qf3 = Bytes.toBytes("testqualifier3");
- byte [] val = Bytes.toBytes("testval");
-
- long ts1 = System.currentTimeMillis();
- long ts2 = ts1++;
- long ts3 = ts2++;
-
- //Creating get
- Get get = new Get(row);
- get.setTimeStamp(ts2);
- NavigableSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- columns.add(qf1);
- columns.add(qf2);
- columns.add(qf3);
- long ttl = Long.MAX_VALUE;
-
- QueryMatcher matcher = new QueryMatcher(get, fam, columns, ttl,
- KeyValue.KEY_COMPARATOR, 1);
-
- //Setting up expected
- List<KeyValue> expected = new ArrayList<KeyValue>();
- KeyValue kv1 = new KeyValue(row, fam ,qf1, ts2, val);
- KeyValue kv2 = new KeyValue(row, fam ,qf2, ts2, val);
- KeyValue kv3 = new KeyValue(row, fam ,qf3, ts2, val);
- expected.add(kv1);
- expected.add(kv2);
- expected.add(kv3);
-
- //Setting up memstore
- memstore.add(new KeyValue(row, fam ,qf1, ts1, val));
- memstore.add(new KeyValue(row, fam ,qf2, ts1, val));
- memstore.add(new KeyValue(row, fam ,qf3, ts1, val));
- memstore.add(kv1);
- memstore.add(kv2);
- memstore.add(kv3);
- memstore.add(new KeyValue(row, fam ,qf1, ts3, val));
- memstore.add(new KeyValue(row, fam ,qf2, ts3, val));
- memstore.add(new KeyValue(row, fam ,qf3, ts3, val));
-
- //Get
- List<KeyValue> result = new ArrayList<KeyValue>();
- memstore.get(matcher, result);
-
- assertEquals(expected.size(), result.size());
- for(int i=0; i<expected.size(); i++){
- assertEquals(expected.get(i), result.get(i));
- }
}
//////////////////////////////////////////////////////////////////////////////