You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2007/08/08 22:30:20 UTC
svn commit: r564012 [3/4] - in /lucene/hadoop/trunk/src/contrib/hbase: ./
src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/util/
src/test/org/apache/hadoop/hbase/
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java Wed Aug 8 13:30:13 2007
@@ -19,54 +19,103 @@
*/
package org.apache.hadoop.hbase;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.Random;
+import java.util.Vector;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.conf.*;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.io.MapFile;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.onelab.filter.Filter;
+import org.onelab.filter.Key;
-import java.io.*;
-import java.util.*;
/**
- * Each HStore maintains a bunch of different data files.
- *
- * An HStoreFile tracks 4 things: its parent dir, the region identifier, the
- * column family, and the file identifier. If you know those four things, you
- * know how to obtain the right HStoreFile.
+ * A HStore data file. HStores usually have one or more of these files. They
+ * are produced by flushing the memcache to disk.
*
- * When merging or splitting HRegions, we might want to modify one of the
- * params for an HStoreFile (effectively moving it elsewhere).
+ * <p>Each HStore maintains a bunch of different data files. The filename is a
+ * mix of the parent dir, the region name, the column name, and a file
+ * identifier. The name may also be a reference to a store file located
+ * elsewhere. This class handles all that path-building stuff for you.
+ *
+ * <p>An HStoreFile usually tracks 4 things: its parent dir, the region
+ * identifier, the column family, and the file identifier. If you know those
+ * four things, you know how to obtain the right HStoreFile. HStoreFiles may
+ * also refernce store files in another region serving either from
+ * the top-half of the remote file or from the bottom-half. Such references
+ * are made fast splitting regions.
*
- * The filename is a mix of the parent dir, the region name, the column name,
- * and the file identifier.
+ * <p>Plain HStoreFiles are named for a randomly generated id as in:
+ * <code>1278437856009925445</code> A file by this name is made in both the
+ * <code>mapfiles</code> and <code>info</code> subdirectories of a
+ * HStore columnfamily directoy: E.g. If the column family is 'anchor:', then
+ * under the region directory there is a subdirectory named 'anchor' within
+ * which is a 'mapfiles' and 'info' subdirectory. In each will be found a
+ * file named something like <code>1278437856009925445</code>, one to hold the
+ * data in 'mapfiles' and one under 'info' that holds the sequence id for this
+ * store file.
*
- * This class handles all that path-building stuff for you.
+ * <p>References to store files located over in some other region look like
+ * this:
+ * <code>1278437856009925445.hbaserepository,qAReLZD-OyQORZWq_vqR1k==,959247014679548184</code>:
+ * i.e. an id followed by the name of the referenced region. The data
+ * ('mapfiles') of HStoreFile references are empty. The accompanying
+ * <code>info</code> file contains the
+ * midkey, the id of the remote store we're referencing and whether we're
+ * to serve the top or bottom region of the remote store file. Note, a region
+ * is not splitable if it has instances of store file references (References
+ * are cleaned up by compactions).
+ *
+ * <p>When merging or splitting HRegions, we might want to modify one of the
+ * params for an HStoreFile (effectively moving it elsewhere).
*/
public class HStoreFile implements HConstants, WritableComparable {
- private static final Log LOG = LogFactory.getLog(HStoreFile.class.getName());
+ static final Log LOG = LogFactory.getLog(HStoreFile.class.getName());
static final byte INFO_SEQ_NUM = 0;
- static final String HSTORE_DATFILE_PREFIX = "mapfile.dat.";
- static final String HSTORE_INFOFILE_PREFIX = "mapfile.info.";
static final String HSTORE_DATFILE_DIR = "mapfiles";
static final String HSTORE_INFO_DIR = "info";
static final String HSTORE_FILTER_DIR = "filter";
+ public static enum Range {top, bottom}
+
+ /*
+ * Regex that will work for straight filenames and for reference names.
+ * If reference, then the regex has more than just one group. Group 1 is
+ * this files id. Group 2 the referenced region name, etc.
+ */
+ private static Pattern REF_NAME_PARSER =
+ Pattern.compile("^(\\d+)(?:\\.(.+))?$");
+
private static Random rand = new Random();
- Path dir;
- Text regionName;
- Text colFamily;
- long fileId;
- Configuration conf;
+ private Path dir;
+ private Text regionName;
+ private Text colFamily;
+ private long fileId;
+ private final Configuration conf;
+ private Reference reference;
- /** Constructor used by Writable */
+ /** Shutdown constructor used by Writable */
HStoreFile(Configuration conf) {
- this.conf = conf;
- this.dir = new Path(Path.CUR_DIR);
- this.regionName = new Text();
- this.colFamily = new Text();
- this.fileId = 0;
+ this(conf, new Path(Path.CUR_DIR), new Text(), new Text(), 0);
}
/**
@@ -77,54 +126,181 @@
* @param colFamily name of the column family
* @param fileId file identifier
*/
+ HStoreFile(final Configuration conf, final Path dir, final Text regionName,
+ final Text colFamily, final long fileId) {
+ this(conf, dir, regionName, colFamily, fileId, null);
+ }
+
+ /**
+ * Constructor that fully initializes the object
+ * @param conf Configuration object
+ * @param dir directory path
+ * @param regionName name of the region
+ * @param colFamily name of the column family
+ * @param fileId file identifier
+ * @param ref Reference to another HStoreFile.
+ */
HStoreFile(Configuration conf, Path dir, Text regionName,
- Text colFamily, long fileId) {
-
+ Text colFamily, long fileId, final Reference ref) {
this.conf = conf;
this.dir = dir;
this.regionName = new Text(regionName);
this.colFamily = new Text(colFamily);
this.fileId = fileId;
+ // If a reference, construction does not write the pointer files. Thats
+ // done by invocations of writeReferenceFiles(hsf, fs). Happens at fast
+ // split time.
+ this.reference = ref;
}
- /** @return the directory path */
- Path getDir() {
- return dir;
+ /*
+ * Data structure to hold reference to a store file over in another region.
+ */
+ static class Reference {
+ Text regionName;
+ long fileid;
+ Range region;
+ HStoreKey midkey;
+
+ Reference(final Text rn, final long fid, final HStoreKey m,
+ final Range fr) {
+ this.regionName = rn;
+ this.fileid = fid;
+ this.region = fr;
+ this.midkey = m;
+ }
+
+ Reference() {
+ this(null, -1, null, Range.bottom);
+ }
+
+ long getFileId() {
+ return this.fileid;
+ }
+
+ Range getFileRegion() {
+ return this.region;
+ }
+
+ HStoreKey getMidkey() {
+ return this.midkey;
+ }
+
+ Text getRegionName() {
+ return this.regionName;
+ }
+
+ public String toString() {
+ return this.regionName + "/" + this.fileid + "/" + this.region;
+ }
+
+ // Make it serializable.
+ public void write(DataOutput out) throws IOException {
+ this.regionName.write(out);
+ out.writeLong(this.fileid);
+ // Write true if we're doing top of the file.
+ out.writeBoolean(isTopFileRegion(this.region));
+ this.midkey.write(out);
+ }
+
+ public void readFields(DataInput in) throws IOException {
+ this.regionName = new Text();
+ this.regionName.readFields(in);
+ this.fileid = in.readLong();
+ boolean tmp = in.readBoolean();
+ // If true, set region to top.
+ this.region = tmp? Range.top: Range.bottom;
+ this.midkey = new HStoreKey();
+ this.midkey.readFields(in);
+ }
+ }
+
+ static boolean isTopFileRegion(final Range r) {
+ return r.equals(Range.top);
}
/** @return the region name */
+ boolean isReference() {
+ return this.reference != null;
+ }
+
+ Reference getReference() {
+ return this.reference;
+ }
+
Text getRegionName() {
- return regionName;
+ return this.regionName;
}
/** @return the column family */
Text getColFamily() {
- return colFamily;
+ return this.colFamily;
}
/** @return the file identifier */
- long fileId() {
- return fileId;
+ long getFileId() {
+ return this.fileId;
}
// Build full filenames from those components
-
/** @return path for MapFile */
Path getMapFilePath() {
- return new Path(HStoreFile.getMapDir(dir, regionName, colFamily),
- HSTORE_DATFILE_PREFIX + fileId);
+ return isReference()?
+ getMapFilePath(this.regionName, this.fileId,
+ this.reference.getRegionName()):
+ getMapFilePath(this.regionName, this.fileId);
+ }
+
+ private Path getMapFilePath(final Reference r) {
+ return r == null?
+ getMapFilePath():
+ getMapFilePath(r.getRegionName(), r.getFileId());
+ }
+
+ private Path getMapFilePath(final Text name, final long fid) {
+ return new Path(HStoreFile.getMapDir(dir, name, colFamily),
+ createHStoreFilename(fid, null));
}
+ private Path getMapFilePath(final Text name, final long fid, final Text rn) {
+ return new Path(HStoreFile.getMapDir(dir, name, colFamily),
+ createHStoreFilename(fid, rn));
+ }
+
/** @return path for info file */
Path getInfoFilePath() {
- return new Path(HStoreFile.getInfoDir(dir, regionName, colFamily),
- HSTORE_INFOFILE_PREFIX + fileId);
+ return isReference()?
+ getInfoFilePath(this.regionName, this.fileId,
+ this.reference.getRegionName()):
+ getInfoFilePath(this.regionName, this.fileId);
+ }
+
+ private Path getInfoFilePath(final Text name, final long fid) {
+ return new Path(HStoreFile.getInfoDir(dir, name, colFamily),
+ createHStoreFilename(fid, null));
+ }
+
+ private Path getInfoFilePath(final Text name, final long fid, final Text rn) {
+ return new Path(HStoreFile.getInfoDir(dir, name, colFamily),
+ createHStoreFilename(fid, rn));
}
// Static methods to build partial paths to internal directories. Useful for
// HStore construction and log-rebuilding.
-
- /** @return the map directory path */
+ private static String createHStoreFilename(final long fid) {
+ return createHStoreFilename(fid, null);
+ }
+
+ private static String createHStoreFilename(final long fid,
+ final Text regionName) {
+ return Long.toString(fid) +
+ ((regionName != null)? "." + regionName.toString(): "");
+ }
+
+ private static String createHStoreInfoFilename(final long fid) {
+ return createHStoreFilename(fid, null);
+ }
+
static Path getMapDir(Path dir, Text regionName, Text colFamily) {
return new Path(dir, new Path(HREGIONDIR_PREFIX + regionName,
new Path(colFamily.toString(), HSTORE_DATFILE_DIR)));
@@ -148,11 +324,6 @@
colFamily.toString()));
}
- /** @return the HRegion directory path */
- static Path getHRegionDir(Path dir, Text regionName) {
- return new Path(dir, new Path(HREGIONDIR_PREFIX + regionName));
- }
-
/**
* @return a brand-new randomly-named HStoreFile.
*
@@ -165,19 +336,18 @@
Path mapdir = HStoreFile.getMapDir(dir, regionName, colFamily);
long fileId = Math.abs(rand.nextLong());
- Path testpath1 = new Path(mapdir, HSTORE_DATFILE_PREFIX + fileId);
- Path testpath2 = new Path(mapdir, HSTORE_INFOFILE_PREFIX + fileId);
+ Path testpath1 = new Path(mapdir, createHStoreFilename(fileId));
+ Path testpath2 = new Path(mapdir, createHStoreInfoFilename(fileId));
while(fs.exists(testpath1) || fs.exists(testpath2)) {
fileId = Math.abs(rand.nextLong());
- testpath1 = new Path(mapdir, HSTORE_DATFILE_PREFIX + fileId);
- testpath2 = new Path(mapdir, HSTORE_INFOFILE_PREFIX + fileId);
+ testpath1 = new Path(mapdir, createHStoreFilename(fileId));
+ testpath2 = new Path(mapdir, createHStoreInfoFilename(fileId));
}
return new HStoreFile(conf, dir, regionName, colFamily, fileId);
}
- /**
+ /*
* Creates a series of HStoreFiles loaded from the given directory.
- *
* There must be a matching 'mapdir' and 'loginfo' pair of files.
* If only one exists, we'll delete it.
*
@@ -186,65 +356,82 @@
* @param regionName region name
* @param colFamily column family
* @param fs file system
- * @return Vector of HStoreFiles
+ * @return List of store file instances loaded from passed dir.
* @throws IOException
*/
static Vector<HStoreFile> loadHStoreFiles(Configuration conf, Path dir,
- Text regionName, Text colFamily, FileSystem fs) throws IOException {
+ Text regionName, Text colFamily, FileSystem fs)
+ throws IOException {
+ // Look first at info files. If a reference, these contain info we need
+ // to create the HStoreFile.
+ Path infodir = HStoreFile.getInfoDir(dir, regionName, colFamily);
+ Path infofiles[] = fs.listPaths(infodir);
+ Vector<HStoreFile> results = new Vector<HStoreFile>(infofiles.length);
+ Vector<Path> mapfiles = new Vector<Path>(infofiles.length);
+ for (int i = 0; i < infofiles.length; i++) {
+ Path p = infofiles[i];
+ Matcher m = REF_NAME_PARSER.matcher(p.getName());
+ boolean isReference = isReference(p, m);
+ long fid = Long.parseLong(m.group(1));
+ HStoreFile curfile = null;
+ if (isReference) {
+ Reference reference = readSplitInfo(infofiles[i], fs);
+ curfile = new HStoreFile(conf, dir, regionName, colFamily, fid,
+ reference);
+ } else {
+ curfile = new HStoreFile(conf, dir, regionName, colFamily, fid);
+ }
+ Path mapfile = curfile.getMapFilePath();
+ if (!fs.exists(mapfile)) {
+ fs.delete(curfile.getInfoFilePath());
+ }
+ // Found map and sympathetic info file. Add this hstorefile to result.
+ results.add(curfile);
+ // Keep list of sympathetic data mapfiles for cleaning info dir in next
+ // section. Make sure path is fully qualified for compare.
+ Path qualified = fs.makeQualified(mapfile);
+ mapfiles.add(qualified);
+ }
- Vector<HStoreFile> results = new Vector<HStoreFile>();
Path mapdir = HStoreFile.getMapDir(dir, regionName, colFamily);
-
+ // List paths by experience returns fully qualified names -- at least when
+ // running on a mini hdfs cluster.
Path datfiles[] = fs.listPaths(mapdir);
- for(int i = 0; i < datfiles.length; i++) {
- String name = datfiles[i].getName();
-
- if(name.startsWith(HSTORE_DATFILE_PREFIX)) {
- Long fileId =
- Long.parseLong(name.substring(HSTORE_DATFILE_PREFIX.length()));
-
- HStoreFile curfile =
- new HStoreFile(conf, dir, regionName, colFamily, fileId);
-
- Path mapfile = curfile.getMapFilePath();
- Path infofile = curfile.getInfoFilePath();
-
- if(fs.exists(infofile)) {
- results.add(curfile);
-
- } else {
- fs.delete(mapfile);
- }
- }
- }
-
- Path infodir = HStoreFile.getInfoDir(dir, regionName, colFamily);
- Path infofiles[] = fs.listPaths(infodir);
- for(int i = 0; i < infofiles.length; i++) {
- String name = infofiles[i].getName();
-
- if(name.startsWith(HSTORE_INFOFILE_PREFIX)) {
- long fileId =
- Long.parseLong(name.substring(HSTORE_INFOFILE_PREFIX.length()));
-
- HStoreFile curfile =
- new HStoreFile(conf, dir, regionName, colFamily, fileId);
-
- Path mapfile = curfile.getMapFilePath();
-
- if(! fs.exists(mapfile)) {
- fs.delete(curfile.getInfoFilePath());
- }
+ for (int i = 0; i < datfiles.length; i++) {
+ // If does not have sympathetic info file, delete.
+ if (!mapfiles.contains(fs.makeQualified(datfiles[i]))) {
+ fs.delete(datfiles[i]);
}
}
return results;
}
+
+ /**
+ * @param p Path to check.
+ * @return True if the path has format of a HStoreFile reference.
+ */
+ static boolean isReference(final Path p) {
+ return isReference(p, REF_NAME_PARSER.matcher(p.getName()));
+ }
+
+ private static boolean isReference(final Path p, final Matcher m) {
+ if (m == null || !m.matches()) {
+ LOG.warn("Failed match of store file name " + p.toString());
+ }
+ return m.groupCount() > 1 && m.group(2) != null;
+ }
// File handling
- /**
- * Break this HStoreFile file into two new parts, which live in different
- * brand-new HRegions.
+ /*
+ * Split by making two new store files that reference top and bottom regions
+ * of original store file.
+ * @param midKey
+ * @param dstA
+ * @param dstB
+ * @param fs
+ * @param c
+ * @throws IOException
*
* @param midKey the key which will be the starting key of the second region
* @param dstA the file which will contain keys from the start of the source
@@ -253,57 +440,66 @@
* @param c configuration
* @throws IOException
*/
- void splitStoreFile(Text midKey, HStoreFile dstA, HStoreFile dstB,
- FileSystem fs, Configuration c) throws IOException {
-
- // Copy the appropriate tuples to one MapFile or the other.
-
- MapFile.Reader in = new MapFile.Reader(fs, getMapFilePath().toString(), c);
- try {
- MapFile.Writer outA = new MapFile.Writer(c, fs,
- dstA.getMapFilePath().toString(), HStoreKey.class,
- ImmutableBytesWritable.class);
-
- try {
- MapFile.Writer outB = new MapFile.Writer(c, fs,
- dstB.getMapFilePath().toString(), HStoreKey.class,
- ImmutableBytesWritable.class);
-
- try {
- long count = 0;
- HStoreKey readkey = new HStoreKey();
- ImmutableBytesWritable readval = new ImmutableBytesWritable();
-
- while(in.next(readkey, readval)) {
- if(readkey.getRow().compareTo(midKey) < 0) {
- outA.append(readkey, readval);
- } else {
- outB.append(readkey, readval);
- }
- if (LOG.isDebugEnabled()) {
- count++;
- if ((count % 10000) == 0) {
- LOG.debug("Write " + count + " records");
- }
- }
- }
-
- } finally {
- outB.close();
- }
-
- } finally {
- outA.close();
- }
-
- } finally {
- in.close();
+ void splitStoreFile(final HStoreFile dstA, final HStoreFile dstB,
+ final FileSystem fs)
+ throws IOException {
+ dstA.writeReferenceFiles(fs);
+ dstB.writeReferenceFiles(fs);
+ }
+
+ void writeReferenceFiles(final FileSystem fs)
+ throws IOException {
+ createOrFail(fs, getMapFilePath());
+ writeSplitInfo(fs);
+ }
+
+ /*
+ * If reference, create and write the remote store file id, the midkey and
+ * whether we're going against the top file region of the referent out to
+ * the info file.
+ * @param p Path to info file.
+ * @param hsf
+ * @param fs
+ * @throws IOException
+ */
+ private void writeSplitInfo(final FileSystem fs)
+ throws IOException {
+ Path p = getInfoFilePath();
+ if (fs.exists(p)) {
+ throw new IOException("File already exists " + p.toString());
}
+ FSDataOutputStream out = fs.create(p);
+ getReference().getRegionName().write(out);
+ getReference().getMidkey().write(out);
+ out.writeLong(getReference().getFileId());
+ out.writeBoolean(isTopFileRegion(getReference().getFileRegion()));
+ out.close();
+ }
+
+ /*
+ * @see writeSplitInfo(Path p, HStoreFile hsf, FileSystem fs)
+ */
+ static Reference readSplitInfo(final Path p, final FileSystem fs)
+ throws IOException {
+ FSDataInputStream in = fs.open(p);
+ Text rn = new Text();
+ rn.readFields(in);
+ HStoreKey midkey = new HStoreKey();
+ midkey.readFields(in);
+ long fid = in.readLong();
+ boolean tmp = in.readBoolean();
+ return new Reference(rn, fid, midkey, tmp? Range.top: Range.bottom);
+
+ }
- // Build an InfoFile for each output
- long seqid = loadInfo(fs);
- dstA.writeInfo(fs, seqid);
- dstB.writeInfo(fs, seqid);
+ private void createOrFail(final FileSystem fs, final Path p)
+ throws IOException {
+ if (fs.exists(p)) {
+ throw new IOException("File already exists " + p.toString());
+ }
+ if (!fs.createNewFile(p)) {
+ throw new IOException("Failed create of " + p);
+ }
}
/**
@@ -315,10 +511,9 @@
* @throws IOException
*/
void mergeStoreFiles(Vector<HStoreFile> srcFiles, FileSystem fs,
- Configuration conf) throws IOException {
-
+ @SuppressWarnings("hiding") Configuration conf)
+ throws IOException {
// Copy all the source MapFile tuples into this HSF's MapFile
-
MapFile.Writer out = new MapFile.Writer(conf, fs,
getMapFilePath().toString(),
HStoreKey.class, ImmutableBytesWritable.class);
@@ -327,7 +522,6 @@
for(HStoreFile src: srcFiles) {
MapFile.Reader in =
new MapFile.Reader(fs, src.getMapFilePath().toString(), conf);
-
try {
HStoreKey readkey = new HStoreKey();
ImmutableBytesWritable readval = new ImmutableBytesWritable();
@@ -343,12 +537,10 @@
} finally {
out.close();
}
-
// Build a unified InfoFile from the source InfoFiles.
long unifiedSeqId = -1;
- for(Iterator<HStoreFile> it = srcFiles.iterator(); it.hasNext(); ) {
- HStoreFile hsf = it.next();
+ for(HStoreFile hsf: srcFiles) {
long curSeqId = hsf.loadInfo(fs);
if(curSeqId > unifiedSeqId) {
unifiedSeqId = curSeqId;
@@ -358,14 +550,17 @@
}
/**
- * Reads in an info file, and gives it a unique ID.
+ * Reads in an info file
*
* @param fs file system
- * @return new unique id
+ * @return The sequence id contained in the info file
* @throws IOException
*/
long loadInfo(FileSystem fs) throws IOException {
- Path p = getInfoFilePath();
+ Path p = isReference()?
+ getInfoFilePath(this.reference.getRegionName(),
+ this.reference.getFileId()):
+ getInfoFilePath();
DataInputStream in = new DataInputStream(fs.open(p));
try {
byte flag = in.readByte();
@@ -388,7 +583,6 @@
void writeInfo(FileSystem fs, long infonum) throws IOException {
Path p = getInfoFilePath();
DataOutputStream out = new DataOutputStream(fs.create(p));
-
try {
out.writeByte(INFO_SEQ_NUM);
out.writeLong(infonum);
@@ -397,8 +591,327 @@
out.close();
}
}
+
+ /**
+ * Delete store map files.
+ * @throws IOException
+ */
+ public void delete() throws IOException {
+ delete(getMapFilePath());
+ delete(getInfoFilePath());
+ }
+
+ private void delete(final Path p) throws IOException {
+ p.getFileSystem(this.conf).delete(p);
+ }
+
+ /**
+ * Renames the mapfiles and info directories under the passed
+ * <code>hsf</code> directory.
+ * @param fs
+ * @param hsf
+ * @throws IOException
+ */
+ public void rename(final FileSystem fs, final HStoreFile hsf)
+ throws IOException {
+ fs.rename(getMapFilePath(), hsf.getMapFilePath());
+ fs.rename(getInfoFilePath(), hsf.getInfoFilePath());
+ }
+
+ /**
+ * A facade for a {@link MapFile.Reader} that serves up either the top or
+ * bottom half of a MapFile (where 'bottom' is the first half of the file
+ * containing the keys that sort lowest and 'top' is the second half of the
+ * file with keys that sort greater than those of the bottom half).
+ * Subclasses BloomFilterMapFile.Reader in case
+ *
+ * <p>This file is not splitable. Calls to {@link #midKey()} return null.
+ */
+ static class HalfMapFileReader extends BloomFilterMapFile.Reader {
+ private final boolean top;
+ private final WritableComparable midkey;
+ private boolean topFirstNextCall = true;
+
+ HalfMapFileReader(final FileSystem fs, final String dirName,
+ final Configuration conf, final Range r,
+ final WritableComparable midKey)
+ throws IOException {
+ this(fs, dirName, conf, r, midKey, null);
+ }
+
+ HalfMapFileReader(final FileSystem fs, final String dirName,
+ final Configuration conf, final Range r,
+ final WritableComparable midKey, final Filter filter)
+ throws IOException {
+ super(fs, dirName, conf, filter);
+ this.top = isTopFileRegion(r);
+ this.midkey = midKey;
+ }
+
+ @SuppressWarnings("unchecked")
+ private void checkKey(final WritableComparable key)
+ throws IOException {
+ if (this.top) {
+ if (key.compareTo(this.midkey) < 0) {
+ throw new IOException("Illegal Access: Key is less than midKey of " +
+ "backing mapfile");
+ }
+ } else if (key.compareTo(this.midkey) >= 0) {
+ throw new IOException("Illegal Access: Key is greater than or equal " +
+ "to midKey of backing mapfile");
+ }
+ }
+
+ @SuppressWarnings({ "unused"})
+ @Override
+ public synchronized void finalKey(WritableComparable key)
+ throws IOException {
+ throw new UnsupportedOperationException("Unsupported");
+ }
+
+ @Override
+ public synchronized Writable get(WritableComparable key, Writable val)
+ throws IOException {
+ checkKey(key);
+ return super.get(key, val);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public synchronized WritableComparable getClosest(WritableComparable key,
+ Writable val)
+ throws IOException {
+ if (this.top) {
+ if (key.compareTo(this.midkey) < 0) {
+ return this.midkey;
+ }
+ } else if (key.compareTo(this.midkey) >= 0) {
+ // Contract says return null if EOF.
+ return null;
+ }
+ return super.getClosest(key, val);
+ }
+
+ @SuppressWarnings("unused")
+ @Override
+ public synchronized WritableComparable midKey() throws IOException {
+ // Returns null to indicate file is not splitable.
+ return null;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public synchronized boolean next(WritableComparable key, Writable val)
+ throws IOException {
+ if (this.top && this.topFirstNextCall) {
+ this.topFirstNextCall = false;
+ return doFirstNextProcessing(key, val);
+ }
+ boolean result = super.next(key, val);
+ if (!top && key.compareTo(this.midkey) >= 0) {
+ result = false;
+ }
+ return result;
+ }
+
+ private boolean doFirstNextProcessing(WritableComparable key, Writable val)
+ throws IOException {
+ // Seek to midkey. Midkey may not exist in this file. That should be
+ // fine. Then we'll either be positioned at end or start of file.
+ WritableComparable nearest = getClosest(this.midkey, val);
+ // Now copy the mid key into the passed key.
+ if (nearest != null) {
+ Writables.copyWritable(nearest, key);
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public synchronized void reset() throws IOException {
+ if (top) {
+ this.topFirstNextCall = true;
+ seek(this.midkey);
+ return;
+ }
+ super.reset();
+ }
+
+ @Override
+ public synchronized boolean seek(WritableComparable key)
+ throws IOException {
+ checkKey(key);
+ return super.seek(key);
+ }
+ }
+
+ /**
+ * On write, all keys are added to a bloom filter. On read, all keys are
+ * tested first against bloom filter. Keys are HStoreKey. If passed bloom
+ * filter is null, just passes invocation to parent.
+ */
+ static class BloomFilterMapFile extends MapFile {
+ protected BloomFilterMapFile() {
+ super();
+ }
+
+ static class Reader extends MapFile.Reader {
+ private final Filter bloomFilter;
+
+ public Reader(FileSystem fs, String dirName, Configuration conf,
+ final Filter filter)
+ throws IOException {
+ super(fs, dirName, conf);
+ this.bloomFilter = filter;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public Writable get(WritableComparable key, Writable val)
+ throws IOException {
+ if (this.bloomFilter == null) {
+ return super.get(key, val);
+ }
+ if(this.bloomFilter.membershipTest(getBloomFilterKey(key))) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("bloom filter reported that key exists");
+ }
+ return super.get(key, val);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("bloom filter reported that key does not exist");
+ }
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public WritableComparable getClosest(WritableComparable key,
+ Writable val)
+ throws IOException {
+ if (this.bloomFilter == null) {
+ return super.getClosest(key, val);
+ }
+ // Note - the key being passed to us is always a HStoreKey
+ if(this.bloomFilter.membershipTest(getBloomFilterKey(key))) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("bloom filter reported that key exists");
+ }
+ return super.getClosest(key, val);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("bloom filter reported that key does not exist");
+ }
+ return null;
+ }
+ }
+
+ static class Writer extends MapFile.Writer {
+ private final Filter bloomFilter;
+
+
+ @SuppressWarnings("unchecked")
+ public Writer(Configuration conf, FileSystem fs, String dirName,
+ Class keyClass, Class valClass,
+ SequenceFile.CompressionType compression, final Filter filter)
+ throws IOException {
+ super(conf, fs, dirName, keyClass, valClass, compression);
+ this.bloomFilter = filter;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void append(WritableComparable key, Writable val)
+ throws IOException {
+ if (this.bloomFilter != null) {
+ this.bloomFilter.add(getBloomFilterKey(key));
+ }
+ super.append(key, val);
+ }
+ }
+ }
+
+ /**
+ * Custom bloom filter key maker.
+ * @param key
+ * @return Key made of bytes of row and column only.
+ * @throws IOException
+ */
+ static Key getBloomFilterKey(WritableComparable key)
+ throws IOException {
+ HStoreKey hsk = (HStoreKey)key;
+ byte [] bytes = null;
+ try {
+ bytes = (hsk.getRow().toString() + hsk.getColumn().toString()).
+ getBytes(UTF8_ENCODING);
+ } catch (UnsupportedEncodingException e) {
+ throw new IOException(e.toString());
+ }
+ return new Key(bytes);
+ }
+
+ /**
+ * Get reader for the store file map file.
+ * Client is responsible for closing file when done.
+ * @param fs
+ * @param bloomFilter If null, no filtering is done.
+ * @return MapFile.Reader
+ * @throws IOException
+ */
+ public synchronized MapFile.Reader getReader(final FileSystem fs,
+ final Filter bloomFilter)
+ throws IOException {
+ return isReference()?
+ new HStoreFile.HalfMapFileReader(fs,
+ getMapFilePath(getReference().getRegionName(),
+ getReference().getFileId()).toString(),
+ this.conf, getReference().getFileRegion(), getReference().getMidkey(),
+ bloomFilter):
+ new BloomFilterMapFile.Reader(fs, getMapFilePath().toString(),
+ this.conf, bloomFilter);
+ }
+
+ /**
+ * Get a store file writer.
+ * Client is responsible for closing file when done.
+ * @param fs
+ * @param compression Pass <code>SequenceFile.CompressionType.NONE</code>
+ * for none.
+ * @param bloomFilter If null, no filtering is done.
+ * @return MapFile.Writer
+ * @throws IOException
+ */
+ public MapFile.Writer getWriter(final FileSystem fs,
+ final SequenceFile.CompressionType compression,
+ final Filter bloomFilter)
+ throws IOException {
+ if (isReference()) {
+ throw new IOException("Illegal Access: Cannot get a writer on a" +
+ "HStoreFile reference");
+ }
+ return new BloomFilterMapFile.Writer(conf, fs,
+ getMapFilePath().toString(), HStoreKey.class,
+ ImmutableBytesWritable.class, compression, bloomFilter);
+ }
+
+ /**
+ * @return Length of the store map file. If a reference, size is
+ * approximation.
+ * @throws IOException
+ */
+ public long length() throws IOException {
+ Path p = new Path(getMapFilePath(getReference()), MapFile.DATA_FILE_NAME);
+ long l = p.getFileSystem(this.conf).getFileStatus(p).getLen();
+ return (isReference())? l / 2: l;
+ }
- /** {@inheritDoc} */
+ @Override
+ public String toString() {
+ return this.regionName.toString() + "/" + this.colFamily.toString() +
+ "/" + this.fileId +
+ (isReference()? "/" + this.reference.toString(): "");
+ }
+
@Override
public boolean equals(Object o) {
return this.compareTo(o) == 0;
@@ -419,9 +932,13 @@
/** {@inheritDoc} */
public void write(DataOutput out) throws IOException {
out.writeUTF(dir.toString());
- regionName.write(out);
- colFamily.write(out);
+ this.regionName.write(out);
+ this.colFamily.write(out);
out.writeLong(fileId);
+ out.writeBoolean(isReference());
+ if (isReference()) {
+ this.reference.write(out);
+ }
}
/** {@inheritDoc} */
@@ -430,6 +947,12 @@
this.regionName.readFields(in);
this.colFamily.readFields(in);
this.fileId = in.readLong();
+ this.reference = null;
+ boolean isReferent = in.readBoolean();
+ this.reference = new HStoreFile.Reference();
+ if (isReferent) {
+ this.reference.readFields(in);
+ }
}
// Comparable
@@ -454,4 +977,4 @@
}
return result;
}
-}
\ No newline at end of file
+}
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java Wed Aug 8 13:30:13 2007
@@ -131,6 +131,11 @@
throw new IllegalStateException("update in progress");
}
}
+
+
+ public Text getTableName() {
+ return this.tableName;
+ }
/**
* Gets the starting row key for every region in the currently open table
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Keying.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Keying.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Keying.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Keying.java Wed Aug 8 13:30:13 2007
@@ -19,10 +19,18 @@
*/
package org.apache.hadoop.hbase.util;
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HStoreKey;
+
/**
* Utility creating hbase friendly keys.
* Use fabricating row names or column qualifiers.
@@ -110,5 +118,62 @@
sb.insert(0, next);
}
return sb.toString();
+ }
+
+ /**
+ * @param i
+ * @return <code>i</code> as byte array.
+ */
+ public static byte[] intToBytes(final int i){
+ ByteBuffer buffer = ByteBuffer.allocate(Integer.SIZE);
+ buffer.putInt(i);
+ return buffer.array();
+ }
+
+ /**
+ * @param l
+ * @return <code>i</code> as byte array.
+ */
+ public static byte[] longToBytes(final long l){
+ ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE);
+ buffer.putLong(l);
+ return buffer.array();
+ }
+
+ /**
+ * Returns row and column bytes out of an HStoreKey.
+ * @param hsk Store key.
+ * @throws UnsupportedEncodingException
+ */
+ public static byte[] getBytes(final HStoreKey hsk)
+ throws UnsupportedEncodingException {
+ StringBuilder s = new StringBuilder(hsk.getRow().toString());
+ s.append(hsk.getColumn().toString());
+ return s.toString().getBytes(HConstants.UTF8_ENCODING);
+ }
+
+ /**
+ * @param bytes
+ * @return String made of the bytes or null if bytes are null.
+ * @throws UnsupportedEncodingException
+ */
+ public static String bytesToString(final byte [] bytes)
+ throws UnsupportedEncodingException {
+ if(bytes == null) {
+ return null;
+ }
+ return new String(bytes, HConstants.UTF8_ENCODING);
+ }
+
+ public static long bytesToLong(final byte [] bytes) throws IOException {
+ long result = -1;
+ DataInputStream dis = null;
+ try {
+ dis = new DataInputStream(new ByteArrayInputStream(bytes));
+ result = dis.readLong();
+ } finally {
+ dis.close();
+ }
+ return result;
}
}
Added: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Writables.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Writables.java?view=auto&rev=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Writables.java (added)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Writables.java Wed Aug 8 13:30:13 2007
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.Writable;
+
+
+public class Writables {
+ /**
+ * @param w
+ * @return The bytes of <code>w</code> gotten by running its
+ * {@link Writable#write(java.io.DataOutput)} method.
+ * @throws IOException
+ * @see #getWritable(byte[], Writable)
+ */
+ public static byte [] getBytes(final Writable w) throws IOException {
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(byteStream);
+ try {
+ w.write(out);
+ out.close();
+ out = null;
+ return byteStream.toByteArray();
+ } finally {
+ if (out != null) {
+ out.close();
+ }
+ }
+ }
+
+ /**
+ * Set bytes into the passed Writable by calling its
+ * {@link Writable#readFields(java.io.DataInput)}.
+ * @param bytes
+ * @param w An empty Writable (usually made by calling the null-arg
+ * constructor).
+ * @return The passed Writable after its readFields has been called fed
+ * by the passed <code>bytes</code> array or null if passed null or
+ * empty <code>bytes</code>.
+ * @throws IOException
+ */
+ public static Writable getWritable(final byte [] bytes, final Writable w)
+ throws IOException {
+ if (bytes == null || bytes.length == 0) {
+ throw new IOException("Con't build a writable with empty bytes array");
+ }
+ DataInputBuffer in = new DataInputBuffer();
+ try {
+ in.reset(bytes, bytes.length);
+ w.readFields(in);
+ return w;
+ } finally {
+ in.close();
+ }
+ }
+
+ /**
+ * Copy one Writable to another. Copies bytes using data streams.
+ * @param src Source Writable
+ * @param tgt Target Writable
+ * @return The target Writable.
+ * @throws IOException
+ */
+ public static Writable copyWritable(final Writable src, final Writable tgt)
+ throws IOException {
+ byte [] bytes = getBytes(src);
+ DataInputStream dis = null;
+ try {
+ dis = new DataInputStream(new ByteArrayInputStream(bytes));
+ tgt.readFields(dis);
+ } finally {
+ dis.close();
+ }
+ return tgt;
+ }
+}
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/HBaseTestCase.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/HBaseTestCase.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/HBaseTestCase.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/HBaseTestCase.java Wed Aug 8 13:30:13 2007
@@ -56,7 +56,7 @@
HTableDescriptor desc, long regionId, Text startKey, Text endKey)
throws IOException {
HRegionInfo info = new HRegionInfo(regionId, desc, startKey, endKey);
- Path regionDir = HStoreFile.getHRegionDir(dir, info.regionName);
+ Path regionDir = HRegion.getRegionDir(dir, info.regionName);
FileSystem fs = dir.getFileSystem(c);
fs.mkdirs(regionDir);
return new HRegion(dir,
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java Wed Aug 8 13:30:13 2007
@@ -33,18 +33,24 @@
/**
* This class creates a single process HBase cluster for junit testing.
* One thread is created for each server.
+ *
+ * <p>TestCases do not need to subclass to start a HBaseCluster. Call
+ * {@link #startMaster(Configuration)} and
+ * {@link #startRegionServers(Configuration, int)} to startup master and
+ * region servers. Save off the returned values and pass them to
+ * {@link #shutdown(org.apache.hadoop.hbase.MiniHBaseCluster.MasterThread, List)}
+ * to shut it all down when done.
+ *
*/
public class MiniHBaseCluster implements HConstants {
- private static final Logger LOG =
+ static final Logger LOG =
Logger.getLogger(MiniHBaseCluster.class.getName());
private Configuration conf;
private MiniDFSCluster cluster;
private FileSystem fs;
private Path parentdir;
- private HMaster master = null;
- private Thread masterThread = null;
- List<HRegionServer> regionServers;
- List<Thread> regionThreads;
+ private MasterThread masterThread = null;
+ ArrayList<RegionServerThread> regionThreads;
private boolean deleteOnExit = true;
/**
@@ -87,8 +93,6 @@
this.conf = conf;
this.cluster = dfsCluster;
- this.regionServers = new ArrayList<HRegionServer>(nRegionNodes);
- this.regionThreads = new ArrayList<Thread>(nRegionNodes);
init(nRegionNodes);
}
@@ -108,13 +112,9 @@
throws IOException {
this.conf = conf;
this.deleteOnExit = deleteOnExit;
- this.regionServers = new ArrayList<HRegionServer>(nRegionNodes);
- this.regionThreads = new ArrayList<Thread>(nRegionNodes);
-
if (miniHdfsFilesystem) {
try {
this.cluster = new MiniDFSCluster(this.conf, 2, format, (String[])null);
-
} catch(Throwable t) {
LOG.error("Failed setup of mini dfs cluster", t);
t.printStackTrace();
@@ -124,45 +124,123 @@
init(nRegionNodes);
}
- private void init(int nRegionNodes) throws IOException {
+ private void init(final int nRegionNodes)
+ throws IOException {
try {
try {
this.fs = FileSystem.get(conf);
this.parentdir = new Path(conf.get(HBASE_DIR, DEFAULT_HBASE_DIR));
fs.mkdirs(parentdir);
-
} catch(IOException e) {
LOG.error("Failed setup of FileSystem", e);
throw e;
}
-
- if(this.conf.get(MASTER_ADDRESS) == null) {
- this.conf.set(MASTER_ADDRESS, "localhost:0");
- }
-
- // Create the master
- this.master = new HMaster(conf);
- this.masterThread = new Thread(this.master, "HMaster");
-
- // Start up the master
- LOG.info("Starting HMaster");
- masterThread.start();
-
- // Set the master's port for the HRegionServers
- String address = master.getMasterAddress().toString();
- this.conf.set(MASTER_ADDRESS, address);
-
- // Start the HRegionServers. Always have regionservers come up on
- // port '0' so there won't be clashes over default port as unit tests
- // start/stop ports at different times during the life of the test.
- this.conf.set(REGIONSERVER_ADDRESS, DEFAULT_HOST + ":0");
- LOG.info("Starting HRegionServers");
- startRegionServers(nRegionNodes);
+ this.masterThread = startMaster(this.conf);
+ this.regionThreads = startRegionServers(this.conf, nRegionNodes);
} catch(IOException e) {
shutdown();
throw e;
}
}
+
+ public static class MasterThread extends Thread {
+ private final HMaster master;
+ MasterThread(final HMaster m) {
+ super(m, "Master:" + m.getMasterAddress().toString());
+ this.master = m;
+ }
+ @Override
+ public void run() {
+ LOG.info("Starting " + getName());
+ super.run();
+ }
+ public HMaster getMaster() {
+ return this.master;
+ }
+ }
+
+ public static class RegionServerThread extends Thread {
+ private final HRegionServer regionServer;
+ RegionServerThread(final HRegionServer r, final int index) {
+ super(r, "RegionServer:" + index);
+ this.regionServer = r;
+ }
+ @Override
+ public void run() {
+ LOG.info("Starting " + getName());
+ super.run();
+ }
+ public HRegionServer getRegionServer() {
+ return this.regionServer;
+ }
+ }
+
+ /**
+ * Use this method to start a master.
+ * If you want to start an hbase cluster
+ * without subclassing this test case, run this method and
+ * {@link #startRegionServers(Configuration, int)} to start servers.
+ * Call {@link #shutdown(org.apache.hadoop.hbase.MiniHBaseCluster.MasterThread, List)}
+ * to shut them down.
+ * @param c
+ * @return Thread running the master.
+ * @throws IOException
+ * @see #startRegionServers(Configuration, int)
+ * @see #shutdown(org.apache.hadoop.hbase.MiniHBaseCluster.MasterThread, List)
+ */
+ public static MasterThread startMaster(final Configuration c)
+ throws IOException {
+ if(c.get(MASTER_ADDRESS) == null) {
+ c.set(MASTER_ADDRESS, "localhost:0");
+ }
+ // Create the master
+ final HMaster m = new HMaster(c);
+ MasterThread masterThread = new MasterThread(m);
+ // Start up the master
+ masterThread.start();
+ // Set the master's port for the HRegionServers
+ c.set(MASTER_ADDRESS, m.getMasterAddress().toString());
+ return masterThread;
+ }
+
+ /**
+ * @param c
+ * @param count
+ * @return List of region server threads started. Synchronize on the
+ * returned list when iterating to avoid ConcurrentModificationExceptions.
+ * @throws IOException
+ * @see #startMaster(Configuration)
+ */
+ public static ArrayList<RegionServerThread> startRegionServers(
+ final Configuration c, final int count)
+ throws IOException {
+ // Start the HRegionServers. Always have regionservers come up on
+ // port '0' so there won't be clashes over default port as unit tests
+ // start/stop ports at different times during the life of the test.
+ c.set(REGIONSERVER_ADDRESS, DEFAULT_HOST + ":0");
+ LOG.info("Starting HRegionServers");
+ ArrayList<RegionServerThread> threads =
+ new ArrayList<RegionServerThread>();
+ for(int i = 0; i < count; i++) {
+ threads.add(startRegionServer(c, i));
+ }
+ return threads;
+ }
+
+ public void startRegionServer() throws IOException {
+ RegionServerThread t =
+ startRegionServer(this.conf, this.regionThreads.size());
+ this.regionThreads.add(t);
+ }
+
+ private static RegionServerThread startRegionServer(final Configuration c,
+ final int index)
+ throws IOException {
+ final HRegionServer hsr = new HRegionServer(c);
+ RegionServerThread t = new RegionServerThread(hsr, index);
+ t.start();
+ return t;
+ }
/**
* Get the cluster on which this HBase cluster is running
@@ -173,27 +251,12 @@
return cluster;
}
- private void startRegionServers(final int nRegionNodes)
- throws IOException {
- for(int i = 0; i < nRegionNodes; i++) {
- startRegionServer();
- }
- }
-
- void startRegionServer() throws IOException {
- HRegionServer hsr = new HRegionServer(this.conf);
- this.regionServers.add(hsr);
- Thread t = new Thread(hsr, "HRegionServer-" + this.regionServers.size());
- t.start();
- this.regionThreads.add(t);
- }
-
/**
* @return Returns the rpc address actually used by the master server, because
* the supplied port is not necessarily the actual port used.
*/
public HServerAddress getHMasterAddress() {
- return master.getMasterAddress();
+ return this.masterThread.getMaster().getMasterAddress();
}
/**
@@ -202,7 +265,9 @@
* @param serverNumber
*/
public void abortRegionServer(int serverNumber) {
- HRegionServer server = this.regionServers.remove(serverNumber);
+ HRegionServer server =
+ this.regionThreads.get(serverNumber).getRegionServer();
+ LOG.info("Aborting " + server.serverInfo.toString());
server.abort();
}
@@ -211,53 +276,77 @@
*
* @param serverNumber
*/
- public void stopRegionServer(int serverNumber) {
- HRegionServer server = this.regionServers.remove(serverNumber);
+ public HRegionServer stopRegionServer(int serverNumber) {
+ HRegionServer server =
+ this.regionThreads.get(serverNumber).getRegionServer();
+ LOG.info("Stopping " + server.toString());
server.stop();
+ return server;
}
/**
* Wait for the specified region server to stop
- *
+ * Removes this thread from list of running threads.
* @param serverNumber
*/
public void waitOnRegionServer(int serverNumber) {
- Thread regionServerThread = this.regionThreads.remove(serverNumber);
+ RegionServerThread regionServerThread =
+ this.regionThreads.remove(serverNumber);
try {
+ LOG.info("Waiting on " +
+ regionServerThread.getRegionServer().serverInfo.toString());
regionServerThread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
-
- /** Shut down the HBase cluster */
- public void shutdown() {
- LOG.info("Shutting down the HBase Cluster");
- for(HRegionServer hsr: this.regionServers) {
- hsr.stop();
- }
- if(master != null) {
- master.shutdown();
- }
- for(Thread t: this.regionThreads) {
- if (t.isAlive()) {
- try {
- t.join();
- } catch (InterruptedException e) {
- // continue
+
+ /**
+ * Shut down HBase cluster started by calling
+ * {@link #startMaster(Configuration)} and then
+ * {@link #startRegionServers(Configuration, int)};
+ * @param masterThread
+ * @param regionServerThreads
+ */
+ public static void shutdown(final MasterThread masterThread,
+ final List<RegionServerThread> regionServerThreads) {
+ LOG.info("Shutting down HBase Cluster");
+ /** This is not needed. Remove.
+ for(RegionServerThread hsr: regionServerThreads) {
+ hsr.getRegionServer().stop();
+ }
+ */
+ if(masterThread != null) {
+ masterThread.getMaster().shutdown();
+ }
+ synchronized(regionServerThreads) {
+ if (regionServerThreads != null) {
+ for(Thread t: regionServerThreads) {
+ if (t.isAlive()) {
+ try {
+ t.join();
+ } catch (InterruptedException e) {
+ // continue
+ }
+ }
}
}
}
if (masterThread != null) {
try {
masterThread.join();
-
} catch(InterruptedException e) {
// continue
}
}
- LOG.info("HBase Cluster shutdown complete");
-
+ LOG.info("Shutdown " +
+ ((masterThread != null)? masterThread.getName(): "0 masters") + " " +
+ ((regionServerThreads == null)? 0: regionServerThreads.size()) +
+ " region server(s)");
+ }
+
+ void shutdown() {
+ shutdown(this.masterThread, this.regionThreads);
// Close the file system. Will complain if files open so helps w/ leaks.
try {
this.cluster.getFileSystem().close();
@@ -285,4 +374,4 @@
}
f.delete();
}
-}
+}
\ No newline at end of file
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/StaticTestEnvironment.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/StaticTestEnvironment.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/StaticTestEnvironment.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/StaticTestEnvironment.java Wed Aug 8 13:30:13 2007
@@ -63,7 +63,7 @@
debugging = true;
Logger rootLogger = Logger.getRootLogger();
- rootLogger.setLevel(Level.WARN);
+ // rootLogger.setLevel(Level.WARN);
Level logLevel = Level.INFO;
value = System.getenv("LOGGING_LEVEL");
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestBatchUpdate.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestBatchUpdate.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestBatchUpdate.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestBatchUpdate.java Wed Aug 8 13:30:13 2007
@@ -35,14 +35,10 @@
private HTableDescriptor desc = null;
private HTable table = null;
- /** constructor */
- public TestBatchUpdate() {
- try {
- value = "abcd".getBytes(HConstants.UTF8_ENCODING);
-
- } catch (UnsupportedEncodingException e) {
- fail();
- }
+ /** constructor
+ * @throws UnsupportedEncodingException */
+ public TestBatchUpdate() throws UnsupportedEncodingException {
+ value = "abcd".getBytes(HConstants.UTF8_ENCODING);
}
/**
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestGet.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestGet.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestGet.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestGet.java Wed Aug 8 13:30:13 2007
@@ -88,7 +88,7 @@
desc.addFamily(new HColumnDescriptor(HConstants.COLUMN_FAMILY.toString()));
HRegionInfo info = new HRegionInfo(0L, desc, null, null);
- Path regionDir = HStoreFile.getHRegionDir(dir, info.regionName);
+ Path regionDir = HRegion.getRegionDir(dir, info.regionName);
fs.mkdirs(regionDir);
HLog log = new HLog(fs, new Path(regionDir, "log"), conf);
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHRegion.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHRegion.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHRegion.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHRegion.java Wed Aug 8 13:30:13 2007
@@ -40,7 +40,8 @@
* HRegions or in the HBaseMaster, so only basic testing is possible.
*/
public class TestHRegion extends HBaseTestCase implements RegionUnavailableListener {
- Logger LOG = Logger.getLogger(this.getClass().getName());
+ private static final Logger LOG =
+ Logger.getLogger(TestHRegion.class.getName());
/** Constructor */
public TestHRegion() {
@@ -51,8 +52,9 @@
* Since all the "tests" depend on the results of the previous test, they are
* not Junit tests that can stand alone. Consequently we have a single Junit
* test that runs the "sub-tests" as private methods.
+ * @throws IOException
*/
- public void testHRegion() {
+ public void testHRegion() throws IOException {
try {
setup();
locks();
@@ -63,13 +65,10 @@
splitAndMerge();
read();
cleanup();
-
- } catch(Exception e) {
+ } finally {
if(cluster != null) {
cluster.shutdown();
}
- e.printStackTrace();
- fail();
}
}
@@ -674,7 +673,7 @@
anchorFetched++;
} else {
- System.out.println(col);
+ System.out.println("UNEXPECTED COLUMN " + col);
}
}
curVals.clear();
Added: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHStoreFile.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHStoreFile.java?view=auto&rev=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHStoreFile.java (added)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHStoreFile.java Wed Aug 8 13:30:13 2007
@@ -0,0 +1,382 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+import junit.framework.TestCase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.io.MapFile;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+
+public class TestHStoreFile extends TestCase {
+ static final Log LOG = LogFactory.getLog(TestHStoreFile.class);
+ private static String DIR = System.getProperty("test.build.data", ".");
+ private static final char FIRST_CHAR = 'a';
+ private static final char LAST_CHAR = 'z';
+ private FileSystem fs;
+ private Configuration conf;
+ private Path dir = null;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ this.conf = new HBaseConfiguration();
+ this.fs = FileSystem.getLocal(this.conf);
+ this.dir = new Path(DIR, getName());
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ if (this.fs.exists(this.dir)) {
+ this.fs.delete(this.dir);
+ }
+ super.tearDown();
+ }
+
+ private Path writeMapFile(final String name)
+ throws IOException {
+ Path path = new Path(DIR, name);
+ MapFile.Writer writer = new MapFile.Writer(this.conf, fs, path.toString(),
+ HStoreKey.class, ImmutableBytesWritable.class);
+ writeStoreFile(writer);
+ return path;
+ }
+
+ private Path writeSmallMapFile(final String name)
+ throws IOException {
+ Path path = new Path(DIR, name);
+ MapFile.Writer writer = new MapFile.Writer(this.conf, fs, path.toString(),
+ HStoreKey.class, ImmutableBytesWritable.class);
+ try {
+ for (char d = FIRST_CHAR; d <= LAST_CHAR; d++) {
+ byte[] b = new byte[] {(byte)d};
+ Text t = new Text(new String(b));
+ writer.append(new HStoreKey(t, t, System.currentTimeMillis()),
+ new ImmutableBytesWritable(t.getBytes()));
+ }
+ } finally {
+ writer.close();
+ }
+ return path;
+ }
+
+ /*
+ * Writes HStoreKey and ImmutableBytes data to passed writer and
+ * then closes it.
+ * @param writer
+ * @throws IOException
+ */
+ private void writeStoreFile(final MapFile.Writer writer)
+ throws IOException {
+ try {
+ for (char d = FIRST_CHAR; d <= LAST_CHAR; d++) {
+ for (char e = FIRST_CHAR; e <= LAST_CHAR; e++) {
+ byte[] b = new byte[] { (byte) d, (byte) e };
+ Text t = new Text(new String(b));
+ writer.append(new HStoreKey(t, t, System.currentTimeMillis()),
+ new ImmutableBytesWritable(t.getBytes()));
+ }
+ }
+ } finally {
+ writer.close();
+ }
+ }
+
+ /**
+ * Test that our mechanism of writing store files in one region to reference
+ * store files in other regions works.
+ * @throws IOException
+ */
+ public void testReference()
+ throws IOException {
+ // Make a store file and write data to it.
+ HStoreFile hsf = new HStoreFile(this.conf, this.dir, new Text(getName()),
+ new Text("colfamily"), 1234567890L);
+ MapFile.Writer writer =
+ hsf.getWriter(this.fs, SequenceFile.CompressionType.NONE, null);
+ writeStoreFile(writer);
+ MapFile.Reader reader = hsf.getReader(this.fs, null);
+ // Split on a row, not in middle of row. Midkey returned by reader
+ // may be in middle of row. Create new one with empty column and
+ // timestamp.
+ HStoreKey midkey = new HStoreKey(((HStoreKey)reader.midKey()).getRow());
+ HStoreKey hsk = new HStoreKey();
+ reader.finalKey(hsk);
+ Text finalKey = hsk.getRow();
+ // Make a reference for the bottom half of the just written file.
+ HStoreFile.Reference reference =
+ new HStoreFile.Reference(hsf.getRegionName(), hsf.getFileId(), midkey,
+ HStoreFile.Range.top);
+ HStoreFile refHsf = new HStoreFile(this.conf, new Path(DIR, getName()),
+ new Text(getName() + "_reference"), hsf.getColFamily(), 456,
+ reference);
+ // Assert that reference files are written and that we can write and
+ // read the info reference file at least.
+ refHsf.writeReferenceFiles(this.fs);
+ assertTrue(this.fs.exists(refHsf.getMapFilePath()));
+ assertTrue(this.fs.exists(refHsf.getInfoFilePath()));
+ HStoreFile.Reference otherReference =
+ HStoreFile.readSplitInfo(refHsf.getInfoFilePath(), this.fs);
+ assertEquals(reference.getRegionName().toString(),
+ otherReference.getRegionName().toString());
+ assertEquals(reference.getFileId(),
+ otherReference.getFileId());
+ assertEquals(reference.getMidkey().toString(),
+ otherReference.getMidkey().toString());
+ // Now confirm that I can read from the reference and that it only gets
+ // keys from top half of the file.
+ MapFile.Reader halfReader = refHsf.getReader(this.fs, null);
+ HStoreKey key = new HStoreKey();
+ ImmutableBytesWritable value = new ImmutableBytesWritable();
+ boolean first = true;
+ while(halfReader.next(key, value)) {
+ if (first) {
+ assertEquals(key.getRow().toString(), midkey.getRow().toString());
+ first = false;
+ }
+ }
+ assertEquals(key.getRow().toString(), finalKey.toString());
+ }
+
+ /**
+ * Write a file and then assert that we can read from top and bottom halves
+ * using two HalfMapFiles.
+ * @throws Exception
+ */
+ public void testBasicHalfMapFile() throws Exception {
+ Path p = writeMapFile(getName());
+ WritableComparable midkey = getMidkey(p);
+ checkHalfMapFile(p, midkey);
+ }
+
+ /**
+ * Check HalfMapFile works even if file we're to go against is smaller than
+ * the default MapFile interval of 128: i.e. index gets entry every 128
+ * keys.
+ * @throws Exception
+ */
+ public void testSmallHalfMapFile() throws Exception {
+ Path p = writeSmallMapFile(getName());
+ // I know keys are a-z. Let the midkey we want to use be 'd'. See if
+ // HalfMapFiles work even if size of file is < than default MapFile
+ // interval.
+ checkHalfMapFile(p, new HStoreKey(new Text("d")));
+ }
+
+ private WritableComparable getMidkey(final Path p) throws IOException {
+ MapFile.Reader reader =
+ new MapFile.Reader(this.fs, p.toString(), this.conf);
+ HStoreKey key = new HStoreKey();
+ ImmutableBytesWritable value = new ImmutableBytesWritable();
+ reader.next(key, value);
+ String firstKey = key.toString();
+ WritableComparable midkey = reader.midKey();
+ reader.finalKey(key);
+ LOG.info("First key " + firstKey + ", midkey " + midkey.toString()
+ + ", last key " + key.toString());
+ reader.close();
+ return midkey;
+ }
+
+ private void checkHalfMapFile(final Path p, WritableComparable midkey)
+ throws IOException {
+ MapFile.Reader top = null;
+ MapFile.Reader bottom = null;
+ HStoreKey key = new HStoreKey();
+ ImmutableBytesWritable value = new ImmutableBytesWritable();
+ String previous = null;
+ try {
+ // Now make two HalfMapFiles and assert they can read the full backing
+ // file, one from the top and the other from the bottom.
+ // Test bottom half first.
+ bottom = new HStoreFile.HalfMapFileReader(this.fs, p.toString(),
+ this.conf, HStoreFile.Range.bottom, midkey);
+ boolean first = true;
+ while (bottom.next(key, value)) {
+ previous = key.toString();
+ if (first) {
+ first = false;
+ LOG.info("First in bottom: " + previous);
+ }
+ assertTrue(key.compareTo(midkey) < 0);
+ }
+ LOG.info("Last in bottom: " + previous.toString());
+ // Now test reading from the top.
+ top = new HStoreFile.HalfMapFileReader(this.fs, p.toString(), this.conf,
+ HStoreFile.Range.top, midkey);
+ first = true;
+ while (top.next(key, value)) {
+ assertTrue(key.compareTo(midkey) >= 0);
+ if (first) {
+ first = false;
+ assertEquals(((HStoreKey)midkey).getRow().toString(),
+ key.getRow().toString());
+ LOG.info("First in top: " + key.toString());
+ }
+ }
+ LOG.info("Last in top: " + key.toString());
+ top.getClosest(midkey, value);
+ // Assert value is same as key.
+ assertEquals(new String(value.get()),
+ ((HStoreKey) midkey).getRow().toString());
+
+ // Next test using a midkey that does not exist in the file.
+ // First, do a key that is < than first key. Ensure splits behave
+ // properly.
+ midkey = new HStoreKey(new Text(" "));
+ bottom = new HStoreFile.HalfMapFileReader(this.fs, p.toString(),
+ this.conf, HStoreFile.Range.bottom, midkey);
+ // When midkey is < than the bottom, should return no values.
+ assertFalse(bottom.next(key, value));
+ // Now read from the top.
+ top = new HStoreFile.HalfMapFileReader(this.fs, p.toString(), this.conf,
+ HStoreFile.Range.top, midkey);
+ first = true;
+ while (top.next(key, value)) {
+ assertTrue(key.compareTo(midkey) >= 0);
+ if (first) {
+ first = false;
+ LOG.info("First top when key < bottom: " + key.toString());
+ String tmp = key.getRow().toString();
+ for (int i = 0; i < tmp.length(); i++) {
+ assertTrue(tmp.charAt(i) == 'a');
+ }
+ }
+ }
+ LOG.info("Last top when key < bottom: " + key.toString());
+ String tmp = key.getRow().toString();
+ for (int i = 0; i < tmp.length(); i++) {
+ assertTrue(tmp.charAt(i) == 'z');
+ }
+
+ // Test when midkey is > than last key in file ('||' > 'zz').
+ midkey = new HStoreKey(new Text("|||"));
+ bottom = new HStoreFile.HalfMapFileReader(this.fs, p.toString(),
+ this.conf, HStoreFile.Range.bottom, midkey);
+ first = true;
+ while (bottom.next(key, value)) {
+ if (first) {
+ first = false;
+ LOG.info("First bottom when key > top: " + key.toString());
+ tmp = key.getRow().toString();
+ for (int i = 0; i < tmp.length(); i++) {
+ assertTrue(tmp.charAt(i) == 'a');
+ }
+ }
+ }
+ LOG.info("Last bottom when key > top: " + key.toString());
+ tmp = key.getRow().toString();
+ for (int i = 0; i < tmp.length(); i++) {
+ assertTrue(tmp.charAt(i) == 'z');
+ }
+ // Now look at top. Should not return any values.
+ top = new HStoreFile.HalfMapFileReader(this.fs, p.toString(), this.conf,
+ HStoreFile.Range.top, midkey);
+ assertFalse(top.next(key, value));
+
+ } finally {
+ if (top != null) {
+ top.close();
+ }
+ if (bottom != null) {
+ bottom.close();
+ }
+ fs.delete(p);
+ }
+ }
+
+ /**
+ * Assert HalFMapFile does right thing when midkey does not exist in the
+ * backing file (its larger or smaller than any of the backing mapfiles keys).
+ *
+ * @throws Exception
+ */
+ public void testOutOfRangeMidkeyHalfMapFile() throws Exception {
+ MapFile.Reader top = null;
+ MapFile.Reader bottom = null;
+ HStoreKey key = new HStoreKey();
+ ImmutableBytesWritable value = new ImmutableBytesWritable();
+ Path p = writeMapFile(getName());
+ try {
+ try {
+ // Test using a midkey that does not exist in the file.
+ // First, do a key that is < than first key. Ensure splits behave
+ // properly.
+ HStoreKey midkey = new HStoreKey(new Text(" "));
+ bottom = new HStoreFile.HalfMapFileReader(this.fs, p.toString(),
+ this.conf, HStoreFile.Range.bottom, midkey);
+ // When midkey is < than the bottom, should return no values.
+ assertFalse(bottom.next(key, value));
+ // Now read from the top.
+ top = new HStoreFile.HalfMapFileReader(this.fs, p.toString(),
+ this.conf, HStoreFile.Range.top, midkey);
+ boolean first = true;
+ while (top.next(key, value)) {
+ assertTrue(key.compareTo(midkey) >= 0);
+ if (first) {
+ first = false;
+ LOG.info("First top when key < bottom: " + key.toString());
+ assertEquals("aa", key.getRow().toString());
+ }
+ }
+ LOG.info("Last top when key < bottom: " + key.toString());
+ assertEquals("zz", key.getRow().toString());
+
+ // Test when midkey is > than last key in file ('||' > 'zz').
+ midkey = new HStoreKey(new Text("|||"));
+ bottom = new HStoreFile.HalfMapFileReader(this.fs, p.toString(),
+ this.conf, HStoreFile.Range.bottom, midkey);
+ first = true;
+ while (bottom.next(key, value)) {
+ if (first) {
+ first = false;
+ LOG.info("First bottom when key > top: " + key.toString());
+ assertEquals("aa", key.getRow().toString());
+ }
+ }
+ LOG.info("Last bottom when key > top: " + key.toString());
+ assertEquals("zz", key.getRow().toString());
+ // Now look at top. Should not return any values.
+ top = new HStoreFile.HalfMapFileReader(this.fs, p.toString(),
+ this.conf, HStoreFile.Range.top, midkey);
+ assertFalse(top.next(key, value));
+ } finally {
+ if (top != null) {
+ top.close();
+ }
+ if (bottom != null) {
+ bottom.close();
+ }
+ fs.delete(p);
+ }
+ } finally {
+ this.fs.delete(p);
+ }
+ }
+}
\ No newline at end of file
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestRegionServerAbort.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestRegionServerAbort.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestRegionServerAbort.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestRegionServerAbort.java Wed Aug 8 13:30:13 2007
@@ -77,10 +77,9 @@
// Now shutdown the region server and wait for it to go down.
this.cluster.abortRegionServer(0);
this.cluster.waitOnRegionServer(0);
-
+
// Verify that the client can find the data after the region has been moved
// to a different server
-
HScannerInterface scanner =
table.obtainScanner(HConstants.COLUMN_FAMILY_ARRAY, new Text());
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner.java Wed Aug 8 13:30:13 2007
@@ -146,7 +146,7 @@
Path dir = new Path("/hbase");
fs.mkdirs(dir);
- Path regionDir = HStoreFile.getHRegionDir(dir, REGION_INFO.regionName);
+ Path regionDir = HRegion.getRegionDir(dir, REGION_INFO.regionName);
fs.mkdirs(regionDir);
HLog log = new HLog(fs, new Path(regionDir, "log"), conf);