You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/03/15 02:57:36 UTC
svn commit: r1081637 - in /hbase/branches/0.90: CHANGES.txt
src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
Author: nspiegelberg
Date: Tue Mar 15 01:57:35 2011
New Revision: 1081637
URL: http://svn.apache.org/viewvc?rev=1081637&view=rev
Log:
HBASE-3496 HFile CLI Improvements
Modified:
hbase/branches/0.90/CHANGES.txt
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
Modified: hbase/branches/0.90/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/CHANGES.txt?rev=1081637&r1=1081636&r2=1081637&view=diff
==============================================================================
--- hbase/branches/0.90/CHANGES.txt (original)
+++ hbase/branches/0.90/CHANGES.txt Tue Mar 15 01:57:35 2011
@@ -46,6 +46,7 @@ Release 0.90.2 - February 9th, 2011
HBASE-3437 Support Explict Split Points from the Shell
HBASE-3448 RegionSplitter, utility class to manually split tables
HBASE-3610 Improve RegionSplitter performance
+ HBASE-3496 HFile CLI Improvements
Release 0.90.1 - February 9th, 2011
Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1081637&r1=1081636&r2=1081637&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java Tue Mar 15 01:57:35 2011
@@ -1713,12 +1713,12 @@ public class HFile {
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
- sb.append("size=" + count);
+ sb.append("size=" + count).append("\n");
for (int i = 0; i < count ; i++) {
- sb.append(", ");
- sb.append("key=").append(Bytes.toStringBinary(blockKeys[i])).
- append(", offset=").append(blockOffsets[i]).
- append(", dataSize=" + blockDataSizes[i]);
+ sb.append("key=").append(KeyValue.keyToString(blockKeys[i])).
+ append("\n offset=").append(blockOffsets[i]).
+ append(", dataSize=" + blockDataSizes[i]).
+ append("\n");
}
return sb.toString();
}
@@ -1839,7 +1839,9 @@ public class HFile {
Options options = new Options();
options.addOption("v", "verbose", false, "Verbose output; emits file and meta data delimiters");
options.addOption("p", "printkv", false, "Print key/value pairs");
+ options.addOption("e", "printkey", false, "Print keys");
options.addOption("m", "printmeta", false, "Print meta data of file");
+ options.addOption("b", "printblocks", false, "Print block index meta data");
options.addOption("k", "checkrow", false,
"Enable row order check; looks for out-of-order keys");
options.addOption("a", "checkfamily", false, "Enable family check");
@@ -1855,8 +1857,10 @@ public class HFile {
CommandLineParser parser = new PosixParser();
CommandLine cmd = parser.parse(options, args);
boolean verbose = cmd.hasOption("v");
- boolean printKeyValue = cmd.hasOption("p");
+ boolean printValue = cmd.hasOption("p");
+ boolean printKey = cmd.hasOption("e") || printValue;
boolean printMeta = cmd.hasOption("m");
+ boolean printBlocks = cmd.hasOption("b");
boolean checkRow = cmd.hasOption("k");
boolean checkFamily = cmd.hasOption("a");
// get configuration, file system and get list of files
@@ -1865,7 +1869,6 @@ public class HFile {
conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR));
conf.set("fs.default.name",
conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR));
- FileSystem fs = FileSystem.get(conf);
ArrayList<Path> files = new ArrayList<Path>();
if (cmd.hasOption("f")) {
files.add(new Path(cmd.getOptionValue("f")));
@@ -1879,7 +1882,8 @@ public class HFile {
String enc = HRegionInfo.encodeRegionName(rn);
Path regionDir = new Path(tableDir, enc);
if (verbose) System.out.println("region dir -> " + regionDir);
- List<Path> regionFiles = getStoreFiles(fs, regionDir);
+ List<Path> regionFiles =
+ getStoreFiles(FileSystem.get(conf), regionDir);
if (verbose) System.out.println("Number of region files found -> " +
regionFiles.size());
if (verbose) {
@@ -1893,6 +1897,7 @@ public class HFile {
// iterate over all files found
for (Path file : files) {
if (verbose) System.out.println("Scanning -> " + file);
+ FileSystem fs = file.getFileSystem(conf);
if (!fs.exists(file)) {
System.err.println("ERROR, file doesnt exist: " + file);
continue;
@@ -1901,7 +1906,7 @@ public class HFile {
HFile.Reader reader = new HFile.Reader(fs, file, null, false);
Map<byte[],byte[]> fileInfo = reader.loadFileInfo();
int count = 0;
- if (verbose || printKeyValue || checkRow || checkFamily) {
+ if (verbose || printKey || checkRow || checkFamily) {
// scan over file and read key/value's and check if requested
HFileScanner scanner = reader.getScanner(false, false);
scanner.seekTo();
@@ -1909,9 +1914,12 @@ public class HFile {
do {
KeyValue kv = scanner.getKeyValue();
// dump key value
- if (printKeyValue) {
- System.out.println("K: " + kv +
- " V: " + Bytes.toStringBinary(kv.getValue()));
+ if (printKey) {
+ System.out.print("K: " + kv);
+ if (printValue) {
+ System.out.print(" V: " + Bytes.toStringBinary(kv.getValue()));
+ }
+ System.out.println();
}
// check if rows are in order
if (checkRow && pkv != null) {
@@ -1941,7 +1949,7 @@ public class HFile {
count++;
} while (scanner.next());
}
- if (verbose || printKeyValue) {
+ if (verbose || printKey) {
System.out.println("Scanned kv count -> " + count);
}
// print meta data
@@ -1982,6 +1990,10 @@ public class HFile {
System.out.println("Could not get bloom data from meta block");
}
}
+ if (printBlocks) {
+ System.out.println("Block Index:");
+ System.out.println(reader.blockIndex);
+ }
reader.close();
}
} catch (Exception e) {
Modified: hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java?rev=1081637&r1=1081636&r2=1081637&view=diff
==============================================================================
--- hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java (original)
+++ hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java Tue Mar 15 01:57:35 2011
@@ -24,6 +24,7 @@ import java.io.IOException;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestCase;
+import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.util.Bytes;
/**
@@ -31,18 +32,28 @@ import org.apache.hadoop.hbase.util.Byte
*/
public class TestSeekTo extends HBaseTestCase {
+ static KeyValue toKV(String row) {
+ return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes
+ .toBytes("qualifier"), Bytes.toBytes("value"));
+ }
+
+ static String toRowStr(KeyValue kv) {
+ return Bytes.toString(kv.getRow());
+ }
+
Path makeNewFile() throws IOException {
Path ncTFile = new Path(this.testDir, "basic.hfile");
FSDataOutputStream fout = this.fs.create(ncTFile);
- HFile.Writer writer = new HFile.Writer(fout, 40, "none", null);
+ int blocksize = toKV("a").getLength() * 3;
+ HFile.Writer writer = new HFile.Writer(fout, blocksize, "none", null);
// 4 bytes * 3 * 2 for each key/value +
// 3 for keys, 15 for values = 42 (woot)
- writer.append(Bytes.toBytes("c"), Bytes.toBytes("value"));
- writer.append(Bytes.toBytes("e"), Bytes.toBytes("value"));
- writer.append(Bytes.toBytes("g"), Bytes.toBytes("value"));
+ writer.append(toKV("c"));
+ writer.append(toKV("e"));
+ writer.append(toKV("g"));
// block transition
- writer.append(Bytes.toBytes("i"), Bytes.toBytes("value"));
- writer.append(Bytes.toBytes("k"), Bytes.toBytes("value"));
+ writer.append(toKV("i"));
+ writer.append(toKV("k"));
writer.close();
fout.close();
return ncTFile;
@@ -52,32 +63,32 @@ public class TestSeekTo extends HBaseTes
HFile.Reader reader = new HFile.Reader(fs, p, null, false);
reader.loadFileInfo();
HFileScanner scanner = reader.getScanner(false, true);
- assertEquals(false, scanner.seekBefore(Bytes.toBytes("a")));
+ assertEquals(false, scanner.seekBefore(toKV("a").getKey()));
- assertEquals(false, scanner.seekBefore(Bytes.toBytes("c")));
+ assertEquals(false, scanner.seekBefore(toKV("c").getKey()));
- assertEquals(true, scanner.seekBefore(Bytes.toBytes("d")));
- assertEquals("c", scanner.getKeyString());
+ assertEquals(true, scanner.seekBefore(toKV("d").getKey()));
+ assertEquals("c", toRowStr(scanner.getKeyValue()));
- assertEquals(true, scanner.seekBefore(Bytes.toBytes("e")));
- assertEquals("c", scanner.getKeyString());
-
- assertEquals(true, scanner.seekBefore(Bytes.toBytes("f")));
- assertEquals("e", scanner.getKeyString());
-
- assertEquals(true, scanner.seekBefore(Bytes.toBytes("g")));
- assertEquals("e", scanner.getKeyString());
-
- assertEquals(true, scanner.seekBefore(Bytes.toBytes("h")));
- assertEquals("g", scanner.getKeyString());
- assertEquals(true, scanner.seekBefore(Bytes.toBytes("i")));
- assertEquals("g", scanner.getKeyString());
- assertEquals(true, scanner.seekBefore(Bytes.toBytes("j")));
- assertEquals("i", scanner.getKeyString());
- assertEquals(true, scanner.seekBefore(Bytes.toBytes("k")));
- assertEquals("i", scanner.getKeyString());
- assertEquals(true, scanner.seekBefore(Bytes.toBytes("l")));
- assertEquals("k", scanner.getKeyString());
+ assertEquals(true, scanner.seekBefore(toKV("e").getKey()));
+ assertEquals("c", toRowStr(scanner.getKeyValue()));
+
+ assertEquals(true, scanner.seekBefore(toKV("f").getKey()));
+ assertEquals("e", toRowStr(scanner.getKeyValue()));
+
+ assertEquals(true, scanner.seekBefore(toKV("g").getKey()));
+ assertEquals("e", toRowStr(scanner.getKeyValue()));
+
+ assertEquals(true, scanner.seekBefore(toKV("h").getKey()));
+ assertEquals("g", toRowStr(scanner.getKeyValue()));
+ assertEquals(true, scanner.seekBefore(toKV("i").getKey()));
+ assertEquals("g", toRowStr(scanner.getKeyValue()));
+ assertEquals(true, scanner.seekBefore(toKV("j").getKey()));
+ assertEquals("i", toRowStr(scanner.getKeyValue()));
+ assertEquals(true, scanner.seekBefore(toKV("k").getKey()));
+ assertEquals("i", toRowStr(scanner.getKeyValue()));
+ assertEquals(true, scanner.seekBefore(toKV("l").getKey()));
+ assertEquals("k", toRowStr(scanner.getKeyValue()));
}
public void testSeekTo() throws Exception {
@@ -87,17 +98,17 @@ public class TestSeekTo extends HBaseTes
assertEquals(2, reader.blockIndex.count);
HFileScanner scanner = reader.getScanner(false, true);
// lies before the start of the file.
- assertEquals(-1, scanner.seekTo(Bytes.toBytes("a")));
+ assertEquals(-1, scanner.seekTo(toKV("a").getKey()));
- assertEquals(1, scanner.seekTo(Bytes.toBytes("d")));
- assertEquals("c", scanner.getKeyString());
+ assertEquals(1, scanner.seekTo(toKV("d").getKey()));
+ assertEquals("c", toRowStr(scanner.getKeyValue()));
// Across a block boundary now.
- assertEquals(1, scanner.seekTo(Bytes.toBytes("h")));
- assertEquals("g", scanner.getKeyString());
+ assertEquals(1, scanner.seekTo(toKV("h").getKey()));
+ assertEquals("g", toRowStr(scanner.getKeyValue()));
- assertEquals(1, scanner.seekTo(Bytes.toBytes("l")));
- assertEquals("k", scanner.getKeyString());
+ assertEquals(1, scanner.seekTo(toKV("l").getKey()));
+ assertEquals("k", toRowStr(scanner.getKeyValue()));
}
public void testBlockContainingKey() throws Exception {
@@ -105,19 +116,27 @@ public class TestSeekTo extends HBaseTes
HFile.Reader reader = new HFile.Reader(fs, p, null, false);
reader.loadFileInfo();
System.out.println(reader.blockIndex.toString());
+ int klen = toKV("a").getKey().length;
// falls before the start of the file.
- assertEquals(-1, reader.blockIndex.blockContainingKey(Bytes.toBytes("a"), 0, 1));
- assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("c"), 0, 1));
- assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("d"), 0, 1));
- assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("e"), 0, 1));
- assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("g"), 0, 1));
- assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("h"), 0, 1));
- assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("i"), 0, 1));
- assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("j"), 0, 1));
- assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("k"), 0, 1));
- assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("l"), 0, 1));
-
-
-
+ assertEquals(-1, reader.blockIndex.blockContainingKey(toKV("a").getKey(),
+ 0, klen));
+ assertEquals(0, reader.blockIndex.blockContainingKey(toKV("c").getKey(), 0,
+ klen));
+ assertEquals(0, reader.blockIndex.blockContainingKey(toKV("d").getKey(), 0,
+ klen));
+ assertEquals(0, reader.blockIndex.blockContainingKey(toKV("e").getKey(), 0,
+ klen));
+ assertEquals(0, reader.blockIndex.blockContainingKey(toKV("g").getKey(), 0,
+ klen));
+ assertEquals(0, reader.blockIndex.blockContainingKey(toKV("h").getKey(), 0,
+ klen));
+ assertEquals(1, reader.blockIndex.blockContainingKey(toKV("i").getKey(), 0,
+ klen));
+ assertEquals(1, reader.blockIndex.blockContainingKey(toKV("j").getKey(), 0,
+ klen));
+ assertEquals(1, reader.blockIndex.blockContainingKey(toKV("k").getKey(), 0,
+ klen));
+ assertEquals(1, reader.blockIndex.blockContainingKey(toKV("l").getKey(), 0,
+ klen));
}
-}
\ No newline at end of file
+}