You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2008/06/19 00:24:35 UTC
svn commit: r669318 - in /hadoop/hbase/trunk: CHANGES.txt bin/Formatter.rb
bin/HBase.rb bin/hbase bin/hirb.rb
src/java/org/apache/hadoop/hbase/HTableDescriptor.java
src/java/org/apache/hadoop/hbase/client/HTable.java
Author: stack
Date: Wed Jun 18 15:24:34 2008
New Revision: 669318
URL: http://svn.apache.org/viewvc?rev=669318&view=rev
Log:
HBASE-487 New shell....
Add in special handling of .META. table that we used have in HQL
(so it prints out the HRegionInfo pretty). Also allow making a scanner
without specifying columns.
M src/java/org/apache/hadoop/hbase/HTableDescriptor.java
Allow getMetadata work if HTable is set against meta tables.
Was failing on isLegalTableName if name was one of the catalog table names.
Needed by shell.
M src/java/org/apache/hadoop/hbase/client/HTable.java
Comment.
M bin/hbase
Remove commented out line.
M bin/HBase.rb
Allow passing just a table name to scanner; let it figure out all families
Added in the special handling of .META. table cells that we used have in HQL
so we can see start/end row, etc. Added in extra testing.
M bin/Formatter.rb
Allow setting width of emitted table in console formatter
M bin/hirb.rb
Allow setting width of emitted table in console formatter
Improved scanner help.:
Modified:
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/bin/Formatter.rb
hadoop/hbase/trunk/bin/HBase.rb
hadoop/hbase/trunk/bin/hbase
hadoop/hbase/trunk/bin/hirb.rb
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HTableDescriptor.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java
Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=669318&r1=669317&r2=669318&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Wed Jun 18 15:24:34 2008
@@ -100,6 +100,7 @@
region state (Jean-Daniel Cryans via Stack)
HBASE-639 Add HBaseAdmin.getTableDescriptor function
HBASE-533 Region Historian
+ HBASE-487 Replace hql w/ a hbase-friendly jirb or jython shell
Release 0.1.2 - 05/13/2008
Modified: hadoop/hbase/trunk/bin/Formatter.rb
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/bin/Formatter.rb?rev=669318&r1=669317&r2=669318&view=diff
==============================================================================
--- hadoop/hbase/trunk/bin/Formatter.rb (original)
+++ hadoop/hbase/trunk/bin/Formatter.rb Wed Jun 18 15:24:34 2008
@@ -3,7 +3,7 @@
# Base abstract class for results formatting.
class Formatter
# Takes an output stream and a print width.
- def initialize(o, w = 80)
+ def initialize(o, w = 100)
raise TypeError.new("Type %s of parameter %s is not IO" % [o.class, o]) \
unless o.instance_of? IO
@out = o
@@ -38,8 +38,8 @@
puts
end
elsif args.length == 2
- col1width = 8
- col2width = 70
+ col1width = @maxWidth / 4
+ col2width = @maxWidth - col1width - 2
splits1 = split(col1width, dump(args[0]))
splits2 = split(col2width, dump(args[1]))
biggest = (splits2.length > splits1.length)? splits2.length: splits1.length
Modified: hadoop/hbase/trunk/bin/HBase.rb
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/bin/HBase.rb?rev=669318&r1=669317&r2=669318&view=diff
==============================================================================
--- hadoop/hbase/trunk/bin/HBase.rb (original)
+++ hadoop/hbase/trunk/bin/HBase.rb Wed Jun 18 15:24:34 2008
@@ -16,6 +16,8 @@
import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.HColumnDescriptor
import org.apache.hadoop.hbase.HTableDescriptor
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.hadoop.hbase.util.Writables
module HBase
COLUMN = "COLUMN"
@@ -25,6 +27,7 @@
VERSIONS = HConstants::VERSIONS
STOPROW = "STOPROW"
STARTROW = "STARTROW"
+ ENDROW = STOPROW
LIMIT = "LIMIT"
# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin
@@ -199,10 +202,25 @@
@formatter.footer(now)
end
+ def getAllColumns
+ htd = @table.getMetadata()
+ result = []
+ for f in htd.getFamilies()
+ n = f.getNameAsString()
+ n << ':'
+ result << n
+ end
+ result
+ end
+
def scan(columns, args = {})
now = Time.now
if not columns or columns.length < 1
- raise ArgumentError.new("Must supply an array of columns to scan")
+ # Make up list of columns.
+ columns = getAllColumns()
+ end
+ if columns.class == String
+ columns = [columns]
end
cs = columns.to_java(java.lang.String)
limit = -1
@@ -221,15 +239,15 @@
end
end
count = 0
- @formatter.header(["Row", "Column+Cell"])
+ @formatter.header(["ROW", "COLUMN+CELL"])
i = s.iterator()
while i.hasNext()
r = i.next()
row = String.from_java_bytes r.getRow()
for k, v in r
column = String.from_java_bytes k
- cell = v.toString()
- @formatter.row([row, "column=%s, %s" % [column, v.toString()]])
+ cell = toString(column, v)
+ @formatter.row([row, "column=%s, %s" % [column, cell]])
end
count += 1
if limit != -1 and count >= limit
@@ -252,6 +270,28 @@
@formatter.header()
@formatter.footer(now)
end
+
+ def isMetaTable()
+ tn = @table.getTableName()
+ return Bytes.equals(tn, HConstants::META_TABLE_NAME) or
+ Bytes.equals(tn, HConstants::META_TABLE_NAME)
+
+ end
+
+ # Make a String of the passed cell.
+ # Intercept cells whose format we know such as the info:regioninfo in .META.
+ def toString(column, cell)
+ if isMetaTable()
+ if column == 'info:regioninfo'
+ hri = Writables.getHRegionInfoOrNull(cell.getValue())
+ return "timestamp=%d, value=%s" % [cell.getTimestamp(), hri.toString()]
+ elsif column == 'info:serverstartcode'
+ return "timestamp=%d, value=%s" % [cell.getTimestamp(), \
+ Bytes.toLong(cell.getValue())]
+ end
+ end
+ cell.toString()
+ end
# Get from table
def get(row, args = {})
@@ -293,12 +333,11 @@
h = nil
if result.instance_of? RowResult
h = String.from_java_bytes result.getRow()
- @formatter.header(["Column", "Cell"])
+ @formatter.header(["COLUMN", "CELL"])
if result
- for column, cell in result
- v = String.from_java_bytes cell.getValue()
- ts = cell.getTimestamp()
- @formatter.row([(String.from_java_bytes column), cell.toString()])
+ for k, v in result
+ column = String.from_java_bytes k
+ @formatter.row([column, toString(column, v)])
end
end
else
@@ -314,8 +353,8 @@
end
end
- # Do a bit of testing.
- # To run this test, do: ./bin/hbase org.jruby.Main bin/HBase.rb
+ # Testing. To run this test, there needs to be an hbase cluster up and
+ # running. Then do: ${HBASE_HOME}/bin/hbase org.jruby.Main bin/HBase.rb
if $0 == __FILE__
# Add this directory to LOAD_PATH; presumption is that Formatter module
# sits beside this one. Then load it up.
@@ -326,7 +365,8 @@
# Now add in java and hbase classes
configuration = HBaseConfiguration.new()
admin = Admin.new(configuration, formatter)
- # Create a table; drop old one if exists first.
+ # Drop old table. If it does not exist, get an exception. Catch and
+ # continue
TESTTABLE = "HBase_rb_testtable"
begin
admin.disable(TESTTABLE)
@@ -338,13 +378,31 @@
# Presume it exists. If it doesn't, next items will fail.
table = Table.new(configuration, TESTTABLE, formatter)
for i in 1..10
- table.put('x', 'x:%d' % i, 'x%d' % i)
+ table.put('x%d' % i, 'x:%d' % i, 'x%d' % i)
end
- table.get('x', {COLUMN => 'x:1'})
+ table.get('x1', {COLUMN => 'x:1'})
if formatter.rowCount() != 1
raise IOError.new("Failed first put")
end
table.scan(['x:'])
+ if formatter.rowCount() != 10
+ raise IOError.new("Failed scan of expected 10 rows")
+ end
+ # Verify that limit works.
+ table.scan(['x:'], {LIMIT => 3})
+ if formatter.rowCount() != 3
+ raise IOError.new("Failed scan of expected 3 rows")
+ end
+ # Should only be two rows if we start at 8 (Row x10 sorts beside x1).
+ table.scan(['x:'], {STARTROW => 'x8', LIMIT => 3})
+ if formatter.rowCount() != 2
+ raise IOError.new("Failed scan of expected 2 rows")
+ end
+ # Scan between two rows
+ table.scan(['x:'], {STARTROW => 'x5', ENDROW => 'x8'})
+ if formatter.rowCount() != 3
+ raise IOError.new("Failed endrow test")
+ end
admin.disable(TESTTABLE)
admin.drop(TESTTABLE)
end
Modified: hadoop/hbase/trunk/bin/hbase
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/bin/hbase?rev=669318&r1=669317&r2=669318&view=diff
==============================================================================
--- hadoop/hbase/trunk/bin/hbase (original)
+++ hadoop/hbase/trunk/bin/hbase Wed Jun 18 15:24:34 2008
@@ -183,7 +183,6 @@
# figure out which class to run
if [ "$COMMAND" = "shell" ] ; then
- # CLASS="org.jruby.Main --command irb -r${HBASE_HOME}/bin/hirb.rb"
CLASS="org.jruby.Main ${HBASE_HOME}/bin/hirb.rb"
elif [ "$COMMAND" = "master" ] ; then
CLASS='org.apache.hadoop.hbase.master.HMaster'
Modified: hadoop/hbase/trunk/bin/hirb.rb
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/bin/hirb.rb?rev=669318&r1=669317&r2=669318&view=diff
==============================================================================
--- hadoop/hbase/trunk/bin/hirb.rb (original)
+++ hadoop/hbase/trunk/bin/hirb.rb Wed Jun 18 15:24:34 2008
@@ -5,9 +5,10 @@
# TODO: Add 'debug' support (client-side logs show in shell). Add it as
# command-line option and as command.
# TODO: Interrupt a table creation or a connection to a bad master. Currently
-# has to time out.
+# has to time out. Below we've set down the retries for rpc and hbase but
+# still can be annoying (And there seem to be times when we'll retry for
+# ever regardless)
# TODO: Add support for listing and manipulating catalog tables, etc.
-# TODO: Fix 'irb: warn: can't alias help from irb_help.' in banner message
# TODO: Encoding; need to know how to go from ruby String to UTF-8 bytes
# Run the java magic include and import basic HBase types that will help ease
@@ -28,12 +29,14 @@
# so they don't go through to irb. Output shell 'usage' if user types '--help'
cmdline_help = <<HERE # HERE document output as shell usage
HBase Shell command-line options:
- format Formatter for outputting results: console | html. Default: console
- master HBase master shell should connect to: e.g --master=example:60000
+ format Formatter for outputting results: console | html. Default: console
+ format.width Width of table outputs. Default: 110 characters.
+ master HBase master shell should connect to: e.g --master=example:60000
HERE
master = nil
-@formatter = Formatter::Console.new(STDOUT)
found = []
+format = 'console'
+format_width = 110
for arg in ARGV
if arg =~ /^--master=(.+)/i
master = $1
@@ -41,12 +44,16 @@
elsif arg =~ /^--format=(.+)/i
format = $1
if format =~ /^html$/i
- @formatter = Formatter::XHTML.new(STDOUT)
+ raise NoMethodError.new("Not yet implemented")
elsif format =~ /^console$/i
# This is default
else
raise ArgumentError.new("Unsupported format " + arg)
end
+ found.push(arg)
+ elsif arg =~ /^--format-width=(.+)/i
+ format_width = $1.to_i
+ found.push(arg)
elsif arg == '-h' || arg == '--help'
puts cmdline_help
exit
@@ -55,6 +62,9 @@
for arg in found
ARGV.delete(arg)
end
+# Presume console format.
+@formatter = Formatter::Console.new(STDOUT, format_width)
+# TODO, etc. @formatter = Formatter::XHTML.new(STDOUT)
# Setup the HBase module. Create a configuration. If a master, set it.
# Turn off retries in hbase and ipc. Human doesn't want to wait on N retries.
@@ -150,13 +160,14 @@
hbase> put 't1', 'r1', 'c1', ts1
- scan Scan a table; pass table name and an array of column names.
- Optionally, pass a dictionary of options that includes one or more
- of the following: LIMIT, FILTER, STARTROW, STOPROW, and TIMESTAMP.
- For example, to scan column 'c1', and 'c2', in table 't1' returning
- 10 rows only:
-
- hbase> scan 't1', ['c1', 'c2'], {LIMIT => 10}
+ scan Scan a table; pass table name and optionally an array of column
+ names and a dictionary of scanner specification that includes one
+ or more of following: LIMIT, FILTER, STARTROW, STOPROW, or TIMESTAMP.
+ Examples:
+
+ hbase> scan '.META.'
+ hbase> scan '.META.', ['info:regioninfo']
+ hbase> scan 't1', ['c1', 'c2'], {LIMIT => 10, STARTROW => 'xyz'}
version Output this HBase version
@@ -242,7 +253,7 @@
table(table).put(row, column, value, timestamp)
end
-def scan(table, columns, args = {})
+def scan(table, columns = [], args = {})
table(table).scan(columns, args)
end
@@ -268,18 +279,27 @@
require "irb"
-# IRB::ExtendCommandBundle.instance_variable_get("@EXTEND_COMMANDS").delete_if{|x| x.first == :irb_help}
-
module IRB
- module ExtendCommandBundle
- # These are attempts at blocking the complaint about :irb_help on startup.
- # @EXTEND_COMMANDS.delete_if{|x| x[0] == :irb_help}
- # @EXTEND_COMMANDS.each{|x| x[3][1] = OVERRIDE_ALL if x[0] == :irb_help}
- # @EXTEND_COMMANDS.each{|x| puts x if x[0] == :irb_help}
- end
-
# Subclass of IRB so can intercept methods
class HIRB < Irb
+ def initialize
+ # This is ugly. Our 'help' method above provokes the following message
+ # on irb construction: 'irb: warn: can't alias help from irb_help.'
+ # Below, we reset the output so its pointed at /dev/null during irb
+ # construction just so this message does not come out after we emit
+ # the banner. Other attempts at playing with the hash of methods
+ # down in IRB didn't seem to work. I think the worst thing that can
+ # happen is the shell exiting because of failed IRB construction with
+ # no error (though we're not blanking STDERR)
+ begin
+ f = File.open("/dev/null", "w")
+ $stdout = f
+ super
+ ensure
+ f.close()
+ $stdout = STDOUT
+ end
+ end
def output_value
# Suppress output if last_value is 'nil'
@@ -295,7 +315,8 @@
$0 = File::basename(ap_path, ".rb") if ap_path
IRB.setup(ap_path)
- @CONF[:IRB_NAME]="hbase"
+ @CONF[:IRB_NAME] = 'hbase'
+ @CONF[:AP_NAME] = 'hbase'
if @CONF[:SCRIPT]
hirb = HIRB.new(nil, @CONF[:SCRIPT])
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HTableDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HTableDescriptor.java?rev=669318&r1=669317&r2=669318&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HTableDescriptor.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HTableDescriptor.java Wed Jun 18 15:24:34 2008
@@ -71,8 +71,7 @@
*/
private HTableDescriptor(final byte [] name, HColumnDescriptor[] families) {
this.name = name.clone();
- this.rootregion = Bytes.equals(name, HConstants.ROOT_TABLE_NAME);
- this.metaregion = true;
+ setMetaFlags(name);
for(HColumnDescriptor descriptor : families) {
this.families.put(Bytes.mapKey(descriptor.getName()), descriptor);
}
@@ -108,9 +107,21 @@
* @see <a href="HADOOP-1581">HADOOP-1581 HBASE: Un-openable tablename bug</a>
*/
public HTableDescriptor(final byte [] name) {
- this.name = isLegalTableName(name);
+ setMetaFlags(name);
+ this.name = this.metaregion? name: isLegalTableName(name);
this.nameAsString = Bytes.toString(this.name);
}
+
+ /*
+ * Set meta flags on this table.
+ * Called by constructors.
+ * @param name
+ */
+ private void setMetaFlags(final byte [] name) {
+ this.rootregion = Bytes.equals(name, HConstants.ROOT_TABLE_NAME);
+ this.metaregion =
+ this.rootregion? true: Bytes.equals(name, HConstants.META_TABLE_NAME);
+ }
/**
* Check passed buffer is legal user-space table name.
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java?rev=669318&r1=669317&r2=669318&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java Wed Jun 18 15:24:34 2008
@@ -341,6 +341,7 @@
* @return table metadata
* @throws IOException
*/
+ // Why is this deprecated? What should be used instead? St.Ack
@Deprecated
public HTableDescriptor getMetadata() throws IOException {
return this.connection.getHTableDescriptor(this.tableName);