You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/09/27 04:41:20 UTC

svn commit: r1176171 [1/9] - in /hbase/branches/0.89: ./ bin/ src/ src/assembly/ src/docs/src/documentation/content/xdocs/ src/main/java/org/apache/hadoop/hbase/ src/main/java/org/apache/hadoop/hbase/client/ src/main/java/org/apache/hadoop/hbase/filter...

Author: nspiegelberg
Date: Tue Sep 27 02:41:16 2011
New Revision: 1176171

URL: http://svn.apache.org/viewvc?rev=1176171&view=rev
Log:
HBASE: Update to trunk

Summary:
Merged with HBase (0.21) public trunk.  revisions 946777 => 952833

Test Plan:
1. mvn test
2. ./bin/start-hbase.sh && ./bin/hbase shell

DiffCamp Revision: 121725
Reviewed By: kannan
CC: isaac, cpiro, nspiegelberg, kannan
Tasks:
#219779: Merge to HBase Trunk

Revert Plan:
OK

Added:
    hbase/branches/0.89/bin/check_meta.rb
    hbase/branches/0.89/src/assembly/bin.xml.orig
      - copied, changed from r1176170, hbase/branches/0.89/src/assembly/bin.xml
    hbase/branches/0.89/src/docs/src/documentation/content/xdocs/bulk-loads.xml
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/InputSampler.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TotalOrderPartitioner.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/DefaultEnvironmentEdge.java
      - copied, changed from r1176170, hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdge.java
      - copied, changed from r1176170, hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManager.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/IncrementingEnvironmentEdge.java
      - copied, changed from r1176170, hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/MD5Hash.java
    hbase/branches/0.89/src/main/resources/hbase-webapps/
    hbase/branches/0.89/src/main/resources/hbase-webapps/master/
    hbase/branches/0.89/src/main/resources/hbase-webapps/master/index.html
    hbase/branches/0.89/src/main/resources/hbase-webapps/master/master.jsp
    hbase/branches/0.89/src/main/resources/hbase-webapps/master/table.jsp
    hbase/branches/0.89/src/main/resources/hbase-webapps/master/zk.jsp
    hbase/branches/0.89/src/main/resources/hbase-webapps/regionserver/
    hbase/branches/0.89/src/main/resources/hbase-webapps/regionserver/index.html
    hbase/branches/0.89/src/main/resources/hbase-webapps/regionserver/regionserver.jsp
    hbase/branches/0.89/src/main/resources/hbase-webapps/static/
    hbase/branches/0.89/src/main/resources/hbase-webapps/static/hbase.css
    hbase/branches/0.89/src/main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd
    hbase/branches/0.89/src/main/resources/webapps/master/WEB-INF/web.xml
    hbase/branches/0.89/src/main/resources/webapps/regionserver/WEB-INF/web.xml
    hbase/branches/0.89/src/site/
    hbase/branches/0.89/src/site/fml/
    hbase/branches/0.89/src/site/fml/faq.fml
    hbase/branches/0.89/src/site/resources/
    hbase/branches/0.89/src/site/resources/css/
    hbase/branches/0.89/src/site/resources/css/site.css
    hbase/branches/0.89/src/site/site.xml
    hbase/branches/0.89/src/site/xdoc/
    hbase/branches/0.89/src/site/xdoc/acid-semantics.xml
    hbase/branches/0.89/src/site/xdoc/bulk-loads.xml
    hbase/branches/0.89/src/site/xdoc/cygwin.xml
    hbase/branches/0.89/src/site/xdoc/index.xml
    hbase/branches/0.89/src/site/xdoc/metrics.xml
    hbase/branches/0.89/src/site/xdoc/old_news.xml
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/MultithreadedTestUtil.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/mapreduce/NMapInputFormat.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/master/TestMasterWrongRS.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java
      - copied, changed from r1176170, hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
      - copied, changed from r1176170, hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/util/TestDefaultEnvironmentEdge.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/util/TestEnvironmentEdgeManager.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/util/TestIncrementingEnvironmentEdge.java
      - copied, changed from r1176170, hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
Modified:
    hbase/branches/0.89/CHANGES.txt
    hbase/branches/0.89/bin/add_table.rb
    hbase/branches/0.89/bin/hirb.rb
    hbase/branches/0.89/pom.xml
    hbase/branches/0.89/src/assembly/bin.xml
    hbase/branches/0.89/src/docs/src/documentation/content/xdocs/site.xml
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HConstants.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HMsg.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HServerInfo.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTable.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/Driver.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/SimpleTotalOrderPartitioner.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/BaseScanner.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/MinorCompactingStoreScanner.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ReadWriteConsistencyControl.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/InfoServer.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
    hbase/branches/0.89/src/main/ruby/hbase/admin.rb
    hbase/branches/0.89/src/main/ruby/hbase/hbase.rb
    hbase/branches/0.89/src/main/ruby/shell/commands/list.rb
    hbase/branches/0.89/src/main/ruby/shell/formatter.rb
    hbase/branches/0.89/src/saveVersion.sh
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/TestEmptyMetaInfo.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/TestImmutableBytesWritable.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/utils/KillProcessesAndVerify.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueScanFixture.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreReconstruction.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/rest/HBaseRESTClusterTestBase.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/rest/TestVersionResource.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/util/TestByteBloomFilter.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
    hbase/branches/0.89/src/test/ruby/hbase/hbase_test.rb
    hbase/branches/0.89/src/test/ruby/shell/shell_test.rb
    hbase/branches/0.89/src/test/ruby/test_helper.rb

Modified: hbase/branches/0.89/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/CHANGES.txt?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/CHANGES.txt (original)
+++ hbase/branches/0.89/CHANGES.txt Tue Sep 27 02:41:16 2011
@@ -23,6 +23,7 @@ Release 0.21.0 - Unreleased
    HBASE-2541  Remove transactional contrib (Clint Morgan via Stack)
    HBASE-2542  Fold stargate contrib into core
    HBASE-2565  Remove contrib module from hbase
+   HBASE-2397  Bytes.toStringBinary escapes printable chars
 
   BUG FIXES
    HBASE-1791  Timeout in IndexRecordWriter (Bradford Stephens via Andrew
@@ -348,6 +349,35 @@ Release 0.21.0 - Unreleased
    HBASE-2582  TestTableSchemaModel not passing after commit of blooms
    HBASE-2583  Make webapps work in distributed mode again and make webapps
                deploy at / instead of at /webapps/master/master.jsp
+   HBASE-2590  Failed parse of branch element in saveVersion.sh
+   HBASE-2591  HBASE-2587 hardcoded the port that dfscluster runs on
+   HBASE-2519  StoreFileScanner.seek swallows IOEs (Todd Lipcon via Stack)
+   HBASE-2516  Ugly IOE when region is being closed; rather, should NSRE
+               (Daniel Ploeg via Stack)
+   HBASE-2589  TestHRegion.testWritesWhileScanning flaky on trunk
+               (Todd Lipcon via Stack)
+   HBASE-2590  Failed parse of branch element in saveVersion.sh
+               (Benoît Sigoure via Stack)
+   HBASE-2586  Move hbase webapps to a hbase-webapps dir (Todd Lipcon via
+               Andrew Purtell)
+   HBASE-2610  ValueFilter copy pasted javadoc from QualifierFilter
+   HBASE-2619  HBase shell 'alter' command cannot set table properties to False
+               (Christo Wilson via Stack)
+   HBASE-2621  Fix bad link to HFile documentation in javadoc
+               (Jeff Hammerbacher via Todd Lipcon)
+   HBASE-2371  Fix 'list' command in shell (Alexey Kovyrin via Todd Lipcon)
+   HBASE-2620  REST tests don't use ephemeral ports
+   HBASE-2635  ImmutableBytesWritable ignores offset in several cases
+   HBASE-2654  Add additional maven repository temporarily to fetch Guava
+   HBASE-2560  Fix IllegalArgumentException when manually splitting table
+               from web UI
+   HBASE-2657  TestTableResource is broken in trunk
+   HBASE-2662  TestScannerResource.testScannerResource broke in trunk
+   HBASE-2667  TestHLog.testSplit failing in trunk (Cosmin and Stack)
+   HBASE-2614  killing server in TestMasterTransitions causes NPEs and test deadlock
+   HBASE-2615  M/R on bulk imported tables
+   HBASE-2676  TestInfoServers should use ephemeral ports
+   HBASE-2616  TestHRegion.testWritesWhileGetting flaky on trunk
 
   IMPROVEMENTS
    HBASE-1760  Cleanup TODOs in HTable
@@ -623,6 +653,29 @@ Release 0.21.0 - Unreleased
                pulled from http://people.apache.org/~rawson/repo/....
    HBASE-2534  Recursive deletes and misc improvements to ZKW
    HBASE-2577  Remove 'core' maven module; move core up a level
+   HBASE-2587  Coral where tests write data when running and make sure clean
+               target removes all written
+   HBASE-2580  Make the hlog file names unique
+   HBASE-2594  Narrow pattern used finding unit tests to run -- make it same
+               was we had in 0.20
+   HBASE-2538  Work on repository order in pom (adding fbmirror to top,
+               ibiblio on bottom)
+   HBASE-2613  Remove the code around MSG_CALL_SERVER_STARTUP
+   HBASE-2599  BaseScanner says "Current assignment of X is not valid" over
+               and over for same region
+   HBASE-2630  HFile should use toStringBinary in various places
+   HBASE-2632  Shell should autodetect terminal width
+   HBASE-2636  Upgrade Jetty to 6.1.24
+   HBASE-2437  Refactor HLog splitLog (Cosmin Lehene via Stack)
+   HBASE-2638  Speed up REST tests
+   HBASE-2653  Remove unused DynamicBloomFilter (especially as its tests are
+               failing hudson on occasion)
+   HBASE-2651  Allow alternate column separators to be specified for ImportTsv
+   HBASE-2661  Add test case for row atomicity guarantee
+   HBASE-2578  Add ability for tests to override server-side timestamp
+	           setting (currentTimeMillis) (Daniel Ploeg via Ryan Rawson)
+   HBASE-2558  Our javadoc overview -- "Getting Started", requirements, etc. --
+               is not carried across by mvn javadoc:javadoc target
 
   NEW FEATURES
    HBASE-1961  HBase EC2 scripts
@@ -659,6 +712,9 @@ Release 0.21.0 - Unreleased
    HBASE-2527  Add the ability to easily extend some HLog actions
    HBASE-2559  Set hbase.hregion.majorcompaction to 0 to disable
    HBASE-1200  Add bloomfilters (Nicolas Spiegelberg via Stack)
+   HBASE-2588  Add easier way to ship HBase dependencies to MR cluster within Job
+   HBASE-1923  Bulk incremental load into an existing table
+   HBASE-2579  Add atomic checkAndDelete support (Michael Dalton via Stack)
 
   OPTIMIZATIONS
    HBASE-410   [testing] Speed up the test suite
@@ -1000,6 +1056,11 @@ Release 0.20.0 - Tue Sep  8 12:53:05 PDT
                (Mathias Herberts via Stack)
    HBASE-1804  Puts are permitted (and stored) when including an appended colon
    HBASE-1715  Compaction failure in ScanWildcardColumnTracker.checkColumn
+   HBASE-2352  Small values for hbase.client.retries.number and
+               ipc.client.connect.max.retries breaks long ops in hbase shell
+               (Alexey Kovyrin via Stack)
+   HBASE-2531  32-bit encoding of regionnames waaaaaaayyyyy too susceptible to
+               hash clashes (Kannan Muthukkaruppan via Stack)
 
   IMPROVEMENTS
    HBASE-1089  Add count of regions on filesystem to master UI; add percentage
@@ -1234,6 +1295,8 @@ Release 0.20.0 - Tue Sep  8 12:53:05 PDT
    HBASE-1743  [debug tool] Add regionsInTransition list to ClusterStatus
                detailed output
    HBASE-1772  Up the default ZK session timeout from 30seconds to 60seconds
+   HBASE-2625  Make testDynamicBloom()'s "randomness" deterministic
+               (Nicolas Spiegelberg via Stack)
 
   OPTIMIZATIONS
    HBASE-1412  Change values for delete column and column family in KeyValue

Modified: hbase/branches/0.89/bin/add_table.rb
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/bin/add_table.rb?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/bin/add_table.rb (original)
+++ hbase/branches/0.89/bin/add_table.rb Tue Sep 27 02:41:16 2011
@@ -19,14 +19,15 @@
 #
 # Script adds a table back to a running hbase.
 # Currently only works on if table data is in place.
-# 
-# To see usage for this script, run: 
+#
+# To see usage for this script, run:
 #
 #  ${HBASE_HOME}/bin/hbase org.jruby.Main addtable.rb
 #
 include Java
 import org.apache.hadoop.hbase.util.Bytes
 import org.apache.hadoop.hbase.HConstants
+import org.apache.hadoop.hbase.regionserver.HRegion
 import org.apache.hadoop.hbase.HRegionInfo
 import org.apache.hadoop.hbase.client.HTable
 import org.apache.hadoop.hbase.client.Delete
@@ -84,7 +85,7 @@ elsif
 end
 HTableDescriptor.isLegalTableName(tableName.to_java_bytes)
 
-# Figure locations under hbase.rootdir 
+# Figure locations under hbase.rootdir
 # Move directories into place; be careful not to overwrite.
 rootdir = FSUtils.getRootDir(c)
 tableDir = fs.makeQualified(Path.new(rootdir, tableName))
@@ -129,15 +130,15 @@ statuses = fs.listStatus(srcdir)
 for status in statuses
   next unless status.isDir()
   next if status.getPath().getName() == "compaction.dir"
-  regioninfofile =  Path.new(status.getPath(), ".regioninfo")
+  regioninfofile =  Path.new(status.getPath(), HRegion::REGIONINFO_FILE)
   unless fs.exists(regioninfofile)
     LOG.warn("Missing .regioninfo: " + regioninfofile.toString())
     next
   end
-  is = fs.open(regioninfofile) 
+  is = fs.open(regioninfofile)
   hri = HRegionInfo.new()
   hri.readFields(is)
-  is.close() 
+  is.close()
   # TODO: Need to redo table descriptor with passed table name and then recalculate the region encoded names.
   p = Put.new(hri.getRegionName())
   p.add(HConstants::CATALOG_FAMILY, HConstants::REGIONINFO_QUALIFIER, Writables.getBytes(hri))

Added: hbase/branches/0.89/bin/check_meta.rb
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/bin/check_meta.rb?rev=1176171&view=auto
==============================================================================
--- hbase/branches/0.89/bin/check_meta.rb (added)
+++ hbase/branches/0.89/bin/check_meta.rb Tue Sep 27 02:41:16 2011
@@ -0,0 +1,160 @@
+# Script looks at hbase .META. table verifying its content is coherent.
+#
+# To see usage for this script, run:
+#
+#  ${HBASE_HOME}/bin/hbase org.jruby.Main check_meta.rb --help
+#
+
+# Copyright 2010 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+include Java
+import org.apache.commons.logging.LogFactory
+import org.apache.hadoop.hbase.util.VersionInfo
+import org.apache.hadoop.hbase.HBaseConfiguration
+import org.apache.hadoop.fs.FileSystem
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.hbase.HConstants
+import org.apache.hadoop.hbase.util.FSUtils
+import org.apache.hadoop.hbase.client.HTable
+import org.apache.hadoop.hbase.client.Scan
+import org.apache.hadoop.hbase.util.Writables
+import org.apache.hadoop.hbase.HRegionInfo
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.hadoop.hbase.HTableDescriptor
+import org.apache.hadoop.hbase.client.Put
+
+# Name of this script
+NAME = 'check_meta'
+
+# Print usage for this script
+def usage
+  puts 'Usage: %s.rb [--fix]' % NAME
+  puts ' fix   Try to fixup meta issues'
+  puts 'Script checks consistency of the .META. table.  It reports if .META. has missing entries.'
+  puts 'If you pass "--fix", it will try looking in the filesystem for the dropped region and if it'
+  puts 'finds a likely candidate, it will try pluggin the .META. hole.'
+  exit!
+end
+
+def isFixup
+  # Are we to do fixup during this run
+  usage if ARGV.size > 1
+  fixup = nil
+  if ARGV.size == 1
+    usage unless ARGV[0].downcase.match('--fix.*')
+    fixup = 1
+  end
+  return fixup
+end
+
+def getConfiguration
+  hbase_twenty = VersionInfo.getVersion().match('0\.20\..*')
+  # Get configuration to use.
+  if hbase_twenty
+    c = HBaseConfiguration.new()
+  else
+    c = HBaseConfiguration.create()
+  end
+  # Set hadoop filesystem configuration using the hbase.rootdir.
+  # Otherwise, we'll always use localhost though the hbase.rootdir
+  # might be pointing at hdfs location. Do old and new key for fs.
+  c.set("fs.default.name", c.get(HConstants::HBASE_DIR))
+  c.set("fs.defaultFS", c.get(HConstants::HBASE_DIR))
+  return c
+end
+
+def fixup(leftEdge, rightEdge, metatable, fs, rootdir)
+  plugged = nil
+  # Try and fix the passed holes in meta.
+  tabledir = HTableDescriptor::getTableDir(rootdir, leftEdge.getTableDesc().getName())
+  statuses = fs.listStatus(tabledir)
+  for status in statuses
+    next unless status.isDir()
+    next if status.getPath().getName() == "compaction.dir"
+    regioninfofile =  Path.new(status.getPath(), ".regioninfo")
+    unless fs.exists(regioninfofile)
+      LOG.warn("Missing .regioninfo: " + regioninfofile.toString())
+      next
+    end
+    is = fs.open(regioninfofile)
+    hri = HRegionInfo.new()
+    hri.readFields(is)
+    is.close()
+    next unless Bytes.equals(leftEdge.getEndKey(), hri.getStartKey())
+    # TODO: Check against right edge to make sure this addition does not overflow right edge.
+    # TODO: Check that the schema matches both left and right edges schemas.
+    p = Put.new(hri.getRegionName())
+    p.add(HConstants::CATALOG_FAMILY, HConstants::REGIONINFO_QUALIFIER, Writables.getBytes(hri))
+    metatable.put(p)
+    LOG.info("Plugged hole in .META. at: " + hri.toString())
+    plugged = true
+  end
+  return plugged
+end
+
+fixup = isFixup()
+
+# Get configuration
+conf = getConfiguration()
+
+# Filesystem
+fs = FileSystem.get(conf)
+
+# Rootdir
+rootdir = FSUtils.getRootDir(conf)
+
+# Get a logger and a metautils instance.
+LOG = LogFactory.getLog(NAME)
+
+# Scan the .META. looking for holes
+metatable = HTable.new(conf, HConstants::META_TABLE_NAME)
+scan = Scan.new()
+scanner = metatable.getScanner(scan)
+oldHRI = nil
+bad = nil
+while (result = scanner.next())
+  rowid = Bytes.toString(result.getRow())
+  rowidStr = java.lang.String.new(rowid)
+  bytes = result.getValue(HConstants::CATALOG_FAMILY, HConstants::REGIONINFO_QUALIFIER)
+  hri = Writables.getHRegionInfo(bytes)
+  if oldHRI
+    if oldHRI.isOffline() && Bytes.equals(oldHRI.getStartKey(), hri.getStartKey())
+      # Presume offlined parent
+    elsif Bytes.equals(oldHRI.getEndKey(), hri.getStartKey())
+      # Start key of next matches end key of previous
+    else
+      LOG.warn("hole after " + oldHRI.toString())
+      if fixup
+        bad = 1 unless fixup(oldHRI, hri, metatable, fs, rootdir)
+      else
+        bad = 1
+      end
+    end
+  end
+  oldHRI = hri
+end
+scanner.close()
+if bad
+  LOG.info(".META. has holes")
+else
+  LOG.info(".META. is healthy")
+end
+
+# Return 0 if meta is good, else non-zero.
+exit bad

Modified: hbase/branches/0.89/bin/hirb.rb
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/bin/hirb.rb?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/bin/hirb.rb (original)
+++ hbase/branches/0.89/bin/hirb.rb Tue Sep 27 02:41:16 2011
@@ -53,12 +53,10 @@ end
 cmdline_help = <<HERE # HERE document output as shell usage
 HBase Shell command-line options:
  format        Formatter for outputting results: console | html. Default: console
- format-width  Width of table outputs. Default: 110 characters.
  -d | --debug  Set DEBUG log levels.
 HERE
 found = []
 format = 'console'
-format_width = 110
 script2run = nil
 log_level = org.apache.log4j.Level::ERROR
 for arg in ARGV
@@ -72,9 +70,6 @@ for arg in ARGV
       raise ArgumentError.new("Unsupported format " + arg)
     end
     found.push(arg)
-  elsif arg =~ /^--format-width=(.+)/i
-    format_width = $1.to_i
-    found.push(arg)
   elsif arg == '-h' || arg == '--help'
     puts cmdline_help
     exit
@@ -110,7 +105,7 @@ require 'shell/formatter'
 
 # Presume console format.
 # Formatter takes an :output_stream parameter, if you don't want STDOUT.
-@formatter = Shell::Formatter::Console.new(:format_width => format_width)
+@formatter = Shell::Formatter::Console.new
 
 # Setup the HBase module.  Create a configuration.
 @hbase = Hbase::Hbase.new

Modified: hbase/branches/0.89/pom.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/pom.xml?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/pom.xml (original)
+++ hbase/branches/0.89/pom.xml Tue Sep 27 02:41:16 2011
@@ -7,10 +7,13 @@
   <packaging>jar</packaging>
   <version>0.21.0-SNAPSHOT</version>
   <name>HBase</name>
-  <description>HBase is the Hadoop database. Use it when you need random, realtime read/write access to your Big Data.
+  <!-- START SNIPPET: description -->
+  <description>
+  HBase is the &amp;lt;a href="http://hadoop.apache.org"&amp;rt;Hadoop&lt;/a&amp;rt; database. Use it when you need random, realtime read/write access to your Big Data.
     This project's goal is the hosting of very large tables -- billions of rows X millions of columns -- atop clusters
     of commodity hardware.
   </description>
+  <!-- END SNIPPET: description -->
   <url>http://hbase.apache.org</url>
 
   <licenses>
@@ -22,6 +25,11 @@
     </license>
   </licenses>
 
+  <organization>
+    <name>Apache Software Foundation</name>
+    <url>http://www.apache.org</url>
+  </organization>
+
   <scm>
     <connection>scm:svn:http://svn.apache.org/repos/asf/hbase/trunk</connection>
     <developerConnection>scm:svn:https://svn.apache.org/repos/asf/hbase/trunk</developerConnection>
@@ -122,11 +130,31 @@
       <organization>StumbleUpon</organization>
       <organizationUrl>http://www.stumbleupon.com/</organizationUrl>
     </developer>
+    <developer>
+      <id>todd</id>
+      <name>Todd Lipcon</name>
+      <email>todd@apache.org</email>
+      <timezone>-8</timezone>
+      <organization>Cloudera</organization>
+      <organizationUrl>http://www.cloudera.com</organizationUrl>
+    </developer>
   </developers>
 
   <repositories>
-     <!-- temporary: should replace jgray.la with our internal mirror once that's setup -->
-     <repository>
+    <repository>
+      <id>mirror.facebook.net</id>
+      <name>facebook mirror</name>
+      <url>http://mirror.facebook.net/maven/repository/</url>
+      <snapshots>
+        <enabled>true</enabled>
+        </snapshots>
+      <releases>
+        <enabled>true</enabled>
+      </releases>
+    </repository>
+
+    <!-- temporary: should replace jgray.la with our internal mirror once that's setup -->
+    <repository>
       <id>jgray.la</id>
       <name>jgray mirror</name>
       <url>http://jgray.la/maven/repository/</url>
@@ -192,6 +220,30 @@
         <enabled>true</enabled>
       </releases>
     </repository>
+    <repository>
+      <!-- Guava on standard maven repos has a bad checksum, hosting
+           here temporarily -->
+      <id>temp-todd-for-guava</id>
+      <name>Todd Lipcon's repository on apache.org</name>
+      <url>http://people.apache.org/~todd/repo/</url>
+      <snapshots>
+        <enabled>false</enabled>
+      </snapshots>
+      <releases>
+        <enabled>true</enabled>
+      </releases>
+    </repository>
+    <repository>
+      <id>ibiblio.org</id>
+      <name>ibiblio mirror</name>
+      <url>http://mirrors.ibiblio.org/pub/mirrors/maven2/</url>
+      <snapshots>
+        <enabled>false</enabled>
+      </snapshots>
+      <releases>
+        <enabled>true</enabled>
+      </releases>
+    </repository>
   </repositories>
 
 
@@ -239,6 +291,15 @@
         <plugin>
           <artifactId>maven-clean-plugin</artifactId>
           <version>2.4</version>
+            <configuration>
+              <filesets>
+                <fileset>
+                  <!--dfs tests have build dir hardcoded. Clean it as part of
+                      clean target-->
+                  <directory>build</directory>
+                </fileset>
+              </filesets>
+            </configuration>
         </plugin>
         <plugin>
           <groupId>org.apache.rat</groupId>
@@ -252,7 +313,7 @@
         <directory>src/main/resources</directory>
       </resource>
       <resource>
-        <directory>src/main/webapps</directory>
+        <directory>src/main/hbase-webapps</directory>
         <includes>
           <include>*/.gif</include>
           <include>*/.css</include>
@@ -341,14 +402,14 @@
             <configuration>
               <tasks>
                 <taskdef classname="org.apache.jasper.JspC" name="jspcompiler" classpathref="maven.compile.classpath"/>
-                <jspcompiler uriroot="${basedir}/src/main/resources/webapps/master"
+                <jspcompiler uriroot="${basedir}/src/main/resources/hbase-webapps/master"
                              outputdir="${project.build.directory}/jspc"
                              package="org.apache.hadoop.hbase.generated.master"
-                             webxml="${basedir}/src/main/resources/webapps/master/WEB-INF/web.xml"/>
-                <jspcompiler uriroot="${basedir}/src/main/resources/webapps/regionserver"
+                             webxml="${basedir}/src/main/resources/hbase-webapps/master/WEB-INF/web.xml"/>
+                <jspcompiler uriroot="${basedir}/src/main/resources/hbase-webapps/regionserver"
                              outputdir="${project.build.directory}/jspc"
                              package="org.apache.hadoop.hbase.generated.regionserver"
-                             webxml="${basedir}/src/main/resources/webapps/regionserver/WEB-INF/web.xml"/>
+                             webxml="${basedir}/src/main/resources/hbase-webapps/regionserver/WEB-INF/web.xml"/>
               </tasks>
             </configuration>
             <goals>
@@ -412,8 +473,10 @@
 
     <commons-cli.version>1.2</commons-cli.version>
     <commons-logging.version>1.1.1</commons-logging.version>
-    <jetty.version>6.1.14</jetty.version>
+    <jetty.version>6.1.24</jetty.version>
+    <jetty.jspapi.version>6.1.14</jetty.jspapi.version>
     <junit.version>4.8.1</junit.version>
+    <mockito-all.version>1.8.4</mockito-all.version>
     <log4j.version>1.2.15</log4j.version>
     <zookeeper.version>3.3.1</zookeeper.version>
 
@@ -429,6 +492,7 @@
     <slf4j.version>1.5.8</slf4j.version>
     <stax-api>1.0.1</stax-api>
     <thrift.version>0.2.0</thrift.version>
+    <guava.version>r03</guava.version>
   </properties>
 
   <dependencyManagement>
@@ -494,7 +558,7 @@
       <dependency>
         <groupId>org.mortbay.jetty</groupId>
         <artifactId>servlet-api-2.5</artifactId>
-        <version>${jetty.version}</version>
+        <version>${jetty.jspapi.version}</version>
       </dependency>
 
       <!-- Test dependencies -->
@@ -510,6 +574,12 @@
         <version>${junit.version}</version>
         <scope>test</scope>
       </dependency>
+      <dependency>
+        <groupId>org.mockito</groupId>
+        <artifactId>mockito-all</artifactId>
+        <version>${mockito-all.version}</version>
+        <scope>test</scope>
+      </dependency>
     </dependencies>
   </dependencyManagement>
   <dependencies>
@@ -574,6 +644,12 @@
     </dependency>
 
     <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>r03</version>
+    </dependency>
+
+    <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-core</artifactId>
     </dependency>
@@ -600,12 +676,12 @@
     <dependency>
       <groupId>org.mortbay.jetty</groupId>
       <artifactId>jsp-2.1</artifactId>
-      <version>${jetty.version}</version>
+      <version>${jetty.jspapi.version}</version>
     </dependency>
     <dependency>
       <groupId>org.mortbay.jetty</groupId>
       <artifactId>jsp-api-2.1</artifactId>
-      <version>${jetty.version}</version>
+      <version>${jetty.jspapi.version}</version>
     </dependency>
     <dependency>
       <groupId>tomcat</groupId>
@@ -673,15 +749,24 @@
       <artifactId>junit</artifactId>
     </dependency>
     <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+    </dependency>
+    <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-math</artifactId>
       <version>${commons-math.version}</version>
       <scope>test</scope>
     </dependency>
-        <dependency>
+     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-test</artifactId>
     </dependency>
+     <dependency>
+       <groupId>com.google.guava</groupId>
+       <artifactId>guava</artifactId>
+       <version>${guava.version}</version>
+    </dependency>
   </dependencies>
 
   <!--
@@ -826,10 +911,12 @@
         <artifactId>maven-surefire-report-plugin</artifactId>
         <version>2.5</version>
       </plugin>
+      <!--
       <plugin>
         <artifactId>maven-dependency-plugin</artifactId>
         <version>2.1</version>
       </plugin>
+      -->
       <plugin>
         <groupId>org.apache.rat</groupId>
         <artifactId>apache-rat-plugin</artifactId>

Modified: hbase/branches/0.89/src/assembly/bin.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/assembly/bin.xml?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/assembly/bin.xml (original)
+++ hbase/branches/0.89/src/assembly/bin.xml Tue Sep 27 02:41:16 2011
@@ -34,12 +34,23 @@
       </includes>
     </fileSet>
     <fileSet>
-      <directory>target/classes/webapps</directory>
-      <outputDirectory>webapps</outputDirectory>
+      <directory>target/classes/hbase-webapps</directory>
+      <outputDirectory>hbase-webapps</outputDirectory>
       <excludes>
         <exclude>**/*.jsp</exclude>
       </excludes>
     </fileSet>
+    <fileSet>
+      <directory>target/site</directory>
+      <outputDirectory>docs</outputDirectory>
+    </fileSet>
+    <fileSet>
+      <directory>src/main/resources/</directory>
+      <outputDirectory>conf</outputDirectory>
+      <includes>
+        <include>hbase-default.xml</include>
+      </includes>
+    </fileSet>
   </fileSets>
   <dependencySets>
     <dependencySet>

Copied: hbase/branches/0.89/src/assembly/bin.xml.orig (from r1176170, hbase/branches/0.89/src/assembly/bin.xml)
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/assembly/bin.xml.orig?p2=hbase/branches/0.89/src/assembly/bin.xml.orig&p1=hbase/branches/0.89/src/assembly/bin.xml&r1=1176170&r2=1176171&rev=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/assembly/bin.xml (original)
+++ hbase/branches/0.89/src/assembly/bin.xml.orig Tue Sep 27 02:41:16 2011
@@ -13,6 +13,7 @@
         <include>${basedir}/README*</include>
         <include>${basedir}/LICENSE*</include>
         <include>${basedir}/NOTICE*</include>
+        <include>${basedir}/lib/**</include>
       </includes>
     </fileSet>
     <fileSet>
@@ -34,8 +35,8 @@
       </includes>
     </fileSet>
     <fileSet>
-      <directory>target/classes/webapps</directory>
-      <outputDirectory>webapps</outputDirectory>
+      <directory>target/classes/hbase-webapps</directory>
+      <outputDirectory>hbase-webapps</outputDirectory>
       <excludes>
         <exclude>**/*.jsp</exclude>
       </excludes>

Added: hbase/branches/0.89/src/docs/src/documentation/content/xdocs/bulk-loads.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/docs/src/documentation/content/xdocs/bulk-loads.xml?rev=1176171&view=auto
==============================================================================
--- hbase/branches/0.89/src/docs/src/documentation/content/xdocs/bulk-loads.xml (added)
+++ hbase/branches/0.89/src/docs/src/documentation/content/xdocs/bulk-loads.xml Tue Sep 27 02:41:16 2011
@@ -0,0 +1,148 @@
+<?xml version="1.0"?>
+<!--
+  Copyright 2010 The Apache Software Foundation
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN"
+          "http://forrest.apache.org/dtd/document-v20.dtd">
+
+
+<document>
+
+  <header>
+    <title>
+      Bulk Loads in HBase
+    </title>
+  </header>
+  <body>
+    <section>
+      <title>Overview</title>
+      <p>
+        HBase includes several methods of loading data into tables.
+        The most straightforward method is to either use the TableOutputFormat
+        class from a MapReduce job, or use the normal client APIs; however,
+        these are not always the most efficient methods.
+      </p>
+      <p>
+        This document describes HBase's bulk load functionality. The bulk load
+        feature uses a MapReduce job to output table data in HBase's internal
+        data format, and then directly loads the data files into a running
+        cluster.
+      </p>
+    </section>
+    <section>
+      <title>Bulk Load Architecture</title>
+      <p>
+        The HBase bulk load process consists of two main steps.
+      </p>
+      <subsection>
+        <title>Preparing data via a MapReduce job</title>
+        <p>
+          The first step of a bulk load is to generate HBase data files from
+          a MapReduce job using HFileOutputFormat. This output format writes
+          out data in HBase's internal storage format so that they can be
+          later loaded very efficiently into the cluster.
+        </p>
+        <p>
+          In order to function efficiently, HFileOutputFormat must be configured
+          such that each output HFile fits within a single region. In order to
+          do this, jobs use Hadoop's TotalOrderPartitioner class to partition the
+          map output into disjoint ranges of the key space, corresponding to the
+          key ranges of the regions in the table.
+        </p>
+        <p>
+          HFileOutputFormat includes a convenience function, <code>configureIncrementalLoad()</code>,
+          which automatically sets up a TotalOrderPartitioner based on the current
+          region boundaries of a table.
+        </p>
+      </subsection>
+      <subsection>
+        <title>Completing the data load</title>
+        <p>
+          After the data has been prepared using <code>HFileOutputFormat</code>, it
+          is loaded into the cluster using a command line tool. This command line tool
+          iterates through the prepared data files, and for each one determines the
+          region the file belongs to. It then contacts the appropriate Region Server
+          which adopts the HFile, moving it into its storage directory and making
+          the data available to clients.
+        </p>
+        <p>
+          If the region boundaries have changed during the course of bulk load
+          preparation, or between the preparation and completion steps, the bulk
+          load commandline utility will automatically split the data files into
+          pieces corresponding to the new boundaries. This process is not
+          optimally efficient, so users should take care to minimize the delay between
+          preparing a bulk load and importing it into the cluster, especially
+          if other clients are simultaneously loading data through other means.
+        </p>
+      </subsection>
+    </section>
+    <section>
+      <title>Preparing a bulk load using the <code>importtsv</code> tool</title>
+      <p>
+        HBase ships with a command line tool called <code>importtsv</code>. This tool
+        is available by running <code>hadoop jar /path/to/hbase-VERSION.jar importtsv</code>.
+        Running this tool with no arguments prints brief usage information:
+      </p>
+      <code><pre>
+Usage: importtsv -Dimporttsv.columns=a,b,c &lt;tablename&gt; &lt;inputdir&gt;
+
+Imports the given input directory of TSV data into the specified table.
+
+The column names of the TSV data must be specified using the -Dimporttsv.columns
+option. This option takes the form of comma-separated column names, where each
+column name is either a simple column family, or a columnfamily:qualifier. The special
+column name HBASE_ROW_KEY is used to designate that this column should be used
+as the row key for each imported record. You must specify exactly one column
+to be the row key.
+
+In order to prepare data for a bulk data load, pass the option:
+  -Dimporttsv.bulk.output=/path/for/output
+
+Other options that may be specified with -D include:
+  -Dimporttsv.skip.bad.lines=false - fail if encountering an invalid line
+</pre></code>
+    </section>
+    <section>
+      <title>Importing the prepared data using the <code>completebulkload</code> tool</title>
+      <p>
+        After a data import has been prepared using the <code>importtsv</code> tool, the
+        <code>completebulkload</code> tool is used to import the data into the running cluster.
+      </p>
+      <p>
+        The <code>completebulkload</code> tool simply takes the same output path where
+        <code>importtsv</code> put its results, and the table name. For example:
+      </p>
+      <code>$ hadoop jar hbase-VERSION.jar completebulkload /user/todd/myoutput mytable</code>
+      <p>
+        This tool will run quickly, after which point the new data will be visible in
+        the cluster.
+      </p>
+    </section>
+    <section>
+      <title>Advanced Usage</title>
+      <p>
+        Although the <code>importtsv</code> tool is useful in many cases, advanced users may
+        want to generate data programatically, or import data from other formats. To get
+        started doing so, dig into <code>ImportTsv.java</code> and check the JavaDoc for
+        HFileOutputFormat.
+      </p>
+      <p>
+        The import step of the bulk load can also be done programatically. See the
+        <code>LoadIncrementalHFiles</code> class for more information.
+      </p>
+    </section>
+  </body>
+</document>
\ No newline at end of file

Modified: hbase/branches/0.89/src/docs/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/docs/src/documentation/content/xdocs/site.xml?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/docs/src/documentation/content/xdocs/site.xml (original)
+++ hbase/branches/0.89/src/docs/src/documentation/content/xdocs/site.xml Tue Sep 27 02:41:16 2011
@@ -35,6 +35,7 @@ See http://forrest.apache.org/docs/linki
     <overview  label="Overview"           href="index.html" />
     <started   label="Getting Started"    href="ext:api/started" />
     <api       label="API Docs"           href="ext:api/index" />
+    <api       label="Bulk Loads"         href="bulk-load.html" />
     <api       label="HBase Metrics"      href="metrics.html" />
     <api       label="HBase Semantics"      href="acid-semantics.html" />
     <api       label="HBase  Default Configuration" href="hbase-conf.html" />

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HConstants.java Tue Sep 27 02:41:16 2011
@@ -29,12 +29,7 @@ public interface HConstants {
 
   /** long constant for zero */
   static final Long ZERO_L = Long.valueOf(0L);
-
-  //TODO: NINES is only used in HBaseAdmin and HConnectionManager. Move to client
-  //      package and change visibility to default
   static final String NINES = "99999999999999";
-  //TODO: ZEROS is only used in HConnectionManager and MetaScanner. Move to
-  //      client package and change visibility to default
   static final String ZEROES = "00000000000000";
 
   // For migration

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HMsg.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HMsg.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HMsg.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HMsg.java Tue Sep 27 02:41:16 2011
@@ -38,8 +38,6 @@ public class HMsg implements Writable {
     new HMsg(Type.MSG_REGIONSERVER_QUIESCE);
   public static final HMsg REGIONSERVER_STOP =
     new HMsg(Type.MSG_REGIONSERVER_STOP);
-  public static final HMsg CALL_SERVER_STARTUP =
-    new HMsg(Type.MSG_CALL_SERVER_STARTUP);
   public static final HMsg [] EMPTY_HMSG_ARRAY = new HMsg[0];
 
   /**
@@ -62,9 +60,6 @@ public class HMsg implements Writable {
     /** Compact the specified region */
     MSG_REGION_COMPACT,
 
-    /** Region server is unknown to master. Restart */
-    MSG_CALL_SERVER_STARTUP,
-
     /** Master tells region server to stop */
     MSG_REGIONSERVER_STOP,
 

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java Tue Sep 27 02:41:16 2011
@@ -24,9 +24,12 @@ import java.io.DataOutput;
 import java.io.IOException;
 import java.util.Arrays;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.KeyValue.KVComparator;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.JenkinsHash;
+import org.apache.hadoop.hbase.util.MD5Hash;
 import org.apache.hadoop.io.VersionedWritable;
 import org.apache.hadoop.io.WritableComparable;
 
@@ -37,13 +40,74 @@ import org.apache.hadoop.io.WritableComp
  */
 public class HRegionInfo extends VersionedWritable implements WritableComparable<HRegionInfo>{
   private static final byte VERSION = 0;
+  private static final Log LOG = LogFactory.getLog(HRegionInfo.class);
+
+  /**
+   * The new format for a region name contains its encodedName at the end.
+   * The encoded name also serves as the directory name for the region
+   * in the filesystem.
+   *
+   * New region name format:
+   *    &lt;tablename>,,&lt;startkey>,&lt;regionIdTimestamp>.&lt;encodedName>.
+   * where,
+   *    &lt;encodedName> is a hex version of the MD5 hash of
+   *    &lt;tablename>,&lt;startkey>,&lt;regionIdTimestamp>
+   *
+   * The old region name format:
+   *    &lt;tablename>,&lt;startkey>,&lt;regionIdTimestamp>
+   * For region names in the old format, the encoded name is a 32-bit
+   * JenkinsHash integer value (in its decimal notation, string form).
+   *<p>
+   * **NOTE**
+   *
+   * ROOT, the first META region, and regions created by an older
+   * version of HBase (0.20 or prior) will continue to use the
+   * old region name format.
+   */
+
+  /** Separator used to demarcate the encodedName in a region name
+   * in the new format. See description on new format above.
+   */
+  private static final int ENC_SEPARATOR = '.';
+  public  static final int MD5_HEX_LENGTH   = 32;
+
+  /**
+   * Does region name contain its encoded name?
+   * @param regionName region name
+   * @return boolean indicating if this a new format region
+   *         name which contains its encoded name.
+   */
+  private static boolean hasEncodedName(final byte[] regionName) {
+    // check if region name ends in ENC_SEPARATOR
+    if ((regionName.length >= 1)
+        && (regionName[regionName.length - 1] == ENC_SEPARATOR)) {
+      // region name is new format. it contains the encoded name.
+      return true;
+    }
+    return false;
+  }
 
   /**
    * @param regionName
    * @return the encodedName
    */
-  public static int encodeRegionName(final byte [] regionName) {
-    return Math.abs(JenkinsHash.getInstance().hash(regionName, regionName.length, 0));
+  public static String encodeRegionName(final byte [] regionName) {
+    String encodedName;
+    if (hasEncodedName(regionName)) {
+      // region is in new format:
+      // <tableName>,<startKey>,<regionIdTimeStamp>/encodedName/
+      encodedName = Bytes.toString(regionName,
+          regionName.length - MD5_HEX_LENGTH - 1,
+          MD5_HEX_LENGTH);
+    } else {
+      // old format region name. ROOT and first META region also
+      // use this format.EncodedName is the JenkinsHash value.
+      int hashVal = Math.abs(JenkinsHash.getInstance().hash(regionName,
+                                                            regionName.length,
+                                                            0));
+      encodedName = String.valueOf(hashVal);
+    }
+    return encodedName;
   }
 
   /** delimiter used between portions of a region name */
@@ -67,8 +131,8 @@ public class HRegionInfo extends Version
   protected HTableDescriptor tableDesc = null;
   private int hashCode = -1;
   //TODO: Move NO_HASH to HStoreFile which is really the only place it is used.
-  public static final int NO_HASH = -1;
-  private volatile int encodedName = NO_HASH;
+  public static final String NO_HASH = null;
+  private volatile String encodedName = NO_HASH;
 
   private void setHashCode() {
     int result = Arrays.hashCode(this.regionName);
@@ -88,7 +152,10 @@ public class HRegionInfo extends Version
     super();
     this.regionId = regionId;
     this.tableDesc = tableDesc;
-    this.regionName = createRegionName(tableDesc.getName(), null, regionId);
+
+    // Note: Root & First Meta regions names are still in old format
+    this.regionName = createRegionName(tableDesc.getName(), null,
+                                       regionId, false);
     this.regionNameStr = Bytes.toStringBinary(this.regionName);
     setHashCode();
   }
@@ -149,7 +216,7 @@ public class HRegionInfo extends Version
     }
     this.offLine = false;
     this.regionId = regionid;
-    this.regionName = createRegionName(tableDesc.getName(), startKey, regionId);
+    this.regionName = createRegionName(tableDesc.getName(), startKey, regionId, true);
     this.regionNameStr = Bytes.toStringBinary(this.regionName);
     this.split = split;
     this.endKey = endKey == null? HConstants.EMPTY_END_ROW: endKey.clone();
@@ -179,8 +246,8 @@ public class HRegionInfo extends Version
   }
 
   private static byte [] createRegionName(final byte [] tableName,
-      final byte [] startKey, final long regionid) {
-    return createRegionName(tableName, startKey, Long.toString(regionid));
+      final byte [] startKey, final long regionid, boolean newFormat) {
+    return createRegionName(tableName, startKey, Long.toString(regionid), newFormat);
   }
 
   /**
@@ -188,23 +255,29 @@ public class HRegionInfo extends Version
    * @param tableName
    * @param startKey Can be null
    * @param id Region id.
+   * @param newFormat should we create the region name in the new format
+   *                  (such that it contains its encoded name?).
    * @return Region name made of passed tableName, startKey and id
    */
   public static byte [] createRegionName(final byte [] tableName,
-      final byte [] startKey, final String id) {
-    return createRegionName(tableName, startKey, Bytes.toBytes(id));
+      final byte [] startKey, final String id, boolean newFormat) {
+    return createRegionName(tableName, startKey, Bytes.toBytes(id), newFormat);
   }
   /**
    * Make a region name of passed parameters.
    * @param tableName
    * @param startKey Can be null
    * @param id Region id
+   * @param newFormat should we create the region name in the new format
+   *                  (such that it contains its encoded name?).
    * @return Region name made of passed tableName, startKey and id
    */
   public static byte [] createRegionName(final byte [] tableName,
-      final byte [] startKey, final byte [] id) {
+      final byte [] startKey, final byte [] id, boolean newFormat) {
     byte [] b = new byte [tableName.length + 2 + id.length +
-       (startKey == null? 0: startKey.length)];
+       (startKey == null? 0: startKey.length) +
+       (newFormat ? (MD5_HEX_LENGTH + 2) : 0)];
+
     int offset = tableName.length;
     System.arraycopy(tableName, 0, b, 0, offset);
     b[offset++] = DELIMITER;
@@ -214,6 +287,31 @@ public class HRegionInfo extends Version
     }
     b[offset++] = DELIMITER;
     System.arraycopy(id, 0, b, offset, id.length);
+    offset += id.length;
+
+    if (newFormat) {
+      //
+      // Encoded name should be built into the region name.
+      //
+      // Use the region name thus far (namely, <tablename>,<startKey>,<id>)
+      // to compute a MD5 hash to be used as the encoded name, and append
+      // it to the byte buffer.
+      //
+      String md5Hash = MD5Hash.getMD5AsHex(b, 0, offset);
+      byte [] md5HashBytes = Bytes.toBytes(md5Hash);
+
+      if (md5HashBytes.length != MD5_HEX_LENGTH) {
+        LOG.error("MD5-hash length mismatch: Expected=" + MD5_HEX_LENGTH +
+                  "; Got=" + md5HashBytes.length);
+      }
+
+      // now append the bytes '.<encodedName>.' to the end
+      b[offset++] = ENC_SEPARATOR;
+      System.arraycopy(md5HashBytes, 0, b, offset, MD5_HEX_LENGTH);
+      offset += MD5_HEX_LENGTH;
+      b[offset++] = ENC_SEPARATOR;
+    }
+
     return b;
   }
 
@@ -259,11 +357,6 @@ public class HRegionInfo extends Version
     return elements;
   }
 
-  /** @return the endKey */
-  public byte [] getEndKey(){
-    return endKey;
-  }
-
   /** @return the regionId */
   public long getRegionId(){
     return regionId;
@@ -281,11 +374,19 @@ public class HRegionInfo extends Version
    * @return Region name as a String for use in logging, etc.
    */
   public String getRegionNameAsString() {
-    return this.regionNameStr;
+    if (hasEncodedName(this.regionName)) {
+      // new format region names already have their encoded name.
+      return this.regionNameStr;
+    }
+
+    // old format. regionNameStr doesn't have the region name.
+    //
+    //
+    return this.regionNameStr + "." + this.getEncodedName();
   }
 
   /** @return the encoded region name */
-  public synchronized int getEncodedName() {
+  public synchronized String getEncodedName() {
     if (this.encodedName == NO_HASH) {
       this.encodedName = encodeRegionName(this.regionName);
     }
@@ -297,6 +398,32 @@ public class HRegionInfo extends Version
     return startKey;
   }
 
+  /** @return the endKey */
+  public byte [] getEndKey(){
+    return endKey;
+  }
+
+  /**
+   * Returns true if the given inclusive range of rows is fully contained
+   * by this region. For example, if the region is foo,a,g and this is
+   * passed ["b","c"] or ["a","c"] it will return true, but if this is passed
+   * ["b","z"] it will return false.
+   * @throws IllegalArgumentException if the range passed is invalid (ie end < start)
+   */
+  public boolean containsRange(byte[] rangeStartKey, byte[] rangeEndKey) {
+    if (Bytes.compareTo(rangeStartKey, rangeEndKey) > 0) {
+      throw new IllegalArgumentException(
+      "Invalid range: " + Bytes.toStringBinary(rangeStartKey) +
+      " > " + Bytes.toStringBinary(rangeEndKey));
+    }
+
+    boolean firstKeyInRange = Bytes.compareTo(rangeStartKey, startKey) >= 0;
+    boolean lastKeyInRange =
+      Bytes.compareTo(rangeEndKey, endKey) < 0 ||
+      Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY);
+    return firstKeyInRange && lastKeyInRange;
+  }
+
   /** @return the tableDesc */
   public HTableDescriptor getTableDesc(){
     return tableDesc;

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HServerInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HServerInfo.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HServerInfo.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HServerInfo.java Tue Sep 27 02:41:16 2011
@@ -1,5 +1,5 @@
 /**
- * Copyright 2007 The Apache Software Foundation
+ * Copyright 2010 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -22,26 +22,33 @@ package org.apache.hadoop.hbase;
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
+import java.net.InetSocketAddress;
+import java.util.Set;
 
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 
 
 /**
- * HServerInfo is meta info about an {@link HRegionServer}.
- * Holds hostname, ports, regionserver startcode, and load.  Each server has
+ * HServerInfo is meta info about an {@link HRegionServer}.  It is the token
+ * by which a master distingushes a particular regionserver from the rest.
+ * It holds hostname, ports, regionserver startcode, and load.  Each server has
  * a <code>servername</code> where servername is made up of a concatenation of
- * hostname, port, and regionserver startcode.
+ * hostname, port, and regionserver startcode.  This servername is used in
+ * various places identifying this regionserver.  Its even used as part of
+ * a pathname in the filesystem.  As part of the initialization,
+ * master will pass the regionserver the address that it knows this regionserver
+ * by.  In subsequent communications, the regionserver will pass a HServerInfo
+ * with the master-supplied address.
  */
 public class HServerInfo implements WritableComparable<HServerInfo> {
-  /**
-   * This character is used as separator making up the <code>servername</code>.
-   * Servername is made of host, port, and startcode formatted as
-   * <code>&lt;hostname> '{@link #SERVERNAME_SEPARATOR}' &lt;port> '{@ink #SEPARATOR"}' &lt;startcode></code>
-   * where {@link SEPARATOR is usually a ','.
+  /*
+   * This character is used as separator between server hostname and port and
+   * its startcode. Servername is formatted as
+   * <code>&lt;hostname> '{@ink #SERVERNAME_SEPARATOR"}' &lt;port> '{@ink #SERVERNAME_SEPARATOR"}' &lt;startcode></code>.
    */
-  public static final String SERVERNAME_SEPARATOR = ",";
+  private static final String SERVERNAME_SEPARATOR = ",";
 
   private HServerAddress serverAddress;
   private long startCode;
@@ -51,11 +58,23 @@ public class HServerInfo implements Writ
   private String serverName = null;
   // Hostname of the regionserver.
   private String hostname;
-  private static Map<String,String> dnsCache = new HashMap<String,String>();
+  private String cachedHostnamePort = null;
 
   public HServerInfo() {
-    this(new HServerAddress(), 0,
-        HConstants.DEFAULT_REGIONSERVER_INFOPORT, "default name");
+    this(new HServerAddress(), 0, HConstants.DEFAULT_REGIONSERVER_INFOPORT,
+      "default name");
+  }
+
+  /**
+   * Constructor that creates a HServerInfo with a generated startcode and an
+   * empty load.
+   * @param serverAddress An {@link InetSocketAddress} encased in a {@link Writable}
+   * @param infoPort Port the webui runs on.
+   * @param hostname Server hostname.
+   */
+  public HServerInfo(HServerAddress serverAddress, final int infoPort,
+      final String hostname) {
+    this(serverAddress, System.currentTimeMillis(), infoPort, hostname);
   }
 
   public HServerInfo(HServerAddress serverAddress, long startCode,
@@ -104,13 +123,27 @@ public class HServerInfo implements Writ
     return this.infoPort;
   }
 
-  public void setInfoPort(int infoPort) {
-    this.infoPort = infoPort;
+  public String getHostname() {
+    return this.hostname;
   }
 
-  public synchronized void setStartCode(long startCode) {
-    this.startCode = startCode;
-    this.serverName = null;
+  /**
+   * @return The hostname and port concatenated with a ':' as separator.
+   */
+  public synchronized String getHostnamePort() {
+    if (this.cachedHostnamePort == null) {
+      this.cachedHostnamePort = getHostnamePort(this.hostname, this.serverAddress.getPort());
+    }
+    return this.cachedHostnamePort;
+  }
+
+  /**
+   * @param hostname
+   * @param port
+   * @return The hostname and port concatenated with a ':' as separator.
+   */
+  public static String getHostnamePort(final String hostname, final int port) {
+    return hostname + ":" + port;
   }
 
   /**
@@ -119,43 +152,18 @@ public class HServerInfo implements Writ
    */
   public synchronized String getServerName() {
     if (this.serverName == null) {
-      // if we have the hostname of the RS, use it
-      if(this.hostname != null) {
-        this.serverName =
-          getServerName(this.hostname, this.serverAddress.getPort(), this.startCode);
-      }
-      // go to DNS name resolution only if we dont have the name of the RS
-      else {
-      this.serverName = getServerName(this.serverAddress, this.startCode);
-    }
+      this.serverName = getServerName(this.hostname,
+        this.serverAddress.getPort(), this.startCode);
     }
     return this.serverName;
   }
 
-  /**
-   * @param serverAddress In form <code>&lt;hostname> ':' &lt;port></code>
-   * @param startCode Server startcode
-   * @return Server name made of the concatenation of hostname, port and
-   * startcode formatted as <code>&lt;hostname> ',' &lt;port> ',' &lt;startcode></code>
-   */
-  public static String getServerName(String serverAddress, long startCode) {
-    String name = null;
-    if (serverAddress != null) {
-      int colonIndex = serverAddress.lastIndexOf(':');
-      if(colonIndex < 0) {
-        throw new IllegalArgumentException("Not a host:port pair: " + serverAddress);
-      }
-      String host = serverAddress.substring(0, colonIndex);
-      int port =
-        Integer.valueOf(serverAddress.substring(colonIndex + 1)).intValue();
-      if(!dnsCache.containsKey(host)) {
-        HServerAddress address = new HServerAddress(serverAddress);
-        dnsCache.put(host, address.getHostname());
-      }
-      host = dnsCache.get(host);
-      name = getServerName(host, port, startCode);
-    }
-    return name;
+  public static synchronized String getServerName(final String hostAndPort,
+      final long startcode) {
+    int index = hostAndPort.indexOf(":");
+    if (index <= 0) throw new IllegalArgumentException("Expected <hostname> ':' <port>");
+    return getServerName(hostAndPort.substring(0, index),
+      Integer.parseInt(hostAndPort.substring(index + 1)), startcode);
   }
 
   /**
@@ -175,7 +183,7 @@ public class HServerInfo implements Writ
    * @return Server name made of the concatenation of hostname, port and
    * startcode formatted as <code>&lt;hostname> ',' &lt;port> ',' &lt;startcode></code>
    */
-  private static String getServerName(String hostName, int port, long startCode) {
+  public static String getServerName(String hostName, int port, long startCode) {
     StringBuilder name = new StringBuilder(hostName);
     name.append(SERVERNAME_SEPARATOR);
     name.append(port);
@@ -233,4 +241,27 @@ public class HServerInfo implements Writ
   public int compareTo(HServerInfo o) {
     return this.getServerName().compareTo(o.getServerName());
   }
+
+  /**
+   * Utility method that does a find of a servername or a hostandport combination
+   * in the passed Set.
+   * @param servers Set of server names
+   * @param serverName Name to look for
+   * @param hostAndPortOnly If <code>serverName</code> is a
+   * <code>hostname ':' port</code>
+   * or <code>hostname , port , startcode</code>.
+   * @return True if <code>serverName</code> found in <code>servers</code>
+   */
+  public static boolean isServer(final Set<String> servers,
+      final String serverName, final boolean hostAndPortOnly) {
+    if (!hostAndPortOnly) return servers.contains(serverName);
+    String serverNameColonReplaced =
+      serverName.replaceFirst(":", SERVERNAME_SEPARATOR);
+    for (String hostPortStartCode: servers) {
+      int index = hostPortStartCode.lastIndexOf(SERVERNAME_SEPARATOR);
+      String hostPortStrippedOfStartCode = hostPortStartCode.substring(0, index);
+      if (hostPortStrippedOfStartCode.equals(serverNameColonReplaced)) return true;
+    }
+    return false;
+  }
 }

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java Tue Sep 27 02:41:16 2011
@@ -797,8 +797,13 @@ public class KeyValue implements Writabl
       HConstants.LATEST_TIMESTAMP_BYTES, 0, Bytes.SIZEOF_LONG) == 0;
   }
 
+  /**
+   * @param now Time to set into <code>this</code> IFF timestamp ==
+   * {@link HConstants#LATEST_TIMESTAMP} (else, its a noop).
+   * @return True is we modified this.
+   */
   public boolean updateLatestStamp(final byte [] now) {
-    if(this.isLatestTimestamp()) {
+    if (this.isLatestTimestamp()) {
       int tsOffset = getTimestampOffset();
       System.arraycopy(now, 0, this.bytes, tsOffset, Bytes.SIZEOF_LONG);
       return true;

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java Tue Sep 27 02:41:16 2011
@@ -915,7 +915,7 @@ public class HBaseAdmin {
   private HRegionLocation getFirstMetaServerForTable(final byte [] tableName)
   throws IOException {
     return connection.locateRegion(HConstants.META_TABLE_NAME,
-      HRegionInfo.createRegionName(tableName, null, HConstants.NINES));
+      HRegionInfo.createRegionName(tableName, null, HConstants.NINES, false));
   }
 
   /**

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java Tue Sep 27 02:41:16 2011
@@ -515,7 +515,7 @@ public class HConnectionManager implemen
       int rowsScanned = 0;
       int rowsOffline = 0;
       byte[] startKey =
-        HRegionInfo.createRegionName(tableName, null, HConstants.ZEROES);
+        HRegionInfo.createRegionName(tableName, null, HConstants.ZEROES, false);
       byte[] endKey;
       HRegionInfo currentRegion;
       Scan scan = new Scan(startKey);
@@ -663,7 +663,7 @@ public class HConnectionManager implemen
       // the extra 9's on the end are necessary to allow "exact" matches
       // without knowing the precise region names.
       byte [] metaKey = HRegionInfo.createRegionName(tableName, row,
-        HConstants.NINES);
+        HConstants.NINES, false);
       for (int tries = 0; true; tries++) {
         if (tries >= numRetries) {
           throw new NoServerForRegionException("Unable to find region for "

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTable.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTable.java Tue Sep 27 02:41:16 2011
@@ -514,6 +514,35 @@ public class HTable implements HTableInt
   }
 
   /**
+   * Atomically checks if a row/family/qualifier value match the expectedValue.
+   * If it does, it adds the delete.  If value == null, checks for non-existence
+   * of the value.
+   *
+   * @param row to check
+   * @param family column family
+   * @param qualifier column qualifier
+   * @param value the expected value
+   * @param delete delete to execute if value matches.
+   * @throws IOException
+   * @return true if the new delete was executed, false otherwise
+   */
+  public boolean checkAndDelete(final byte [] row,
+      final byte [] family, final byte [] qualifier, final byte [] value,
+      final Delete delete)
+  throws IOException {
+    return connection.getRegionServerWithRetries(
+        new ServerCallable<Boolean>(connection, tableName, row) {
+          public Boolean call() throws IOException {
+            return server.checkAndDelete(
+                location.getRegionInfo().getRegionName(),
+                row, family, qualifier, value, delete)
+            ? Boolean.TRUE : Boolean.FALSE;
+          }
+        }
+    );
+  }
+
+  /**
    * Test for the existence of columns in the table, as specified in the Get.<p>
    *
    * This will return true if the Get matches one or more keys, false if not.<p>

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java Tue Sep 27 02:41:16 2011
@@ -190,6 +190,22 @@ public interface HTableInterface {
   void delete(List<Delete> deletes) throws IOException;
 
   /**
+   * Atomically checks if a row/family/qualifier value matches the expected
+   * value. If it does, it adds the delete.  If the passed value is null, the
+   * check is for the lack of column (ie: non-existance)
+   *
+   * @param row to check
+   * @param family column family to check
+   * @param qualifier column qualifier to check
+   * @param value the expected value
+   * @param delete data to delete if check succeeds
+   * @throws IOException e
+   * @return true if the new delete was executed, false otherwise
+   */
+  boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier,
+      byte[] value, Delete delete) throws IOException;
+
+  /**
    * Atomically increments a column value.
    * <p>
    * Equivalent to {@code {@link #incrementColumnValue(byte[], byte[], byte[],

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java Tue Sep 27 02:41:16 2011
@@ -63,7 +63,7 @@ class MetaScanner implements HConstants 
     HConnection connection = HConnectionManager.getConnection(configuration);
     byte [] startRow = tableName == null || tableName.length == 0 ?
         HConstants.EMPTY_START_ROW :
-          HRegionInfo.createRegionName(tableName, null, ZEROES);
+          HRegionInfo.createRegionName(tableName, null, ZEROES, false);
 
     // Scan over each meta region
     ScannerCallable callable;

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java Tue Sep 27 02:41:16 2011
@@ -80,15 +80,15 @@ public class ScannerCallable extends Ser
         if (e instanceof RemoteException) {
           ioe = RemoteExceptionHandler.decodeRemoteException((RemoteException)e);
         }
-        if (ioe != null) {
-          if (ioe instanceof NotServingRegionException) {
+        if (ioe == null) throw new IOException(e);
+        if (ioe instanceof NotServingRegionException) {
           // Throw a DNRE so that we break out of cycle of calling NSRE
           // when what we need is to open scanner against new location.
           // Attach NSRE to signal client that it needs to resetup scanner.
           throw new DoNotRetryIOException("Reset scanner", ioe);
-          } else if (ioe instanceof DoNotRetryIOException) {
-            throw ioe;
-          }
+        } else {
+          // The outer layers will retry
+          throw ioe;
         }
       }
       return rrs;

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java Tue Sep 27 02:41:16 2011
@@ -45,8 +45,8 @@ public class ValueFilter extends Compare
 
   /**
    * Constructor.
-   * @param valueCompareOp the compare op for column qualifier matching
-   * @param valueComparator the comparator for column qualifier matching
+   * @param valueCompareOp the compare op for value matching
+   * @param valueComparator the comparator for value matching
    */
   public ValueFilter(final CompareOp valueCompareOp,
       final WritableByteArrayComparable valueComparator) {

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java Tue Sep 27 02:41:16 2011
@@ -23,6 +23,7 @@ package org.apache.hadoop.hbase.io;
 import java.io.IOException;
 import java.io.DataInput;
 import java.io.DataOutput;
+import java.util.Arrays;
 import java.util.List;
 
 import org.apache.hadoop.io.BytesWritable;
@@ -83,7 +84,7 @@ implements WritableComparable<ImmutableB
 
   /**
    * Get the data from the BytesWritable.
-   * @return The data is only valid between 0 and getSize() - 1.
+   * @return The data is only valid between offset and offset+length.
    */
   public byte [] get() {
     if (this.bytes == null) {
@@ -112,7 +113,7 @@ implements WritableComparable<ImmutableB
   }
 
   /**
-   * @return the current size of the buffer.
+   * @return the number of valid bytes in the buffer
    */
   public int getSize() {
     if (this.bytes == null) {
@@ -123,7 +124,7 @@ implements WritableComparable<ImmutableB
   }
 
   /**
-   * @return the current length of the buffer. same as getSize()
+   * @return the number of valid bytes in the buffer
    */
   //Should probably deprecate getSize() so that we keep the same calls for all
   //byte []
@@ -155,20 +156,24 @@ implements WritableComparable<ImmutableB
   }
 
   // Below methods copied from BytesWritable
-
   @Override
   public int hashCode() {
-    return WritableComparator.hashBytes(bytes, this.length);
+    int hash = 1;
+    for (int i = offset; i < offset + length; i++)
+      hash = (31 * hash) + (int)bytes[i];
+    return hash;
   }
 
   /**
    * Define the sort order of the BytesWritable.
-   * @param right_obj The other bytes writable
+   * @param that The other bytes writable
    * @return Positive if left is bigger than right, 0 if they are equal, and
    *         negative if left is smaller than right.
    */
-  public int compareTo(ImmutableBytesWritable right_obj) {
-    return compareTo(right_obj.get());
+  public int compareTo(ImmutableBytesWritable that) {
+    return WritableComparator.compareBytes(
+      this.bytes, this.offset, this.length,
+      that.bytes, that.offset, that.length);
   }
 
   /**
@@ -178,8 +183,9 @@ implements WritableComparable<ImmutableB
    *         negative if left is smaller than right.
    */
   public int compareTo(final byte [] that) {
-    return WritableComparator.compareBytes(this.bytes, 0, this.length, that,
-        0, that.length);
+    return WritableComparator.compareBytes(
+      this.bytes, this.offset, this.length,
+      that, 0, that.length);
   }
 
   /**
@@ -202,9 +208,9 @@ implements WritableComparable<ImmutableB
   @Override
   public String toString() {
     StringBuilder sb = new StringBuilder(3*this.bytes.length);
-    for (int idx = 0; idx < this.bytes.length; idx++) {
+    for (int idx = offset; idx < offset + length; idx++) {
       // if not the first, put a blank separator in
-      if (idx != 0) {
+      if (idx != offset) {
         sb.append(' ');
       }
       String num = Integer.toHexString(bytes[idx]);
@@ -253,4 +259,11 @@ implements WritableComparable<ImmutableB
     }
     return results;
   }
+
+  /**
+   * Returns a copy of the bytes referred to by this writable
+   */
+  public byte[] copyBytes() {
+    return Arrays.copyOfRange(bytes, offset, offset+length);
+  }
 }