You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/09/27 04:41:20 UTC
svn commit: r1176171 [1/9] - in /hbase/branches/0.89: ./ bin/ src/
src/assembly/ src/docs/src/documentation/content/xdocs/
src/main/java/org/apache/hadoop/hbase/
src/main/java/org/apache/hadoop/hbase/client/
src/main/java/org/apache/hadoop/hbase/filter...
Author: nspiegelberg
Date: Tue Sep 27 02:41:16 2011
New Revision: 1176171
URL: http://svn.apache.org/viewvc?rev=1176171&view=rev
Log:
HBASE: Update to trunk
Summary:
Merged with HBase (0.21) public trunk. revisions 946777 => 952833
Test Plan:
1. mvn test
2. ./bin/start-hbase.sh && ./bin/hbase shell
DiffCamp Revision: 121725
Reviewed By: kannan
CC: isaac, cpiro, nspiegelberg, kannan
Tasks:
#219779: Merge to HBase Trunk
Revert Plan:
OK
Added:
hbase/branches/0.89/bin/check_meta.rb
hbase/branches/0.89/src/assembly/bin.xml.orig
- copied, changed from r1176170, hbase/branches/0.89/src/assembly/bin.xml
hbase/branches/0.89/src/docs/src/documentation/content/xdocs/bulk-loads.xml
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/InputSampler.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TotalOrderPartitioner.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/DefaultEnvironmentEdge.java
- copied, changed from r1176170, hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdge.java
- copied, changed from r1176170, hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManager.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/IncrementingEnvironmentEdge.java
- copied, changed from r1176170, hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/MD5Hash.java
hbase/branches/0.89/src/main/resources/hbase-webapps/
hbase/branches/0.89/src/main/resources/hbase-webapps/master/
hbase/branches/0.89/src/main/resources/hbase-webapps/master/index.html
hbase/branches/0.89/src/main/resources/hbase-webapps/master/master.jsp
hbase/branches/0.89/src/main/resources/hbase-webapps/master/table.jsp
hbase/branches/0.89/src/main/resources/hbase-webapps/master/zk.jsp
hbase/branches/0.89/src/main/resources/hbase-webapps/regionserver/
hbase/branches/0.89/src/main/resources/hbase-webapps/regionserver/index.html
hbase/branches/0.89/src/main/resources/hbase-webapps/regionserver/regionserver.jsp
hbase/branches/0.89/src/main/resources/hbase-webapps/static/
hbase/branches/0.89/src/main/resources/hbase-webapps/static/hbase.css
hbase/branches/0.89/src/main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd
hbase/branches/0.89/src/main/resources/webapps/master/WEB-INF/web.xml
hbase/branches/0.89/src/main/resources/webapps/regionserver/WEB-INF/web.xml
hbase/branches/0.89/src/site/
hbase/branches/0.89/src/site/fml/
hbase/branches/0.89/src/site/fml/faq.fml
hbase/branches/0.89/src/site/resources/
hbase/branches/0.89/src/site/resources/css/
hbase/branches/0.89/src/site/resources/css/site.css
hbase/branches/0.89/src/site/site.xml
hbase/branches/0.89/src/site/xdoc/
hbase/branches/0.89/src/site/xdoc/acid-semantics.xml
hbase/branches/0.89/src/site/xdoc/bulk-loads.xml
hbase/branches/0.89/src/site/xdoc/cygwin.xml
hbase/branches/0.89/src/site/xdoc/index.xml
hbase/branches/0.89/src/site/xdoc/metrics.xml
hbase/branches/0.89/src/site/xdoc/old_news.xml
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/MultithreadedTestUtil.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/mapreduce/NMapInputFormat.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/master/TestMasterWrongRS.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java
- copied, changed from r1176170, hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
- copied, changed from r1176170, hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/util/TestDefaultEnvironmentEdge.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/util/TestEnvironmentEdgeManager.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/util/TestIncrementingEnvironmentEdge.java
- copied, changed from r1176170, hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
Modified:
hbase/branches/0.89/CHANGES.txt
hbase/branches/0.89/bin/add_table.rb
hbase/branches/0.89/bin/hirb.rb
hbase/branches/0.89/pom.xml
hbase/branches/0.89/src/assembly/bin.xml
hbase/branches/0.89/src/docs/src/documentation/content/xdocs/site.xml
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HConstants.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HMsg.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HServerInfo.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTable.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/Driver.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/SimpleTotalOrderPartitioner.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/BaseScanner.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/MinorCompactingStoreScanner.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ReadWriteConsistencyControl.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/InfoServer.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
hbase/branches/0.89/src/main/ruby/hbase/admin.rb
hbase/branches/0.89/src/main/ruby/hbase/hbase.rb
hbase/branches/0.89/src/main/ruby/shell/commands/list.rb
hbase/branches/0.89/src/main/ruby/shell/formatter.rb
hbase/branches/0.89/src/saveVersion.sh
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/TestEmptyMetaInfo.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/TestImmutableBytesWritable.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/utils/KillProcessesAndVerify.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueScanFixture.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreReconstruction.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/rest/HBaseRESTClusterTestBase.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/rest/TestVersionResource.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/util/TestByteBloomFilter.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
hbase/branches/0.89/src/test/ruby/hbase/hbase_test.rb
hbase/branches/0.89/src/test/ruby/shell/shell_test.rb
hbase/branches/0.89/src/test/ruby/test_helper.rb
Modified: hbase/branches/0.89/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/CHANGES.txt?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/CHANGES.txt (original)
+++ hbase/branches/0.89/CHANGES.txt Tue Sep 27 02:41:16 2011
@@ -23,6 +23,7 @@ Release 0.21.0 - Unreleased
HBASE-2541 Remove transactional contrib (Clint Morgan via Stack)
HBASE-2542 Fold stargate contrib into core
HBASE-2565 Remove contrib module from hbase
+ HBASE-2397 Bytes.toStringBinary escapes printable chars
BUG FIXES
HBASE-1791 Timeout in IndexRecordWriter (Bradford Stephens via Andrew
@@ -348,6 +349,35 @@ Release 0.21.0 - Unreleased
HBASE-2582 TestTableSchemaModel not passing after commit of blooms
HBASE-2583 Make webapps work in distributed mode again and make webapps
deploy at / instead of at /webapps/master/master.jsp
+ HBASE-2590 Failed parse of branch element in saveVersion.sh
+ HBASE-2591 HBASE-2587 hardcoded the port that dfscluster runs on
+ HBASE-2519 StoreFileScanner.seek swallows IOEs (Todd Lipcon via Stack)
+ HBASE-2516 Ugly IOE when region is being closed; rather, should NSRE
+ (Daniel Ploeg via Stack)
+ HBASE-2589 TestHRegion.testWritesWhileScanning flaky on trunk
+ (Todd Lipcon via Stack)
+ HBASE-2590 Failed parse of branch element in saveVersion.sh
+ (Benoît Sigoure via Stack)
+ HBASE-2586 Move hbase webapps to a hbase-webapps dir (Todd Lipcon via
+ Andrew Purtell)
+ HBASE-2610 ValueFilter copy pasted javadoc from QualifierFilter
+ HBASE-2619 HBase shell 'alter' command cannot set table properties to False
+ (Christo Wilson via Stack)
+ HBASE-2621 Fix bad link to HFile documentation in javadoc
+ (Jeff Hammerbacher via Todd Lipcon)
+ HBASE-2371 Fix 'list' command in shell (Alexey Kovyrin via Todd Lipcon)
+ HBASE-2620 REST tests don't use ephemeral ports
+ HBASE-2635 ImmutableBytesWritable ignores offset in several cases
+ HBASE-2654 Add additional maven repository temporarily to fetch Guava
+ HBASE-2560 Fix IllegalArgumentException when manually splitting table
+ from web UI
+ HBASE-2657 TestTableResource is broken in trunk
+ HBASE-2662 TestScannerResource.testScannerResource broke in trunk
+ HBASE-2667 TestHLog.testSplit failing in trunk (Cosmin and Stack)
+ HBASE-2614 killing server in TestMasterTransitions causes NPEs and test deadlock
+ HBASE-2615 M/R on bulk imported tables
+ HBASE-2676 TestInfoServers should use ephemeral ports
+ HBASE-2616 TestHRegion.testWritesWhileGetting flaky on trunk
IMPROVEMENTS
HBASE-1760 Cleanup TODOs in HTable
@@ -623,6 +653,29 @@ Release 0.21.0 - Unreleased
pulled from http://people.apache.org/~rawson/repo/....
HBASE-2534 Recursive deletes and misc improvements to ZKW
HBASE-2577 Remove 'core' maven module; move core up a level
+ HBASE-2587 Coral where tests write data when running and make sure clean
+ target removes all written
+ HBASE-2580 Make the hlog file names unique
+ HBASE-2594 Narrow pattern used finding unit tests to run -- make it same
+ was we had in 0.20
+ HBASE-2538 Work on repository order in pom (adding fbmirror to top,
+ ibiblio on bottom)
+ HBASE-2613 Remove the code around MSG_CALL_SERVER_STARTUP
+ HBASE-2599 BaseScanner says "Current assignment of X is not valid" over
+ and over for same region
+ HBASE-2630 HFile should use toStringBinary in various places
+ HBASE-2632 Shell should autodetect terminal width
+ HBASE-2636 Upgrade Jetty to 6.1.24
+ HBASE-2437 Refactor HLog splitLog (Cosmin Lehene via Stack)
+ HBASE-2638 Speed up REST tests
+ HBASE-2653 Remove unused DynamicBloomFilter (especially as its tests are
+ failing hudson on occasion)
+ HBASE-2651 Allow alternate column separators to be specified for ImportTsv
+ HBASE-2661 Add test case for row atomicity guarantee
+ HBASE-2578 Add ability for tests to override server-side timestamp
+ setting (currentTimeMillis) (Daniel Ploeg via Ryan Rawson)
+ HBASE-2558 Our javadoc overview -- "Getting Started", requirements, etc. --
+ is not carried across by mvn javadoc:javadoc target
NEW FEATURES
HBASE-1961 HBase EC2 scripts
@@ -659,6 +712,9 @@ Release 0.21.0 - Unreleased
HBASE-2527 Add the ability to easily extend some HLog actions
HBASE-2559 Set hbase.hregion.majorcompaction to 0 to disable
HBASE-1200 Add bloomfilters (Nicolas Spiegelberg via Stack)
+ HBASE-2588 Add easier way to ship HBase dependencies to MR cluster within Job
+ HBASE-1923 Bulk incremental load into an existing table
+ HBASE-2579 Add atomic checkAndDelete support (Michael Dalton via Stack)
OPTIMIZATIONS
HBASE-410 [testing] Speed up the test suite
@@ -1000,6 +1056,11 @@ Release 0.20.0 - Tue Sep 8 12:53:05 PDT
(Mathias Herberts via Stack)
HBASE-1804 Puts are permitted (and stored) when including an appended colon
HBASE-1715 Compaction failure in ScanWildcardColumnTracker.checkColumn
+ HBASE-2352 Small values for hbase.client.retries.number and
+ ipc.client.connect.max.retries breaks long ops in hbase shell
+ (Alexey Kovyrin via Stack)
+ HBASE-2531 32-bit encoding of regionnames waaaaaaayyyyy too susceptible to
+ hash clashes (Kannan Muthukkaruppan via Stack)
IMPROVEMENTS
HBASE-1089 Add count of regions on filesystem to master UI; add percentage
@@ -1234,6 +1295,8 @@ Release 0.20.0 - Tue Sep 8 12:53:05 PDT
HBASE-1743 [debug tool] Add regionsInTransition list to ClusterStatus
detailed output
HBASE-1772 Up the default ZK session timeout from 30seconds to 60seconds
+ HBASE-2625 Make testDynamicBloom()'s "randomness" deterministic
+ (Nicolas Spiegelberg via Stack)
OPTIMIZATIONS
HBASE-1412 Change values for delete column and column family in KeyValue
Modified: hbase/branches/0.89/bin/add_table.rb
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/bin/add_table.rb?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/bin/add_table.rb (original)
+++ hbase/branches/0.89/bin/add_table.rb Tue Sep 27 02:41:16 2011
@@ -19,14 +19,15 @@
#
# Script adds a table back to a running hbase.
# Currently only works on if table data is in place.
-#
-# To see usage for this script, run:
+#
+# To see usage for this script, run:
#
# ${HBASE_HOME}/bin/hbase org.jruby.Main addtable.rb
#
include Java
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.hbase.HConstants
+import org.apache.hadoop.hbase.regionserver.HRegion
import org.apache.hadoop.hbase.HRegionInfo
import org.apache.hadoop.hbase.client.HTable
import org.apache.hadoop.hbase.client.Delete
@@ -84,7 +85,7 @@ elsif
end
HTableDescriptor.isLegalTableName(tableName.to_java_bytes)
-# Figure locations under hbase.rootdir
+# Figure locations under hbase.rootdir
# Move directories into place; be careful not to overwrite.
rootdir = FSUtils.getRootDir(c)
tableDir = fs.makeQualified(Path.new(rootdir, tableName))
@@ -129,15 +130,15 @@ statuses = fs.listStatus(srcdir)
for status in statuses
next unless status.isDir()
next if status.getPath().getName() == "compaction.dir"
- regioninfofile = Path.new(status.getPath(), ".regioninfo")
+ regioninfofile = Path.new(status.getPath(), HRegion::REGIONINFO_FILE)
unless fs.exists(regioninfofile)
LOG.warn("Missing .regioninfo: " + regioninfofile.toString())
next
end
- is = fs.open(regioninfofile)
+ is = fs.open(regioninfofile)
hri = HRegionInfo.new()
hri.readFields(is)
- is.close()
+ is.close()
# TODO: Need to redo table descriptor with passed table name and then recalculate the region encoded names.
p = Put.new(hri.getRegionName())
p.add(HConstants::CATALOG_FAMILY, HConstants::REGIONINFO_QUALIFIER, Writables.getBytes(hri))
Added: hbase/branches/0.89/bin/check_meta.rb
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/bin/check_meta.rb?rev=1176171&view=auto
==============================================================================
--- hbase/branches/0.89/bin/check_meta.rb (added)
+++ hbase/branches/0.89/bin/check_meta.rb Tue Sep 27 02:41:16 2011
@@ -0,0 +1,160 @@
+# Script looks at hbase .META. table verifying its content is coherent.
+#
+# To see usage for this script, run:
+#
+# ${HBASE_HOME}/bin/hbase org.jruby.Main check_meta.rb --help
+#
+
+# Copyright 2010 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+include Java
+import org.apache.commons.logging.LogFactory
+import org.apache.hadoop.hbase.util.VersionInfo
+import org.apache.hadoop.hbase.HBaseConfiguration
+import org.apache.hadoop.fs.FileSystem
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.hbase.HConstants
+import org.apache.hadoop.hbase.util.FSUtils
+import org.apache.hadoop.hbase.client.HTable
+import org.apache.hadoop.hbase.client.Scan
+import org.apache.hadoop.hbase.util.Writables
+import org.apache.hadoop.hbase.HRegionInfo
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.hadoop.hbase.HTableDescriptor
+import org.apache.hadoop.hbase.client.Put
+
+# Name of this script
+NAME = 'check_meta'
+
+# Print usage for this script
+def usage
+ puts 'Usage: %s.rb [--fix]' % NAME
+ puts ' fix Try to fixup meta issues'
+ puts 'Script checks consistency of the .META. table. It reports if .META. has missing entries.'
+ puts 'If you pass "--fix", it will try looking in the filesystem for the dropped region and if it'
+ puts 'finds a likely candidate, it will try pluggin the .META. hole.'
+ exit!
+end
+
+def isFixup
+ # Are we to do fixup during this run
+ usage if ARGV.size > 1
+ fixup = nil
+ if ARGV.size == 1
+ usage unless ARGV[0].downcase.match('--fix.*')
+ fixup = 1
+ end
+ return fixup
+end
+
+def getConfiguration
+ hbase_twenty = VersionInfo.getVersion().match('0\.20\..*')
+ # Get configuration to use.
+ if hbase_twenty
+ c = HBaseConfiguration.new()
+ else
+ c = HBaseConfiguration.create()
+ end
+ # Set hadoop filesystem configuration using the hbase.rootdir.
+ # Otherwise, we'll always use localhost though the hbase.rootdir
+ # might be pointing at hdfs location. Do old and new key for fs.
+ c.set("fs.default.name", c.get(HConstants::HBASE_DIR))
+ c.set("fs.defaultFS", c.get(HConstants::HBASE_DIR))
+ return c
+end
+
+def fixup(leftEdge, rightEdge, metatable, fs, rootdir)
+ plugged = nil
+ # Try and fix the passed holes in meta.
+ tabledir = HTableDescriptor::getTableDir(rootdir, leftEdge.getTableDesc().getName())
+ statuses = fs.listStatus(tabledir)
+ for status in statuses
+ next unless status.isDir()
+ next if status.getPath().getName() == "compaction.dir"
+ regioninfofile = Path.new(status.getPath(), ".regioninfo")
+ unless fs.exists(regioninfofile)
+ LOG.warn("Missing .regioninfo: " + regioninfofile.toString())
+ next
+ end
+ is = fs.open(regioninfofile)
+ hri = HRegionInfo.new()
+ hri.readFields(is)
+ is.close()
+ next unless Bytes.equals(leftEdge.getEndKey(), hri.getStartKey())
+ # TODO: Check against right edge to make sure this addition does not overflow right edge.
+ # TODO: Check that the schema matches both left and right edges schemas.
+ p = Put.new(hri.getRegionName())
+ p.add(HConstants::CATALOG_FAMILY, HConstants::REGIONINFO_QUALIFIER, Writables.getBytes(hri))
+ metatable.put(p)
+ LOG.info("Plugged hole in .META. at: " + hri.toString())
+ plugged = true
+ end
+ return plugged
+end
+
+fixup = isFixup()
+
+# Get configuration
+conf = getConfiguration()
+
+# Filesystem
+fs = FileSystem.get(conf)
+
+# Rootdir
+rootdir = FSUtils.getRootDir(conf)
+
+# Get a logger and a metautils instance.
+LOG = LogFactory.getLog(NAME)
+
+# Scan the .META. looking for holes
+metatable = HTable.new(conf, HConstants::META_TABLE_NAME)
+scan = Scan.new()
+scanner = metatable.getScanner(scan)
+oldHRI = nil
+bad = nil
+while (result = scanner.next())
+ rowid = Bytes.toString(result.getRow())
+ rowidStr = java.lang.String.new(rowid)
+ bytes = result.getValue(HConstants::CATALOG_FAMILY, HConstants::REGIONINFO_QUALIFIER)
+ hri = Writables.getHRegionInfo(bytes)
+ if oldHRI
+ if oldHRI.isOffline() && Bytes.equals(oldHRI.getStartKey(), hri.getStartKey())
+ # Presume offlined parent
+ elsif Bytes.equals(oldHRI.getEndKey(), hri.getStartKey())
+ # Start key of next matches end key of previous
+ else
+ LOG.warn("hole after " + oldHRI.toString())
+ if fixup
+ bad = 1 unless fixup(oldHRI, hri, metatable, fs, rootdir)
+ else
+ bad = 1
+ end
+ end
+ end
+ oldHRI = hri
+end
+scanner.close()
+if bad
+ LOG.info(".META. has holes")
+else
+ LOG.info(".META. is healthy")
+end
+
+# Return 0 if meta is good, else non-zero.
+exit bad
Modified: hbase/branches/0.89/bin/hirb.rb
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/bin/hirb.rb?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/bin/hirb.rb (original)
+++ hbase/branches/0.89/bin/hirb.rb Tue Sep 27 02:41:16 2011
@@ -53,12 +53,10 @@ end
cmdline_help = <<HERE # HERE document output as shell usage
HBase Shell command-line options:
format Formatter for outputting results: console | html. Default: console
- format-width Width of table outputs. Default: 110 characters.
-d | --debug Set DEBUG log levels.
HERE
found = []
format = 'console'
-format_width = 110
script2run = nil
log_level = org.apache.log4j.Level::ERROR
for arg in ARGV
@@ -72,9 +70,6 @@ for arg in ARGV
raise ArgumentError.new("Unsupported format " + arg)
end
found.push(arg)
- elsif arg =~ /^--format-width=(.+)/i
- format_width = $1.to_i
- found.push(arg)
elsif arg == '-h' || arg == '--help'
puts cmdline_help
exit
@@ -110,7 +105,7 @@ require 'shell/formatter'
# Presume console format.
# Formatter takes an :output_stream parameter, if you don't want STDOUT.
-@formatter = Shell::Formatter::Console.new(:format_width => format_width)
+@formatter = Shell::Formatter::Console.new
# Setup the HBase module. Create a configuration.
@hbase = Hbase::Hbase.new
Modified: hbase/branches/0.89/pom.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/pom.xml?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/pom.xml (original)
+++ hbase/branches/0.89/pom.xml Tue Sep 27 02:41:16 2011
@@ -7,10 +7,13 @@
<packaging>jar</packaging>
<version>0.21.0-SNAPSHOT</version>
<name>HBase</name>
- <description>HBase is the Hadoop database. Use it when you need random, realtime read/write access to your Big Data.
+ <!-- START SNIPPET: description -->
+ <description>
+ HBase is the &lt;a href="http://hadoop.apache.org"&rt;Hadoop</a&rt; database. Use it when you need random, realtime read/write access to your Big Data.
This project's goal is the hosting of very large tables -- billions of rows X millions of columns -- atop clusters
of commodity hardware.
</description>
+ <!-- END SNIPPET: description -->
<url>http://hbase.apache.org</url>
<licenses>
@@ -22,6 +25,11 @@
</license>
</licenses>
+ <organization>
+ <name>Apache Software Foundation</name>
+ <url>http://www.apache.org</url>
+ </organization>
+
<scm>
<connection>scm:svn:http://svn.apache.org/repos/asf/hbase/trunk</connection>
<developerConnection>scm:svn:https://svn.apache.org/repos/asf/hbase/trunk</developerConnection>
@@ -122,11 +130,31 @@
<organization>StumbleUpon</organization>
<organizationUrl>http://www.stumbleupon.com/</organizationUrl>
</developer>
+ <developer>
+ <id>todd</id>
+ <name>Todd Lipcon</name>
+ <email>todd@apache.org</email>
+ <timezone>-8</timezone>
+ <organization>Cloudera</organization>
+ <organizationUrl>http://www.cloudera.com</organizationUrl>
+ </developer>
</developers>
<repositories>
- <!-- temporary: should replace jgray.la with our internal mirror once that's setup -->
- <repository>
+ <repository>
+ <id>mirror.facebook.net</id>
+ <name>facebook mirror</name>
+ <url>http://mirror.facebook.net/maven/repository/</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ </repository>
+
+ <!-- temporary: should replace jgray.la with our internal mirror once that's setup -->
+ <repository>
<id>jgray.la</id>
<name>jgray mirror</name>
<url>http://jgray.la/maven/repository/</url>
@@ -192,6 +220,30 @@
<enabled>true</enabled>
</releases>
</repository>
+ <repository>
+ <!-- Guava on standard maven repos has a bad checksum, hosting
+ here temporarily -->
+ <id>temp-todd-for-guava</id>
+ <name>Todd Lipcon's repository on apache.org</name>
+ <url>http://people.apache.org/~todd/repo/</url>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ </repository>
+ <repository>
+ <id>ibiblio.org</id>
+ <name>ibiblio mirror</name>
+ <url>http://mirrors.ibiblio.org/pub/mirrors/maven2/</url>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ </repository>
</repositories>
@@ -239,6 +291,15 @@
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<version>2.4</version>
+ <configuration>
+ <filesets>
+ <fileset>
+ <!--dfs tests have build dir hardcoded. Clean it as part of
+ clean target-->
+ <directory>build</directory>
+ </fileset>
+ </filesets>
+ </configuration>
</plugin>
<plugin>
<groupId>org.apache.rat</groupId>
@@ -252,7 +313,7 @@
<directory>src/main/resources</directory>
</resource>
<resource>
- <directory>src/main/webapps</directory>
+ <directory>src/main/hbase-webapps</directory>
<includes>
<include>*/.gif</include>
<include>*/.css</include>
@@ -341,14 +402,14 @@
<configuration>
<tasks>
<taskdef classname="org.apache.jasper.JspC" name="jspcompiler" classpathref="maven.compile.classpath"/>
- <jspcompiler uriroot="${basedir}/src/main/resources/webapps/master"
+ <jspcompiler uriroot="${basedir}/src/main/resources/hbase-webapps/master"
outputdir="${project.build.directory}/jspc"
package="org.apache.hadoop.hbase.generated.master"
- webxml="${basedir}/src/main/resources/webapps/master/WEB-INF/web.xml"/>
- <jspcompiler uriroot="${basedir}/src/main/resources/webapps/regionserver"
+ webxml="${basedir}/src/main/resources/hbase-webapps/master/WEB-INF/web.xml"/>
+ <jspcompiler uriroot="${basedir}/src/main/resources/hbase-webapps/regionserver"
outputdir="${project.build.directory}/jspc"
package="org.apache.hadoop.hbase.generated.regionserver"
- webxml="${basedir}/src/main/resources/webapps/regionserver/WEB-INF/web.xml"/>
+ webxml="${basedir}/src/main/resources/hbase-webapps/regionserver/WEB-INF/web.xml"/>
</tasks>
</configuration>
<goals>
@@ -412,8 +473,10 @@
<commons-cli.version>1.2</commons-cli.version>
<commons-logging.version>1.1.1</commons-logging.version>
- <jetty.version>6.1.14</jetty.version>
+ <jetty.version>6.1.24</jetty.version>
+ <jetty.jspapi.version>6.1.14</jetty.jspapi.version>
<junit.version>4.8.1</junit.version>
+ <mockito-all.version>1.8.4</mockito-all.version>
<log4j.version>1.2.15</log4j.version>
<zookeeper.version>3.3.1</zookeeper.version>
@@ -429,6 +492,7 @@
<slf4j.version>1.5.8</slf4j.version>
<stax-api>1.0.1</stax-api>
<thrift.version>0.2.0</thrift.version>
+ <guava.version>r03</guava.version>
</properties>
<dependencyManagement>
@@ -494,7 +558,7 @@
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>servlet-api-2.5</artifactId>
- <version>${jetty.version}</version>
+ <version>${jetty.jspapi.version}</version>
</dependency>
<!-- Test dependencies -->
@@ -510,6 +574,12 @@
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${mockito-all.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</dependencyManagement>
<dependencies>
@@ -574,6 +644,12 @@
</dependency>
<dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>r03</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
</dependency>
@@ -600,12 +676,12 @@
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-2.1</artifactId>
- <version>${jetty.version}</version>
+ <version>${jetty.jspapi.version}</version>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-api-2.1</artifactId>
- <version>${jetty.version}</version>
+ <version>${jetty.jspapi.version}</version>
</dependency>
<dependency>
<groupId>tomcat</groupId>
@@ -673,15 +749,24 @@
<artifactId>junit</artifactId>
</dependency>
<dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math</artifactId>
<version>${commons-math.version}</version>
<scope>test</scope>
</dependency>
- <dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-test</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ </dependency>
</dependencies>
<!--
@@ -826,10 +911,12 @@
<artifactId>maven-surefire-report-plugin</artifactId>
<version>2.5</version>
</plugin>
+ <!--
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.1</version>
</plugin>
+ -->
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
Modified: hbase/branches/0.89/src/assembly/bin.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/assembly/bin.xml?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/assembly/bin.xml (original)
+++ hbase/branches/0.89/src/assembly/bin.xml Tue Sep 27 02:41:16 2011
@@ -34,12 +34,23 @@
</includes>
</fileSet>
<fileSet>
- <directory>target/classes/webapps</directory>
- <outputDirectory>webapps</outputDirectory>
+ <directory>target/classes/hbase-webapps</directory>
+ <outputDirectory>hbase-webapps</outputDirectory>
<excludes>
<exclude>**/*.jsp</exclude>
</excludes>
</fileSet>
+ <fileSet>
+ <directory>target/site</directory>
+ <outputDirectory>docs</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/resources/</directory>
+ <outputDirectory>conf</outputDirectory>
+ <includes>
+ <include>hbase-default.xml</include>
+ </includes>
+ </fileSet>
</fileSets>
<dependencySets>
<dependencySet>
Copied: hbase/branches/0.89/src/assembly/bin.xml.orig (from r1176170, hbase/branches/0.89/src/assembly/bin.xml)
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/assembly/bin.xml.orig?p2=hbase/branches/0.89/src/assembly/bin.xml.orig&p1=hbase/branches/0.89/src/assembly/bin.xml&r1=1176170&r2=1176171&rev=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/assembly/bin.xml (original)
+++ hbase/branches/0.89/src/assembly/bin.xml.orig Tue Sep 27 02:41:16 2011
@@ -13,6 +13,7 @@
<include>${basedir}/README*</include>
<include>${basedir}/LICENSE*</include>
<include>${basedir}/NOTICE*</include>
+ <include>${basedir}/lib/**</include>
</includes>
</fileSet>
<fileSet>
@@ -34,8 +35,8 @@
</includes>
</fileSet>
<fileSet>
- <directory>target/classes/webapps</directory>
- <outputDirectory>webapps</outputDirectory>
+ <directory>target/classes/hbase-webapps</directory>
+ <outputDirectory>hbase-webapps</outputDirectory>
<excludes>
<exclude>**/*.jsp</exclude>
</excludes>
Added: hbase/branches/0.89/src/docs/src/documentation/content/xdocs/bulk-loads.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/docs/src/documentation/content/xdocs/bulk-loads.xml?rev=1176171&view=auto
==============================================================================
--- hbase/branches/0.89/src/docs/src/documentation/content/xdocs/bulk-loads.xml (added)
+++ hbase/branches/0.89/src/docs/src/documentation/content/xdocs/bulk-loads.xml Tue Sep 27 02:41:16 2011
@@ -0,0 +1,148 @@
+<?xml version="1.0"?>
+<!--
+ Copyright 2010 The Apache Software Foundation
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN"
+ "http://forrest.apache.org/dtd/document-v20.dtd">
+
+
+<document>
+
+ <header>
+ <title>
+ Bulk Loads in HBase
+ </title>
+ </header>
+ <body>
+ <section>
+ <title>Overview</title>
+ <p>
+ HBase includes several methods of loading data into tables.
+ The most straightforward method is to either use the TableOutputFormat
+ class from a MapReduce job, or use the normal client APIs; however,
+ these are not always the most efficient methods.
+ </p>
+ <p>
+ This document describes HBase's bulk load functionality. The bulk load
+ feature uses a MapReduce job to output table data in HBase's internal
+ data format, and then directly loads the data files into a running
+ cluster.
+ </p>
+ </section>
+ <section>
+ <title>Bulk Load Architecture</title>
+ <p>
+ The HBase bulk load process consists of two main steps.
+ </p>
+ <subsection>
+ <title>Preparing data via a MapReduce job</title>
+ <p>
+ The first step of a bulk load is to generate HBase data files from
+ a MapReduce job using HFileOutputFormat. This output format writes
+ out data in HBase's internal storage format so that they can be
+ later loaded very efficiently into the cluster.
+ </p>
+ <p>
+ In order to function efficiently, HFileOutputFormat must be configured
+ such that each output HFile fits within a single region. In order to
+ do this, jobs use Hadoop's TotalOrderPartitioner class to partition the
+ map output into disjoint ranges of the key space, corresponding to the
+ key ranges of the regions in the table.
+ </p>
+ <p>
+ HFileOutputFormat includes a convenience function, <code>configureIncrementalLoad()</code>,
+ which automatically sets up a TotalOrderPartitioner based on the current
+ region boundaries of a table.
+ </p>
+ </subsection>
+ <subsection>
+ <title>Completing the data load</title>
+ <p>
+ After the data has been prepared using <code>HFileOutputFormat</code>, it
+ is loaded into the cluster using a command line tool. This command line tool
+ iterates through the prepared data files, and for each one determines the
+ region the file belongs to. It then contacts the appropriate Region Server
+ which adopts the HFile, moving it into its storage directory and making
+ the data available to clients.
+ </p>
+ <p>
+ If the region boundaries have changed during the course of bulk load
+ preparation, or between the preparation and completion steps, the bulk
+ load commandline utility will automatically split the data files into
+ pieces corresponding to the new boundaries. This process is not
+ optimally efficient, so users should take care to minimize the delay between
+ preparing a bulk load and importing it into the cluster, especially
+ if other clients are simultaneously loading data through other means.
+ </p>
+ </subsection>
+ </section>
+ <section>
+ <title>Preparing a bulk load using the <code>importtsv</code> tool</title>
+ <p>
+ HBase ships with a command line tool called <code>importtsv</code>. This tool
+ is available by running <code>hadoop jar /path/to/hbase-VERSION.jar importtsv</code>.
+ Running this tool with no arguments prints brief usage information:
+ </p>
+ <code><pre>
+Usage: importtsv -Dimporttsv.columns=a,b,c <tablename> <inputdir>
+
+Imports the given input directory of TSV data into the specified table.
+
+The column names of the TSV data must be specified using the -Dimporttsv.columns
+option. This option takes the form of comma-separated column names, where each
+column name is either a simple column family, or a columnfamily:qualifier. The special
+column name HBASE_ROW_KEY is used to designate that this column should be used
+as the row key for each imported record. You must specify exactly one column
+to be the row key.
+
+In order to prepare data for a bulk data load, pass the option:
+ -Dimporttsv.bulk.output=/path/for/output
+
+Other options that may be specified with -D include:
+ -Dimporttsv.skip.bad.lines=false - fail if encountering an invalid line
+</pre></code>
+ </section>
+ <section>
+ <title>Importing the prepared data using the <code>completebulkload</code> tool</title>
+ <p>
+ After a data import has been prepared using the <code>importtsv</code> tool, the
+ <code>completebulkload</code> tool is used to import the data into the running cluster.
+ </p>
+ <p>
+ The <code>completebulkload</code> tool simply takes the same output path where
+ <code>importtsv</code> put its results, and the table name. For example:
+ </p>
+ <code>$ hadoop jar hbase-VERSION.jar completebulkload /user/todd/myoutput mytable</code>
+ <p>
+ This tool will run quickly, after which point the new data will be visible in
+ the cluster.
+ </p>
+ </section>
+ <section>
+ <title>Advanced Usage</title>
+ <p>
+ Although the <code>importtsv</code> tool is useful in many cases, advanced users may
+ want to generate data programatically, or import data from other formats. To get
+ started doing so, dig into <code>ImportTsv.java</code> and check the JavaDoc for
+ HFileOutputFormat.
+ </p>
+ <p>
+ The import step of the bulk load can also be done programatically. See the
+ <code>LoadIncrementalHFiles</code> class for more information.
+ </p>
+ </section>
+ </body>
+</document>
\ No newline at end of file
Modified: hbase/branches/0.89/src/docs/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/docs/src/documentation/content/xdocs/site.xml?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/docs/src/documentation/content/xdocs/site.xml (original)
+++ hbase/branches/0.89/src/docs/src/documentation/content/xdocs/site.xml Tue Sep 27 02:41:16 2011
@@ -35,6 +35,7 @@ See http://forrest.apache.org/docs/linki
<overview label="Overview" href="index.html" />
<started label="Getting Started" href="ext:api/started" />
<api label="API Docs" href="ext:api/index" />
+ <api label="Bulk Loads" href="bulk-load.html" />
<api label="HBase Metrics" href="metrics.html" />
<api label="HBase Semantics" href="acid-semantics.html" />
<api label="HBase Default Configuration" href="hbase-conf.html" />
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HConstants.java Tue Sep 27 02:41:16 2011
@@ -29,12 +29,7 @@ public interface HConstants {
/** long constant for zero */
static final Long ZERO_L = Long.valueOf(0L);
-
- //TODO: NINES is only used in HBaseAdmin and HConnectionManager. Move to client
- // package and change visibility to default
static final String NINES = "99999999999999";
- //TODO: ZEROS is only used in HConnectionManager and MetaScanner. Move to
- // client package and change visibility to default
static final String ZEROES = "00000000000000";
// For migration
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HMsg.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HMsg.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HMsg.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HMsg.java Tue Sep 27 02:41:16 2011
@@ -38,8 +38,6 @@ public class HMsg implements Writable {
new HMsg(Type.MSG_REGIONSERVER_QUIESCE);
public static final HMsg REGIONSERVER_STOP =
new HMsg(Type.MSG_REGIONSERVER_STOP);
- public static final HMsg CALL_SERVER_STARTUP =
- new HMsg(Type.MSG_CALL_SERVER_STARTUP);
public static final HMsg [] EMPTY_HMSG_ARRAY = new HMsg[0];
/**
@@ -62,9 +60,6 @@ public class HMsg implements Writable {
/** Compact the specified region */
MSG_REGION_COMPACT,
- /** Region server is unknown to master. Restart */
- MSG_CALL_SERVER_STARTUP,
-
/** Master tells region server to stop */
MSG_REGIONSERVER_STOP,
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java Tue Sep 27 02:41:16 2011
@@ -24,9 +24,12 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.Arrays;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.KeyValue.KVComparator;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.JenkinsHash;
+import org.apache.hadoop.hbase.util.MD5Hash;
import org.apache.hadoop.io.VersionedWritable;
import org.apache.hadoop.io.WritableComparable;
@@ -37,13 +40,74 @@ import org.apache.hadoop.io.WritableComp
*/
public class HRegionInfo extends VersionedWritable implements WritableComparable<HRegionInfo>{
private static final byte VERSION = 0;
+ private static final Log LOG = LogFactory.getLog(HRegionInfo.class);
+
+ /**
+ * The new format for a region name contains its encodedName at the end.
+ * The encoded name also serves as the directory name for the region
+ * in the filesystem.
+ *
+ * New region name format:
+ * <tablename>,,<startkey>,<regionIdTimestamp>.<encodedName>.
+ * where,
+ * <encodedName> is a hex version of the MD5 hash of
+ * <tablename>,<startkey>,<regionIdTimestamp>
+ *
+ * The old region name format:
+ * <tablename>,<startkey>,<regionIdTimestamp>
+ * For region names in the old format, the encoded name is a 32-bit
+ * JenkinsHash integer value (in its decimal notation, string form).
+ *<p>
+ * **NOTE**
+ *
+ * ROOT, the first META region, and regions created by an older
+ * version of HBase (0.20 or prior) will continue to use the
+ * old region name format.
+ */
+
+ /** Separator used to demarcate the encodedName in a region name
+ * in the new format. See description on new format above.
+ */
+ private static final int ENC_SEPARATOR = '.';
+ public static final int MD5_HEX_LENGTH = 32;
+
+ /**
+ * Does region name contain its encoded name?
+ * @param regionName region name
+ * @return boolean indicating if this a new format region
+ * name which contains its encoded name.
+ */
+ private static boolean hasEncodedName(final byte[] regionName) {
+ // check if region name ends in ENC_SEPARATOR
+ if ((regionName.length >= 1)
+ && (regionName[regionName.length - 1] == ENC_SEPARATOR)) {
+ // region name is new format. it contains the encoded name.
+ return true;
+ }
+ return false;
+ }
/**
* @param regionName
* @return the encodedName
*/
- public static int encodeRegionName(final byte [] regionName) {
- return Math.abs(JenkinsHash.getInstance().hash(regionName, regionName.length, 0));
+ public static String encodeRegionName(final byte [] regionName) {
+ String encodedName;
+ if (hasEncodedName(regionName)) {
+ // region is in new format:
+ // <tableName>,<startKey>,<regionIdTimeStamp>/encodedName/
+ encodedName = Bytes.toString(regionName,
+ regionName.length - MD5_HEX_LENGTH - 1,
+ MD5_HEX_LENGTH);
+ } else {
+ // old format region name. ROOT and first META region also
+ // use this format.EncodedName is the JenkinsHash value.
+ int hashVal = Math.abs(JenkinsHash.getInstance().hash(regionName,
+ regionName.length,
+ 0));
+ encodedName = String.valueOf(hashVal);
+ }
+ return encodedName;
}
/** delimiter used between portions of a region name */
@@ -67,8 +131,8 @@ public class HRegionInfo extends Version
protected HTableDescriptor tableDesc = null;
private int hashCode = -1;
//TODO: Move NO_HASH to HStoreFile which is really the only place it is used.
- public static final int NO_HASH = -1;
- private volatile int encodedName = NO_HASH;
+ public static final String NO_HASH = null;
+ private volatile String encodedName = NO_HASH;
private void setHashCode() {
int result = Arrays.hashCode(this.regionName);
@@ -88,7 +152,10 @@ public class HRegionInfo extends Version
super();
this.regionId = regionId;
this.tableDesc = tableDesc;
- this.regionName = createRegionName(tableDesc.getName(), null, regionId);
+
+ // Note: Root & First Meta regions names are still in old format
+ this.regionName = createRegionName(tableDesc.getName(), null,
+ regionId, false);
this.regionNameStr = Bytes.toStringBinary(this.regionName);
setHashCode();
}
@@ -149,7 +216,7 @@ public class HRegionInfo extends Version
}
this.offLine = false;
this.regionId = regionid;
- this.regionName = createRegionName(tableDesc.getName(), startKey, regionId);
+ this.regionName = createRegionName(tableDesc.getName(), startKey, regionId, true);
this.regionNameStr = Bytes.toStringBinary(this.regionName);
this.split = split;
this.endKey = endKey == null? HConstants.EMPTY_END_ROW: endKey.clone();
@@ -179,8 +246,8 @@ public class HRegionInfo extends Version
}
private static byte [] createRegionName(final byte [] tableName,
- final byte [] startKey, final long regionid) {
- return createRegionName(tableName, startKey, Long.toString(regionid));
+ final byte [] startKey, final long regionid, boolean newFormat) {
+ return createRegionName(tableName, startKey, Long.toString(regionid), newFormat);
}
/**
@@ -188,23 +255,29 @@ public class HRegionInfo extends Version
* @param tableName
* @param startKey Can be null
* @param id Region id.
+ * @param newFormat should we create the region name in the new format
+ * (such that it contains its encoded name?).
* @return Region name made of passed tableName, startKey and id
*/
public static byte [] createRegionName(final byte [] tableName,
- final byte [] startKey, final String id) {
- return createRegionName(tableName, startKey, Bytes.toBytes(id));
+ final byte [] startKey, final String id, boolean newFormat) {
+ return createRegionName(tableName, startKey, Bytes.toBytes(id), newFormat);
}
/**
* Make a region name of passed parameters.
* @param tableName
* @param startKey Can be null
* @param id Region id
+ * @param newFormat should we create the region name in the new format
+ * (such that it contains its encoded name?).
* @return Region name made of passed tableName, startKey and id
*/
public static byte [] createRegionName(final byte [] tableName,
- final byte [] startKey, final byte [] id) {
+ final byte [] startKey, final byte [] id, boolean newFormat) {
byte [] b = new byte [tableName.length + 2 + id.length +
- (startKey == null? 0: startKey.length)];
+ (startKey == null? 0: startKey.length) +
+ (newFormat ? (MD5_HEX_LENGTH + 2) : 0)];
+
int offset = tableName.length;
System.arraycopy(tableName, 0, b, 0, offset);
b[offset++] = DELIMITER;
@@ -214,6 +287,31 @@ public class HRegionInfo extends Version
}
b[offset++] = DELIMITER;
System.arraycopy(id, 0, b, offset, id.length);
+ offset += id.length;
+
+ if (newFormat) {
+ //
+ // Encoded name should be built into the region name.
+ //
+ // Use the region name thus far (namely, <tablename>,<startKey>,<id>)
+ // to compute a MD5 hash to be used as the encoded name, and append
+ // it to the byte buffer.
+ //
+ String md5Hash = MD5Hash.getMD5AsHex(b, 0, offset);
+ byte [] md5HashBytes = Bytes.toBytes(md5Hash);
+
+ if (md5HashBytes.length != MD5_HEX_LENGTH) {
+ LOG.error("MD5-hash length mismatch: Expected=" + MD5_HEX_LENGTH +
+ "; Got=" + md5HashBytes.length);
+ }
+
+ // now append the bytes '.<encodedName>.' to the end
+ b[offset++] = ENC_SEPARATOR;
+ System.arraycopy(md5HashBytes, 0, b, offset, MD5_HEX_LENGTH);
+ offset += MD5_HEX_LENGTH;
+ b[offset++] = ENC_SEPARATOR;
+ }
+
return b;
}
@@ -259,11 +357,6 @@ public class HRegionInfo extends Version
return elements;
}
- /** @return the endKey */
- public byte [] getEndKey(){
- return endKey;
- }
-
/** @return the regionId */
public long getRegionId(){
return regionId;
@@ -281,11 +374,19 @@ public class HRegionInfo extends Version
* @return Region name as a String for use in logging, etc.
*/
public String getRegionNameAsString() {
- return this.regionNameStr;
+ if (hasEncodedName(this.regionName)) {
+ // new format region names already have their encoded name.
+ return this.regionNameStr;
+ }
+
+ // old format. regionNameStr doesn't have the region name.
+ //
+ //
+ return this.regionNameStr + "." + this.getEncodedName();
}
/** @return the encoded region name */
- public synchronized int getEncodedName() {
+ public synchronized String getEncodedName() {
if (this.encodedName == NO_HASH) {
this.encodedName = encodeRegionName(this.regionName);
}
@@ -297,6 +398,32 @@ public class HRegionInfo extends Version
return startKey;
}
+ /** @return the endKey */
+ public byte [] getEndKey(){
+ return endKey;
+ }
+
+ /**
+ * Returns true if the given inclusive range of rows is fully contained
+ * by this region. For example, if the region is foo,a,g and this is
+ * passed ["b","c"] or ["a","c"] it will return true, but if this is passed
+ * ["b","z"] it will return false.
+ * @throws IllegalArgumentException if the range passed is invalid (ie end < start)
+ */
+ public boolean containsRange(byte[] rangeStartKey, byte[] rangeEndKey) {
+ if (Bytes.compareTo(rangeStartKey, rangeEndKey) > 0) {
+ throw new IllegalArgumentException(
+ "Invalid range: " + Bytes.toStringBinary(rangeStartKey) +
+ " > " + Bytes.toStringBinary(rangeEndKey));
+ }
+
+ boolean firstKeyInRange = Bytes.compareTo(rangeStartKey, startKey) >= 0;
+ boolean lastKeyInRange =
+ Bytes.compareTo(rangeEndKey, endKey) < 0 ||
+ Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY);
+ return firstKeyInRange && lastKeyInRange;
+ }
+
/** @return the tableDesc */
public HTableDescriptor getTableDesc(){
return tableDesc;
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HServerInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HServerInfo.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HServerInfo.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/HServerInfo.java Tue Sep 27 02:41:16 2011
@@ -1,5 +1,5 @@
/**
- * Copyright 2007 The Apache Software Foundation
+ * Copyright 2010 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -22,26 +22,33 @@ package org.apache.hadoop.hbase;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
+import java.net.InetSocketAddress;
+import java.util.Set;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
/**
- * HServerInfo is meta info about an {@link HRegionServer}.
- * Holds hostname, ports, regionserver startcode, and load. Each server has
+ * HServerInfo is meta info about an {@link HRegionServer}. It is the token
+ * by which a master distingushes a particular regionserver from the rest.
+ * It holds hostname, ports, regionserver startcode, and load. Each server has
* a <code>servername</code> where servername is made up of a concatenation of
- * hostname, port, and regionserver startcode.
+ * hostname, port, and regionserver startcode. This servername is used in
+ * various places identifying this regionserver. Its even used as part of
+ * a pathname in the filesystem. As part of the initialization,
+ * master will pass the regionserver the address that it knows this regionserver
+ * by. In subsequent communications, the regionserver will pass a HServerInfo
+ * with the master-supplied address.
*/
public class HServerInfo implements WritableComparable<HServerInfo> {
- /**
- * This character is used as separator making up the <code>servername</code>.
- * Servername is made of host, port, and startcode formatted as
- * <code><hostname> '{@link #SERVERNAME_SEPARATOR}' <port> '{@ink #SEPARATOR"}' <startcode></code>
- * where {@link SEPARATOR is usually a ','.
+ /*
+ * This character is used as separator between server hostname and port and
+ * its startcode. Servername is formatted as
+ * <code><hostname> '{@ink #SERVERNAME_SEPARATOR"}' <port> '{@ink #SERVERNAME_SEPARATOR"}' <startcode></code>.
*/
- public static final String SERVERNAME_SEPARATOR = ",";
+ private static final String SERVERNAME_SEPARATOR = ",";
private HServerAddress serverAddress;
private long startCode;
@@ -51,11 +58,23 @@ public class HServerInfo implements Writ
private String serverName = null;
// Hostname of the regionserver.
private String hostname;
- private static Map<String,String> dnsCache = new HashMap<String,String>();
+ private String cachedHostnamePort = null;
public HServerInfo() {
- this(new HServerAddress(), 0,
- HConstants.DEFAULT_REGIONSERVER_INFOPORT, "default name");
+ this(new HServerAddress(), 0, HConstants.DEFAULT_REGIONSERVER_INFOPORT,
+ "default name");
+ }
+
+ /**
+ * Constructor that creates a HServerInfo with a generated startcode and an
+ * empty load.
+ * @param serverAddress An {@link InetSocketAddress} encased in a {@link Writable}
+ * @param infoPort Port the webui runs on.
+ * @param hostname Server hostname.
+ */
+ public HServerInfo(HServerAddress serverAddress, final int infoPort,
+ final String hostname) {
+ this(serverAddress, System.currentTimeMillis(), infoPort, hostname);
}
public HServerInfo(HServerAddress serverAddress, long startCode,
@@ -104,13 +123,27 @@ public class HServerInfo implements Writ
return this.infoPort;
}
- public void setInfoPort(int infoPort) {
- this.infoPort = infoPort;
+ public String getHostname() {
+ return this.hostname;
}
- public synchronized void setStartCode(long startCode) {
- this.startCode = startCode;
- this.serverName = null;
+ /**
+ * @return The hostname and port concatenated with a ':' as separator.
+ */
+ public synchronized String getHostnamePort() {
+ if (this.cachedHostnamePort == null) {
+ this.cachedHostnamePort = getHostnamePort(this.hostname, this.serverAddress.getPort());
+ }
+ return this.cachedHostnamePort;
+ }
+
+ /**
+ * @param hostname
+ * @param port
+ * @return The hostname and port concatenated with a ':' as separator.
+ */
+ public static String getHostnamePort(final String hostname, final int port) {
+ return hostname + ":" + port;
}
/**
@@ -119,43 +152,18 @@ public class HServerInfo implements Writ
*/
public synchronized String getServerName() {
if (this.serverName == null) {
- // if we have the hostname of the RS, use it
- if(this.hostname != null) {
- this.serverName =
- getServerName(this.hostname, this.serverAddress.getPort(), this.startCode);
- }
- // go to DNS name resolution only if we dont have the name of the RS
- else {
- this.serverName = getServerName(this.serverAddress, this.startCode);
- }
+ this.serverName = getServerName(this.hostname,
+ this.serverAddress.getPort(), this.startCode);
}
return this.serverName;
}
- /**
- * @param serverAddress In form <code><hostname> ':' <port></code>
- * @param startCode Server startcode
- * @return Server name made of the concatenation of hostname, port and
- * startcode formatted as <code><hostname> ',' <port> ',' <startcode></code>
- */
- public static String getServerName(String serverAddress, long startCode) {
- String name = null;
- if (serverAddress != null) {
- int colonIndex = serverAddress.lastIndexOf(':');
- if(colonIndex < 0) {
- throw new IllegalArgumentException("Not a host:port pair: " + serverAddress);
- }
- String host = serverAddress.substring(0, colonIndex);
- int port =
- Integer.valueOf(serverAddress.substring(colonIndex + 1)).intValue();
- if(!dnsCache.containsKey(host)) {
- HServerAddress address = new HServerAddress(serverAddress);
- dnsCache.put(host, address.getHostname());
- }
- host = dnsCache.get(host);
- name = getServerName(host, port, startCode);
- }
- return name;
+ public static synchronized String getServerName(final String hostAndPort,
+ final long startcode) {
+ int index = hostAndPort.indexOf(":");
+ if (index <= 0) throw new IllegalArgumentException("Expected <hostname> ':' <port>");
+ return getServerName(hostAndPort.substring(0, index),
+ Integer.parseInt(hostAndPort.substring(index + 1)), startcode);
}
/**
@@ -175,7 +183,7 @@ public class HServerInfo implements Writ
* @return Server name made of the concatenation of hostname, port and
* startcode formatted as <code><hostname> ',' <port> ',' <startcode></code>
*/
- private static String getServerName(String hostName, int port, long startCode) {
+ public static String getServerName(String hostName, int port, long startCode) {
StringBuilder name = new StringBuilder(hostName);
name.append(SERVERNAME_SEPARATOR);
name.append(port);
@@ -233,4 +241,27 @@ public class HServerInfo implements Writ
public int compareTo(HServerInfo o) {
return this.getServerName().compareTo(o.getServerName());
}
+
+ /**
+ * Utility method that does a find of a servername or a hostandport combination
+ * in the passed Set.
+ * @param servers Set of server names
+ * @param serverName Name to look for
+ * @param hostAndPortOnly If <code>serverName</code> is a
+ * <code>hostname ':' port</code>
+ * or <code>hostname , port , startcode</code>.
+ * @return True if <code>serverName</code> found in <code>servers</code>
+ */
+ public static boolean isServer(final Set<String> servers,
+ final String serverName, final boolean hostAndPortOnly) {
+ if (!hostAndPortOnly) return servers.contains(serverName);
+ String serverNameColonReplaced =
+ serverName.replaceFirst(":", SERVERNAME_SEPARATOR);
+ for (String hostPortStartCode: servers) {
+ int index = hostPortStartCode.lastIndexOf(SERVERNAME_SEPARATOR);
+ String hostPortStrippedOfStartCode = hostPortStartCode.substring(0, index);
+ if (hostPortStrippedOfStartCode.equals(serverNameColonReplaced)) return true;
+ }
+ return false;
+ }
}
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java Tue Sep 27 02:41:16 2011
@@ -797,8 +797,13 @@ public class KeyValue implements Writabl
HConstants.LATEST_TIMESTAMP_BYTES, 0, Bytes.SIZEOF_LONG) == 0;
}
+ /**
+ * @param now Time to set into <code>this</code> IFF timestamp ==
+ * {@link HConstants#LATEST_TIMESTAMP} (else, its a noop).
+ * @return True is we modified this.
+ */
public boolean updateLatestStamp(final byte [] now) {
- if(this.isLatestTimestamp()) {
+ if (this.isLatestTimestamp()) {
int tsOffset = getTimestampOffset();
System.arraycopy(now, 0, this.bytes, tsOffset, Bytes.SIZEOF_LONG);
return true;
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java Tue Sep 27 02:41:16 2011
@@ -915,7 +915,7 @@ public class HBaseAdmin {
private HRegionLocation getFirstMetaServerForTable(final byte [] tableName)
throws IOException {
return connection.locateRegion(HConstants.META_TABLE_NAME,
- HRegionInfo.createRegionName(tableName, null, HConstants.NINES));
+ HRegionInfo.createRegionName(tableName, null, HConstants.NINES, false));
}
/**
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java Tue Sep 27 02:41:16 2011
@@ -515,7 +515,7 @@ public class HConnectionManager implemen
int rowsScanned = 0;
int rowsOffline = 0;
byte[] startKey =
- HRegionInfo.createRegionName(tableName, null, HConstants.ZEROES);
+ HRegionInfo.createRegionName(tableName, null, HConstants.ZEROES, false);
byte[] endKey;
HRegionInfo currentRegion;
Scan scan = new Scan(startKey);
@@ -663,7 +663,7 @@ public class HConnectionManager implemen
// the extra 9's on the end are necessary to allow "exact" matches
// without knowing the precise region names.
byte [] metaKey = HRegionInfo.createRegionName(tableName, row,
- HConstants.NINES);
+ HConstants.NINES, false);
for (int tries = 0; true; tries++) {
if (tries >= numRetries) {
throw new NoServerForRegionException("Unable to find region for "
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTable.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTable.java Tue Sep 27 02:41:16 2011
@@ -514,6 +514,35 @@ public class HTable implements HTableInt
}
/**
+ * Atomically checks if a row/family/qualifier value match the expectedValue.
+ * If it does, it adds the delete. If value == null, checks for non-existence
+ * of the value.
+ *
+ * @param row to check
+ * @param family column family
+ * @param qualifier column qualifier
+ * @param value the expected value
+ * @param delete delete to execute if value matches.
+ * @throws IOException
+ * @return true if the new delete was executed, false otherwise
+ */
+ public boolean checkAndDelete(final byte [] row,
+ final byte [] family, final byte [] qualifier, final byte [] value,
+ final Delete delete)
+ throws IOException {
+ return connection.getRegionServerWithRetries(
+ new ServerCallable<Boolean>(connection, tableName, row) {
+ public Boolean call() throws IOException {
+ return server.checkAndDelete(
+ location.getRegionInfo().getRegionName(),
+ row, family, qualifier, value, delete)
+ ? Boolean.TRUE : Boolean.FALSE;
+ }
+ }
+ );
+ }
+
+ /**
* Test for the existence of columns in the table, as specified in the Get.<p>
*
* This will return true if the Get matches one or more keys, false if not.<p>
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java Tue Sep 27 02:41:16 2011
@@ -190,6 +190,22 @@ public interface HTableInterface {
void delete(List<Delete> deletes) throws IOException;
/**
+ * Atomically checks if a row/family/qualifier value matches the expected
+ * value. If it does, it adds the delete. If the passed value is null, the
+ * check is for the lack of column (ie: non-existance)
+ *
+ * @param row to check
+ * @param family column family to check
+ * @param qualifier column qualifier to check
+ * @param value the expected value
+ * @param delete data to delete if check succeeds
+ * @throws IOException e
+ * @return true if the new delete was executed, false otherwise
+ */
+ boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier,
+ byte[] value, Delete delete) throws IOException;
+
+ /**
* Atomically increments a column value.
* <p>
* Equivalent to {@code {@link #incrementColumnValue(byte[], byte[], byte[],
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java Tue Sep 27 02:41:16 2011
@@ -63,7 +63,7 @@ class MetaScanner implements HConstants
HConnection connection = HConnectionManager.getConnection(configuration);
byte [] startRow = tableName == null || tableName.length == 0 ?
HConstants.EMPTY_START_ROW :
- HRegionInfo.createRegionName(tableName, null, ZEROES);
+ HRegionInfo.createRegionName(tableName, null, ZEROES, false);
// Scan over each meta region
ScannerCallable callable;
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java Tue Sep 27 02:41:16 2011
@@ -80,15 +80,15 @@ public class ScannerCallable extends Ser
if (e instanceof RemoteException) {
ioe = RemoteExceptionHandler.decodeRemoteException((RemoteException)e);
}
- if (ioe != null) {
- if (ioe instanceof NotServingRegionException) {
+ if (ioe == null) throw new IOException(e);
+ if (ioe instanceof NotServingRegionException) {
// Throw a DNRE so that we break out of cycle of calling NSRE
// when what we need is to open scanner against new location.
// Attach NSRE to signal client that it needs to resetup scanner.
throw new DoNotRetryIOException("Reset scanner", ioe);
- } else if (ioe instanceof DoNotRetryIOException) {
- throw ioe;
- }
+ } else {
+ // The outer layers will retry
+ throw ioe;
}
}
return rrs;
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java Tue Sep 27 02:41:16 2011
@@ -45,8 +45,8 @@ public class ValueFilter extends Compare
/**
* Constructor.
- * @param valueCompareOp the compare op for column qualifier matching
- * @param valueComparator the comparator for column qualifier matching
+ * @param valueCompareOp the compare op for value matching
+ * @param valueComparator the comparator for value matching
*/
public ValueFilter(final CompareOp valueCompareOp,
final WritableByteArrayComparable valueComparator) {
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java?rev=1176171&r1=1176170&r2=1176171&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java Tue Sep 27 02:41:16 2011
@@ -23,6 +23,7 @@ package org.apache.hadoop.hbase.io;
import java.io.IOException;
import java.io.DataInput;
import java.io.DataOutput;
+import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.io.BytesWritable;
@@ -83,7 +84,7 @@ implements WritableComparable<ImmutableB
/**
* Get the data from the BytesWritable.
- * @return The data is only valid between 0 and getSize() - 1.
+ * @return The data is only valid between offset and offset+length.
*/
public byte [] get() {
if (this.bytes == null) {
@@ -112,7 +113,7 @@ implements WritableComparable<ImmutableB
}
/**
- * @return the current size of the buffer.
+ * @return the number of valid bytes in the buffer
*/
public int getSize() {
if (this.bytes == null) {
@@ -123,7 +124,7 @@ implements WritableComparable<ImmutableB
}
/**
- * @return the current length of the buffer. same as getSize()
+ * @return the number of valid bytes in the buffer
*/
//Should probably deprecate getSize() so that we keep the same calls for all
//byte []
@@ -155,20 +156,24 @@ implements WritableComparable<ImmutableB
}
// Below methods copied from BytesWritable
-
@Override
public int hashCode() {
- return WritableComparator.hashBytes(bytes, this.length);
+ int hash = 1;
+ for (int i = offset; i < offset + length; i++)
+ hash = (31 * hash) + (int)bytes[i];
+ return hash;
}
/**
* Define the sort order of the BytesWritable.
- * @param right_obj The other bytes writable
+ * @param that The other bytes writable
* @return Positive if left is bigger than right, 0 if they are equal, and
* negative if left is smaller than right.
*/
- public int compareTo(ImmutableBytesWritable right_obj) {
- return compareTo(right_obj.get());
+ public int compareTo(ImmutableBytesWritable that) {
+ return WritableComparator.compareBytes(
+ this.bytes, this.offset, this.length,
+ that.bytes, that.offset, that.length);
}
/**
@@ -178,8 +183,9 @@ implements WritableComparable<ImmutableB
* negative if left is smaller than right.
*/
public int compareTo(final byte [] that) {
- return WritableComparator.compareBytes(this.bytes, 0, this.length, that,
- 0, that.length);
+ return WritableComparator.compareBytes(
+ this.bytes, this.offset, this.length,
+ that, 0, that.length);
}
/**
@@ -202,9 +208,9 @@ implements WritableComparable<ImmutableB
@Override
public String toString() {
StringBuilder sb = new StringBuilder(3*this.bytes.length);
- for (int idx = 0; idx < this.bytes.length; idx++) {
+ for (int idx = offset; idx < offset + length; idx++) {
// if not the first, put a blank separator in
- if (idx != 0) {
+ if (idx != offset) {
sb.append(' ');
}
String num = Integer.toHexString(bytes[idx]);
@@ -253,4 +259,11 @@ implements WritableComparable<ImmutableB
}
return results;
}
+
+ /**
+ * Returns a copy of the bytes referred to by this writable
+ */
+ public byte[] copyBytes() {
+ return Arrays.copyOfRange(bytes, offset, offset+length);
+ }
}