You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/10/11 03:59:52 UTC
svn commit: r1181336 [1/3] - in /hbase/branches/0.89: ./ bin/ conf/
src/main/java/org/apache/hadoop/hbase/io/
src/main/java/org/apache/hadoop/hbase/master/
src/main/java/org/apache/hadoop/hbase/regionserver/
src/main/java/org/apache/hadoop/hbase/region...
Author: nspiegelberg
Date: Tue Oct 11 01:59:50 2011
New Revision: 1181336
URL: http://svn.apache.org/viewvc?rev=1181336&view=rev
Log:
HBase: refresh internal branch from trunk (964185 -> 966084)
Summary:
http://svn.apache.org/repos/asf/hbase/trunk
r964185 -> r966084
Picks up among other things:
* Fix for HBase-2727 (followup to HBase-1025).
* Nicolas's pseudo-distrbuted stuff
* Pranav's early exit optimzation for timestamp range queries (HBASE-2517).
Test Plan:
Running mvn test right now.
Will also run cluster tests.
DiffCamp Revision: 135655
Reviewed By: nspiegelberg
CC: davidrecordon, nspiegelberg, achao, hbase@lists
Revert Plan:
OK
Added:
hbase/branches/0.89/bin/local-master-backup.sh (with props)
hbase/branches/0.89/bin/local-regionservers.sh (with props)
hbase/branches/0.89/conf/hbase-site.xml.psuedo-distributed.template
hbase/branches/0.89/src/main/javadoc/org/apache/hadoop/hbase/replication/
hbase/branches/0.89/src/main/javadoc/org/apache/hadoop/hbase/replication/package.html
- copied, changed from r1176177, hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/replication/package.html
hbase/branches/0.89/src/site/resources/images/
hbase/branches/0.89/src/site/resources/images/architecture.gif
hbase/branches/0.89/src/site/resources/images/asf_logo_wide.png
hbase/branches/0.89/src/site/resources/images/favicon.ico
hbase/branches/0.89/src/site/resources/images/hadoop-logo.jpg
hbase/branches/0.89/src/site/resources/images/hbase_logo_med.gif
hbase/branches/0.89/src/site/resources/images/hbase_small.gif
hbase/branches/0.89/src/site/resources/images/replication_overview.png
hbase/branches/0.89/src/site/xdoc/pseudo-distributed.xml
hbase/branches/0.89/src/site/xdoc/replication.xml
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogMethods.java
Removed:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteCompare.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/GetDeleteTracker.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/QueryMatcher.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileGetScan.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/WildcardColumnTracker.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/replication/package.html
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/master/TestMasterWrongRS.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteCompare.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetDeleteTracker.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestWildcardColumnTracker.java
Modified:
hbase/branches/0.89/CHANGES.txt
hbase/branches/0.89/bin/hbase
hbase/branches/0.89/pom.xml
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ReadWriteConsistencyControl.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/rest/Main.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
hbase/branches/0.89/src/main/javadoc/overview.html
hbase/branches/0.89/src/main/ruby/shell.rb
hbase/branches/0.89/src/site/site.xml
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/client/TestMultipleTimestamps.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/util/TestByteBloomFilter.java
Modified: hbase/branches/0.89/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/CHANGES.txt?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/CHANGES.txt (original)
+++ hbase/branches/0.89/CHANGES.txt Tue Oct 11 01:59:50 2011
@@ -25,6 +25,8 @@ Release 0.21.0 - Unreleased
HBASE-2565 Remove contrib module from hbase
HBASE-2397 Bytes.toStringBinary escapes printable chars
HBASE-2771 Update our hadoop jar to be latest from 0.20-append branch
+ HBASE-2803 Remove remaining Get code from Store.java,etc
+ HBASE-2553 Revisit IncrementColumnValue implementation in 0.22
BUG FIXES
HBASE-1791 Timeout in IndexRecordWriter (Bradford Stephens via Andrew
@@ -434,6 +436,17 @@ Release 0.21.0 - Unreleased
(Nicolas Spiegelberg via Stack)
HBASE-2781 ZKW.createUnassignedRegion doesn't make sure existing znode is
in the right state (Karthik Ranganathan via JD)
+ HBASE-2727 Splits writing one file only is untenable; need dir of recovered
+ edits ordered by sequenceid
+ HBASE-2843 Readd bloomfilter test over zealously removed by HBASE-2625
+ HBASE-2846 Make rest server be same as thrift and avro servers
+ HBASE-1511 Pseudo distributed mode in LocalHBaseCluster
+ (Nicolas Spiegelberg via Stack)
+ HBASE-2851 Remove testDynamicBloom() unit test
+ (Nicolas Spiegelberg via Stack)
+ HBASE-2853 TestLoadIncrementalHFiles fails on TRUNK
+ HBASE-2854 broken tests on trunk
+ HBASE-2859 Cleanup deprecated stuff in TestHLog (Alex Newman via Stack)
IMPROVEMENTS
HBASE-1760 Cleanup TODOs in HTable
@@ -763,7 +776,17 @@ Release 0.21.0 - Unreleased
HBASE-2828 HTable unnecessarily coupled with HMaster
(Nicolas Spiegelberg via Stack)
HBASE-2265 HFile and Memstore should maintain minimum and maximum timestamps
- (Pranav via Ryan)
+ (Pranav via Ryan)
+ HBASE-2836 Speed mvn site building by removing generation of useless reports
+ HBASE-2808 Document the implementation of replication
+ HBASE-2517 During reads when passed the specified time range, seek to
+ next column (Pranav via jgray)
+ HBASE-2835 Update hadoop jar to head of branch-0.20-append to catch three
+ added patches
+ HBASE-2840 Remove the final remnants of the old Get code - the query matchers
+ and other helper classes
+ HBASE-2845 Small edit of shell main help page cutting down some on white
+ space and text
NEW FEATURES
HBASE-1961 HBase EC2 scripts
Modified: hbase/branches/0.89/bin/hbase
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/bin/hbase?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/bin/hbase (original)
+++ hbase/branches/0.89/bin/hbase Tue Oct 11 01:59:50 2011
@@ -67,11 +67,12 @@ if [ $# = 0 ]; then
if $in_sources_dir; then
echo " shell-tests run the HBase shell tests"
fi
- echo " master run an HBase HMaster node"
- echo " regionserver run an HBase HRegionServer node"
- echo " thrift run an HBase Thrift server"
- echo " avro run an HBase Avro server"
+ echo " master run an HBase HMaster node"
+ echo " regionserver run an HBase HRegionServer node"
echo " zookeeper run a Zookeeper server"
+ echo " rest run an HBase REST server"
+ echo " thrift run an HBase Thrift server"
+ echo " avro run an HBase Avro server"
echo " migrate upgrade an hbase.rootdir"
echo " hbck run the hbase 'fsck' tool"
echo " or"
@@ -239,6 +240,11 @@ elif [ "$COMMAND" = "thrift" ] ; then
if [ "$1" != "stop" ] ; then
HBASE_OPTS="$HBASE_OPTS $HBASE_THRIFT_OPTS"
fi
+elif [ "$COMMAND" = "rest" ] ; then
+ CLASS='org.apache.hadoop.hbase.rest.Main'
+ if [ "$1" != "stop" ] ; then
+ HBASE_OPTS="$HBASE_OPTS $HBASE_REST_OPTS"
+ fi
elif [ "$COMMAND" = "avro" ] ; then
CLASS='org.apache.hadoop.hbase.avro.AvroServer'
if [ "$1" != "stop" ] ; then
Added: hbase/branches/0.89/bin/local-master-backup.sh
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/bin/local-master-backup.sh?rev=1181336&view=auto
==============================================================================
--- hbase/branches/0.89/bin/local-master-backup.sh (added)
+++ hbase/branches/0.89/bin/local-master-backup.sh Tue Oct 11 01:59:50 2011
@@ -0,0 +1,35 @@
+#!/bin/sh
+# This is used for starting multiple masters on the same machine.
+# run it from hbase-dir/ just like 'bin/hbase'
+# Supports up to 10 masters (limitation = overlapping ports)
+
+bin=`dirname "$0"`
+bin=`cd "$bin" >/dev/null && pwd`
+
+if [ $# -lt 2 ]; then
+ S=`basename $0`
+ echo "Usage: $S [start|stop] offset(s)"
+ echo ""
+ echo " e.g. $S start 1"
+ exit
+fi
+
+# sanity check: make sure your master opts don't use ports [i.e. JMX/DBG]
+export HBASE_MASTER_OPTS=" "
+
+run_master () {
+ DN=$2
+ export HBASE_IDENT_STRING="$USER-$DN"
+ HBASE_MASTER_ARGS="\
+ -D hbase.master.port=`expr 60000 + $DN` \
+ -D hbase.master.info.port=`expr 60010 + $DN`"
+ "$bin"/hbase-daemon.sh $1 master $HBASE_MASTER_ARGS
+}
+
+cmd=$1
+shift;
+
+for i in $*
+do
+ run_master $cmd $i
+done
Propchange: hbase/branches/0.89/bin/local-master-backup.sh
------------------------------------------------------------------------------
svn:executable = *
Added: hbase/branches/0.89/bin/local-regionservers.sh
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/bin/local-regionservers.sh?rev=1181336&view=auto
==============================================================================
--- hbase/branches/0.89/bin/local-regionservers.sh (added)
+++ hbase/branches/0.89/bin/local-regionservers.sh Tue Oct 11 01:59:50 2011
@@ -0,0 +1,35 @@
+#!/bin/sh
+# This is used for starting multiple regionservers on the same machine.
+# run it from hbase-dir/ just like 'bin/hbase'
+# Supports up to 100 regionservers (limitation = overlapping ports)
+
+bin=`dirname "$0"`
+bin=`cd "$bin" >/dev/null && pwd`
+
+if [ $# -lt 2 ]; then
+ S=`basename $0`
+ echo "Usage: $S [start|stop] offset(s)"
+ echo ""
+ echo " e.g. $S start 1 2"
+ exit
+fi
+
+# sanity check: make sure your regionserver opts don't use ports [i.e. JMX/DBG]
+export HBASE_REGIONSERVER_OPTS=" "
+
+run_regionserver () {
+ DN=$2
+ export HBASE_IDENT_STRING="$USER-$DN"
+ HBASE_REGIONSERVER_ARGS="\
+ -D hbase.regionserver.port=`expr 60200 + $DN` \
+ -D hbase.regionserver.info.port=`expr 60300 + $DN`"
+ "$bin"/hbase-daemon.sh $1 regionserver $HBASE_REGIONSERVER_ARGS
+}
+
+cmd=$1
+shift;
+
+for i in $*
+do
+ run_regionserver $cmd $i
+done
Propchange: hbase/branches/0.89/bin/local-regionservers.sh
------------------------------------------------------------------------------
svn:executable = *
Added: hbase/branches/0.89/conf/hbase-site.xml.psuedo-distributed.template
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/conf/hbase-site.xml.psuedo-distributed.template?rev=1181336&view=auto
==============================================================================
--- hbase/branches/0.89/conf/hbase-site.xml.psuedo-distributed.template (added)
+++ hbase/branches/0.89/conf/hbase-site.xml.psuedo-distributed.template Tue Oct 11 01:59:50 2011
@@ -0,0 +1,77 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+
+<!-- NEEDED WHETHER OR NOT YOU ARE RUNNING OVER HDFS -->
+<property>
+ <name>hbase.cluster.distributed</name>
+ <value>true</value>
+ <description>For psuedo-distributed, you want to set this to true.
+ false means that HBase tries to put Master + RegionServers in one process.
+ Pseudo-distributed = seperate processes/pids</description>
+</property> <property>
+ <name>hbase.regionserver.hlog.replication</name>
+ <value>1</value>
+ <description>For HBase to offer good data durability, we roll logs if
+ filesystem replication falls below a certain amount. In psuedo-distributed
+ mode, you normally only have the local filesystem or 1 HDFS DataNode, so you
+ don't want to roll logs constantly.</description>
+</property>
+<property>
+ <name>hbase.tmp.dir</name>
+ <value>/tmp/hbase-testing</value>
+ <description>Temporary directory on the local filesystem.</description>
+</property>
+
+<!-- DEFAULT = use local filesystem, not HDFS
+ ADD THESE LINES if you have a copy of HDFS source and want to run HBase
+ psuedo-distributed over a psuedo-distributed HDFS cluster.
+ For HDFS psuedo-distributed setup, see their documentation:
+
+ http://hadoop.apache.org/common/docs/r0.20.2/quickstart.html#PseudoDistributed
+
+
+<property>
+ <name>hbase.rootdir</name>
+ <value>hdfs://localhost:9000/hbase-testing</value>
+ <description>The directory shared by region servers.
+ Should be fully-qualified to include the filesystem to use.
+ E.g: hdfs://NAMENODE_SERVER:PORT/HBASE_ROOTDIR
+ </description>
+</property>
+-->
+
+<!-- OPTIONAL: You might want to add these options depending upon your use case
+
+
+<property>
+ <name>dfs.support.append</name>
+ <value>true</value>
+ <description>Allow append support (if you want to test data durability with HDFS)
+ </description>
+</property>
+-->
+
+
+</configuration>
Modified: hbase/branches/0.89/pom.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/pom.xml?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/pom.xml (original)
+++ hbase/branches/0.89/pom.xml Tue Oct 11 01:59:50 2011
@@ -820,6 +820,20 @@
<plugin>
<artifactId>maven-project-info-reports-plugin</artifactId>
<version>2.1.2</version>
+ <reportSets>
+ <reportSet>
+ <reports>
+ <report>project-team</report>
+ <report>mailing-list</report>
+ <report>cim</report>
+ <report>issue-tracking</report>
+ <report>license</report>
+ <report>scm</report>
+ <report>index</report>
+ </reports>
+ </reportSet>
+ </reportSets>
+
</plugin>
<!-- Disabled for now.
<plugin>
@@ -860,6 +874,14 @@
<configuration>
<docfilessubdirs>true</docfilessubdirs>
</configuration>
+ <reportSets>
+ <reportSet>
+ <id>default</id>
+ <reports>
+ <report>javadoc</report>
+ </reports>
+ </reportSet>
+ </reportSets>
<!--
This is probably not needed, given the smallness of the HBase source code, but left here in case
<minmemory>128m</minmemory>
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java Tue Oct 11 01:59:50 2011
@@ -147,6 +147,23 @@ public class TimeRange implements Writab
return (timestamp >= minStamp);
}
+ /**
+ * Compare the timestamp to timerange
+ * @param timestamp
+ * @return -1 if timestamp is less than timerange,
+ * 0 if timestamp is within timerange,
+ * 1 if timestamp is greater than timerange
+ */
+ public int compare(long timestamp) {
+ if (timestamp < minStamp) {
+ return -1;
+ } else if (timestamp >= maxStamp) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Tue Oct 11 01:59:50 2011
@@ -37,6 +37,10 @@ import java.util.concurrent.atomic.Atomi
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -1199,6 +1203,7 @@ public class HMaster extends Thread impl
System.err.println(" stop Start cluster shutdown; Master signals RegionServer shutdown");
System.err.println(" where [opts] are:");
System.err.println(" --minServers=<servers> Minimum RegionServers needed to host user tables.");
+ System.err.println(" -D opt=<value> Override HBase configuration settings.");
System.exit(0);
}
@@ -1250,20 +1255,34 @@ public class HMaster extends Thread impl
protected static void doMain(String [] args,
Class<? extends HMaster> masterClass) {
- if (args.length < 1) {
- printUsageAndExit();
- }
Configuration conf = HBaseConfiguration.create();
- // Process command-line args.
- for (String cmd: args) {
- if (cmd.startsWith("--minServers=")) {
+ Options opt = new Options();
+ opt.addOption("minServers", true, "Minimum RegionServers needed to host user tables");
+ opt.addOption("D", true, "Override HBase Configuration Settings");
+ try {
+ CommandLine cmd = new GnuParser().parse(opt, args);
+
+ if (cmd.hasOption("minServers")) {
+ String val = cmd.getOptionValue("minServers");
conf.setInt("hbase.regions.server.count.min",
- Integer.valueOf(cmd.substring(13)));
- continue;
+ Integer.valueOf(val));
+ LOG.debug("minServers set to " + val);
}
- if (cmd.equalsIgnoreCase("start")) {
+ if (cmd.hasOption("D")) {
+ for (String confOpt : cmd.getOptionValues("D")) {
+ String[] kv = confOpt.split("=", 2);
+ if (kv.length == 2) {
+ conf.set(kv[0], kv[1]);
+ LOG.debug("-D configuration override: " + kv[0] + "=" + kv[1]);
+ } else {
+ throw new ParseException("-D option format invalid: " + confOpt);
+ }
+ }
+ }
+
+ if (cmd.getArgList().contains("start")) {
try {
// Print out vm stats before starting up.
RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();
@@ -1312,10 +1331,7 @@ public class HMaster extends Thread impl
LOG.error("Failed to start master", t);
System.exit(-1);
}
- break;
- }
-
- if (cmd.equalsIgnoreCase("stop")) {
+ } else if (cmd.getArgList().contains("stop")) {
HBaseAdmin adm = null;
try {
adm = new HBaseAdmin(conf);
@@ -1329,10 +1345,12 @@ public class HMaster extends Thread impl
LOG.error("Failed to stop master", t);
System.exit(-1);
}
- break;
+ } else {
+ throw new ParseException("Unknown argument(s): " +
+ org.apache.commons.lang.StringUtils.join(cmd.getArgs(), " "));
}
-
- // Print out usage if we get to here.
+ } catch (ParseException e) {
+ LOG.error("Could not parse: ", e);
printUsageAndExit();
}
}
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java Tue Oct 11 01:59:50 2011
@@ -19,8 +19,6 @@
*/
package org.apache.hadoop.hbase.regionserver;
-import org.apache.hadoop.hbase.regionserver.QueryMatcher.MatchCode;
-
/**
* Implementing classes of this interface will be used for the tracking
* and enforcement of columns and numbers of versions during the course of a
@@ -29,12 +27,10 @@ import org.apache.hadoop.hbase.regionser
* Currently there are two different types of Store/Family-level queries.
* <ul><li>{@link ExplicitColumnTracker} is used when the query specifies
* one or more column qualifiers to return in the family.
- * <li>{@link WildcardColumnTracker} is used when the query asks for all
- * qualifiers within the family.
* <p>
- * This class is utilized by {@link QueryMatcher} through two methods:
+ * This class is utilized by {@link ScanQueryMatcher} through two methods:
* <ul><li>{@link #checkColumn} is called when a Put satisfies all other
- * conditions of the query. This method returns a {@link MatchCode} to define
+ * conditions of the query. This method returns a {@link org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode} to define
* what action should be taken.
* <li>{@link #update} is called at the end of every StoreFile or memstore.
* <p>
@@ -48,7 +44,7 @@ public interface ColumnTracker {
* @param length
* @return The match code instance.
*/
- public MatchCode checkColumn(byte [] bytes, int offset, int length);
+ public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset, int length);
/**
* Updates internal variables in between files
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java Tue Oct 11 01:59:50 2011
@@ -22,7 +22,7 @@ package org.apache.hadoop.hbase.regionse
import java.util.ArrayList;
import java.util.List;
import java.util.NavigableSet;
-import org.apache.hadoop.hbase.regionserver.QueryMatcher.MatchCode;
+
import org.apache.hadoop.hbase.util.Bytes;
/**
@@ -36,9 +36,9 @@ import org.apache.hadoop.hbase.util.Byte
* between rows.
*
* <p>
- * This class is utilized by {@link QueryMatcher} through two methods:
+ * This class is utilized by {@link ScanQueryMatcher} through two methods:
* <ul><li>{@link #checkColumn} is called when a Put satisfies all other
- * conditions of the query. This method returns a {@link MatchCode} to define
+ * conditions of the query. This method returns a {@link org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode} to define
* what action should be taken.
* <li>{@link #update} is called at the end of every StoreFile or memstore.
* <p>
@@ -84,18 +84,18 @@ public class ExplicitColumnTracker imple
* @param bytes KeyValue buffer
* @param offset offset to the start of the qualifier
* @param length length of the qualifier
- * @return MatchCode telling QueryMatcher what action to take
+ * @return MatchCode telling ScanQueryMatcher what action to take
*/
- public MatchCode checkColumn(byte [] bytes, int offset, int length) {
+ public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset, int length) {
do {
// No more columns left, we are done with this query
if(this.columns.size() == 0) {
- return MatchCode.DONE; // done_row
+ return ScanQueryMatcher.MatchCode.DONE; // done_row
}
// No more columns to match against, done with storefile
if(this.column == null) {
- return MatchCode.NEXT; // done_row
+ return ScanQueryMatcher.MatchCode.NEXT; // done_row
}
// Compare specific column to current column
@@ -114,13 +114,13 @@ public class ExplicitColumnTracker imple
this.column = this.columns.get(this.index);
}
}
- return MatchCode.INCLUDE;
+ return ScanQueryMatcher.MatchCode.INCLUDE;
}
if (ret > 0) {
// Specified column is smaller than the current, skip to next column.
- return MatchCode.SKIP;
+ return ScanQueryMatcher.MatchCode.SKIP;
}
// Specified column is bigger than current column
@@ -128,7 +128,7 @@ public class ExplicitColumnTracker imple
if(ret <= -1) {
if(++this.index == this.columns.size()) {
// No more to match, do not include, done with storefile
- return MatchCode.NEXT; // done_row
+ return ScanQueryMatcher.MatchCode.NEXT; // done_row
}
// This is the recursive case.
this.column = this.columns.get(this.index);
@@ -163,4 +163,39 @@ public class ExplicitColumnTracker imple
col.setCount(this.maxVersions);
}
}
+
+ /**
+ * This method is used to inform the column tracker that we are done with
+ * this column. We may get this information from external filters or
+ * timestamp range and we then need to indicate this information to
+ * tracker. It is required only in case of ExplicitColumnTracker.
+ * @param bytes
+ * @param offset
+ * @param length
+ */
+ public void doneWithColumn(byte [] bytes, int offset, int length) {
+ while (this.column != null) {
+ int compare = Bytes.compareTo(column.getBuffer(), column.getOffset(),
+ column.getLength(), bytes, offset, length);
+ if (compare == 0) {
+ this.columns.remove(this.index);
+ if (this.columns.size() == this.index) {
+ // Will not hit any more columns in this storefile
+ this.column = null;
+ } else {
+ this.column = this.columns.get(this.index);
+ }
+ return;
+ } else if ( compare <= -1) {
+ if(++this.index != this.columns.size()) {
+ this.column = this.columns.get(this.index);
+ } else {
+ this.column = null;
+ }
+ } else {
+ return;
+ }
+ }
+ }
+
}
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java Tue Oct 11 01:59:50 2011
@@ -31,7 +31,7 @@ import org.apache.hadoop.hbase.util.Byte
/**
* State and utility processing {@link HRegion#getClosestRowBefore(byte[], byte[])}.
- * Like {@link GetDeleteTracker} and {@link ScanDeleteTracker} but does not
+ * Like {@link ScanDeleteTracker} and {@link ScanDeleteTracker} but does not
* implement the {@link DeleteTracker} interface since state spans rows (There
* is no update nor reset method).
*/
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Tue Oct 11 01:59:50 2011
@@ -19,6 +19,27 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Constructor;
+import java.util.AbstractList;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -64,28 +85,6 @@ import org.apache.hadoop.util.StringUtil
import com.google.common.collect.Lists;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.lang.reflect.Constructor;
-import java.util.AbstractList;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.Random;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
/**
* HRegion stores data for a certain region of a table. It stores all columns
* for each row. A given table consists of one or more HRegions.
@@ -126,6 +125,7 @@ public class HRegion implements HeapSize
public static final Log LOG = LogFactory.getLog(HRegion.class);
static final String SPLITDIR = "splits";
static final String MERGEDIR = "merges";
+
final AtomicBoolean closed = new AtomicBoolean(false);
/* Closing can take some time; use the closing flag if there is stuff we don't
* want to do while in closing state; e.g. like offer this region up to the
@@ -330,9 +330,8 @@ public class HRegion implements HeapSize
// Remove temporary data left over from old regions
cleanupTmpDir();
- // Load in all the HStores. Get min and max seqids across all families.
+ // Load in all the HStores. Get maximum seqid.
long maxSeqId = -1;
- long minSeqId = Integer.MAX_VALUE;
for (HColumnDescriptor c : this.regionInfo.getTableDesc().getFamilies()) {
Store store = instantiateHStore(this.tableDir, c);
this.stores.put(c.getName(), store);
@@ -340,12 +339,9 @@ public class HRegion implements HeapSize
if (storeSeqId > maxSeqId) {
maxSeqId = storeSeqId;
}
- if (minSeqId > storeSeqId) {
- minSeqId = storeSeqId;
- }
}
// Recover any edits if available.
- long seqid = replayRecoveredEditsIfAny(this.regiondir, minSeqId, reporter);
+ maxSeqId = replayRecoveredEditsIfAny(this.regiondir, maxSeqId, reporter);
// Get rid of any splits or merges that were lost in-progress. Clean out
// these directories here on open. We may be opening a region that was
@@ -362,7 +358,7 @@ public class HRegion implements HeapSize
this.lastFlushTime = EnvironmentEdgeManager.currentTimeMillis();
// Use maximum of log sequenceid or that which was found in stores
// (particularly if no recovered edits, seqid will be -1).
- long nextSeqid = Math.max(seqid, maxSeqId) + 1;
+ long nextSeqid = maxSeqId + 1;
LOG.info("Onlined " + this.toString() + "; next sequenceid=" + nextSeqid);
return nextSeqid;
}
@@ -902,7 +898,9 @@ public class HRegion implements HeapSize
}
/**
- * Flushing the cache is a little tricky. We have a lot of updates in the
+ * Flush the memstore.
+ *
+ * Flushing the memstore is a little tricky. We have a lot of updates in the
* memstore, all of which have also been written to the log. We need to
* write those updates in the memstore out to disk, while being able to
* process reads/writes as much as possible during the flush operation. Also,
@@ -934,6 +932,19 @@ public class HRegion implements HeapSize
* because a Snapshot was not properly persisted.
*/
protected boolean internalFlushcache() throws IOException {
+ return internalFlushcache(this.log, -1);
+ }
+
+ /**
+ * @param wal Null if we're NOT to go via hlog/wal.
+ * @param myseqid The seqid to use if <code>wal</code> is null writing out
+ * flush file.
+ * @return true if the region needs compacting
+ * @throws IOException
+ * @see {@link #internalFlushcache()}
+ */
+ protected boolean internalFlushcache(final HLog wal, final long myseqid)
+ throws IOException {
final long startTime = EnvironmentEdgeManager.currentTimeMillis();
// Clear flush flag.
// Record latest flush time
@@ -945,7 +956,8 @@ public class HRegion implements HeapSize
if (LOG.isDebugEnabled()) {
LOG.debug("Started memstore flush for region " + this +
". Current region memstore size " +
- StringUtils.humanReadableInt(this.memstoreSize.get()));
+ StringUtils.humanReadableInt(this.memstoreSize.get()) +
+ ((wal != null)? "": "; wal is null, using passed myseqid=" + myseqid));
}
// Stop updates while we snapshot the memstore of all stores. We only have
@@ -958,14 +970,14 @@ public class HRegion implements HeapSize
long sequenceId = -1L;
long completeSequenceId = -1L;
- // we have to take a write lock during snapshot, or else a write could
+ // We have to take a write lock during snapshot, or else a write could
// end up in both snapshot and memstore (makes it difficult to do atomic
// rows then)
this.updatesLock.writeLock().lock();
final long currentMemStoreSize = this.memstoreSize.get();
List<StoreFlusher> storeFlushers = new ArrayList<StoreFlusher>(stores.size());
try {
- sequenceId = log.startCacheFlush();
+ sequenceId = (wal == null)? myseqid: wal.startCacheFlush();
completeSequenceId = this.getCompleteCacheFlushSequenceId(sequenceId);
for (Store s : stores.values()) {
@@ -1009,9 +1021,9 @@ public class HRegion implements HeapSize
}
try {
- if (atomicWork != null) {
- atomicWork.call();
- }
+ if (atomicWork != null) {
+ atomicWork.call();
+ }
// Switch snapshot (in memstore) -> new hfile (thus causing
// all the store scanners to reset/reseek).
@@ -1038,7 +1050,7 @@ public class HRegion implements HeapSize
// We used to only catch IOEs but its possible that we'd get other
// exceptions -- e.g. HBASE-659 was about an NPE -- so now we catch
// all and sundry.
- this.log.abortCacheFlush();
+ if (wal != null) wal.abortCacheFlush();
DroppedSnapshotException dse = new DroppedSnapshotException("region: " +
Bytes.toStringBinary(getRegionName()));
dse.initCause(t);
@@ -1052,9 +1064,11 @@ public class HRegion implements HeapSize
// This tells future readers that the HStores were emitted correctly,
// and that all updates to the log for this regionName that have lower
// log-sequence-ids can be safely ignored.
- this.log.completeCacheFlush(getRegionName(),
+ if (wal != null) {
+ wal.completeCacheFlush(getRegionName(),
regionInfo.getTableDesc().getName(), completeSequenceId,
this.getRegionInfo().isMetaRegion());
+ }
// C. Finally notify anyone waiting on memstore to clear:
// e.g. checkResources().
@@ -1067,12 +1081,12 @@ public class HRegion implements HeapSize
LOG.info("Finished memstore flush of ~" +
StringUtils.humanReadableInt(currentMemStoreSize) + " for region " +
this + " in " + (now - startTime) + "ms, sequence id=" + sequenceId +
- ", compaction requested=" + compactionRequested);
+ ", compaction requested=" + compactionRequested +
+ ((wal == null)? "; wal=null": ""));
}
return compactionRequested;
}
-
/**
* A hook for sub classed wishing to perform operations prior to the cache
* flush commit stage.
@@ -1853,46 +1867,75 @@ public class HRegion implements HeapSize
* Read the edits log put under this region by wal log splitting process. Put
* the recovered edits back up into this region.
*
- * We can ignore any log message that has a sequence ID that's equal to or
+ * <p>We can ignore any log message that has a sequence ID that's equal to or
* lower than minSeqId. (Because we know such log messages are already
* reflected in the HFiles.)
+ *
+ * <p>While this is running we are putting pressure on memory yet we are
+ * outside of our usual accounting because we are not yet an onlined region
+ * (this stuff is being run as part of Region initialization). This means
+ * that if we're up against global memory limits, we'll not be flagged to flush
+ * because we are not online. We can't be flushed by usual mechanisms anyways;
+ * we're not yet online so our relative sequenceids are not yet aligned with
+ * HLog sequenceids -- not till we come up online, post processing of split
+ * edits.
+ *
+ * <p>But to help relieve memory pressure, at least manage our own heap size
+ * flushing if are in excess of per-region limits. Flushing, though, we have
+ * to be careful and avoid using the regionserver/hlog sequenceid. Its running
+ * on a different line to whats going on in here in this region context so if we
+ * crashed replaying these edits, but in the midst had a flush that used the
+ * regionserver log with a sequenceid in excess of whats going on in here
+ * in this region and with its split editlogs, then we could miss edits the
+ * next time we go to recover. So, we have to flush inline, using seqids that
+ * make sense in a this single region context only -- until we online.
+ *
* @param regiondir
- * @param minSeqId Minimum sequenceid found in a store file. Edits in log
- * must be larger than this to be replayed.
+ * @param minSeqId Any edit found in split editlogs needs to be in excess of
+ * this minSeqId to be applied, else its skipped.
* @param reporter
* @return the sequence id of the last edit added to this region out of the
- * recovered edits log, or -1 if no log recovered
+ * recovered edits log or <code>minSeqId</code> if nothing added from editlogs.
* @throws UnsupportedEncodingException
* @throws IOException
*/
protected long replayRecoveredEditsIfAny(final Path regiondir,
final long minSeqId, final Progressable reporter)
throws UnsupportedEncodingException, IOException {
- Path edits = new Path(regiondir, HLog.RECOVERED_EDITS);
- if (edits == null || !this.fs.exists(edits)) return -1;
- if (isZeroLengthThenDelete(this.fs, edits)) return -1;
- long maxSeqIdInLog = -1;
- try {
- maxSeqIdInLog = replayRecoveredEdits(edits, minSeqId, reporter);
- LOG.debug("Deleting recovered edits file: " + edits);
- if (!this.fs.delete(edits, false)) {
- LOG.error("Failed delete of " + edits);
- }
- } catch (IOException e) {
- boolean skipErrors = conf.getBoolean("hbase.skip.errors", false);
- if (skipErrors) {
- Path moveAsideName = new Path(edits.getParent(), edits.getName() + "." +
- System.currentTimeMillis());
- LOG.error("hbase.skip.errors=true so continuing. Renamed " + edits +
- " as " + moveAsideName, e);
- if (!this.fs.rename(edits, moveAsideName)) {
- LOG.error("hbase.skip.errors=true so continuing. Rename failed");
+ long seqid = minSeqId;
+ NavigableSet<Path> files = HLog.getSplitEditFilesSorted(this.fs, regiondir);
+ if (files == null || files.isEmpty()) return seqid;
+ for (Path edits: files) {
+ if (edits == null || !this.fs.exists(edits)) {
+ LOG.warn("Null or non-existent edits file: " + edits);
+ continue;
+ }
+ if (isZeroLengthThenDelete(this.fs, edits)) continue;
+ try {
+ seqid = replayRecoveredEdits(edits, seqid, reporter);
+ } catch (IOException e) {
+ boolean skipErrors = conf.getBoolean("hbase.skip.errors", false);
+ if (skipErrors) {
+ Path p = HLog.moveAsideBadEditsFile(fs, edits);
+ LOG.error("hbase.skip.errors=true so continuing. Renamed " + edits +
+ " as " + p, e);
+ } else {
+ throw e;
+ }
+ }
+ }
+ if (seqid > minSeqId) {
+ // Then we added some edits to memory. Flush and cleanup split edit files.
+ internalFlushcache(null, seqid);
+ for (Path file: files) {
+ if (!this.fs.delete(file, false)) {
+ LOG.error("Failed delete of " + file);
+ } else {
+ LOG.debug("Deleted recovered.edits file=" + file);
}
- } else {
- throw e;
}
}
- return maxSeqIdInLog;
+ return seqid;
}
/*
@@ -1901,12 +1944,13 @@ public class HRegion implements HeapSize
* must be larger than this to be replayed.
* @param reporter
* @return the sequence id of the last edit added to this region out of the
- * recovered edits log, or -1 if no log recovered
+ * recovered edits log or <code>minSeqId</code> if nothing added from editlogs.
* @throws IOException
*/
private long replayRecoveredEdits(final Path edits,
final long minSeqId, final Progressable reporter)
throws IOException {
+ LOG.info("Replaying edits from " + edits + "; minSeqId=" + minSeqId);
HLog.Reader reader = HLog.getReader(this.fs, edits, conf);
try {
return replayRecoveredEdits(reader, minSeqId, reporter);
@@ -1916,26 +1960,22 @@ public class HRegion implements HeapSize
}
/* @param reader Reader against file of recovered edits.
- * @param minSeqId Minimum sequenceid found in a store file. Edits in log
- * must be larger than this to be replayed.
+ * @param minSeqId Any edit found in split editlogs needs to be in excess of
+ * this minSeqId to be applied, else its skipped.
* @param reporter
* @return the sequence id of the last edit added to this region out of the
- * recovered edits log, or -1 if no log recovered
+ * recovered edits log or <code>minSeqId</code> if nothing added from editlogs.
* @throws IOException
*/
private long replayRecoveredEdits(final HLog.Reader reader,
final long minSeqId, final Progressable reporter)
throws IOException {
- long currentEditSeqId = -1;
+ long currentEditSeqId = minSeqId;
long firstSeqIdInLog = -1;
long skippedEdits = 0;
long editsCount = 0;
HLog.Entry entry;
Store store = null;
- // Get map of family name to maximum sequence id. Do it here up front
- // because as we progress, the sequence id can change if we happen to flush
- // The flush ups the seqid for the Store. The new seqid can cause us skip edits.
- Map<byte [], Long> familyToOriginalMaxSeqId = familyToMaxSeqId(this.stores);
// How many edits to apply before we send a progress report.
int interval = this.conf.getInt("hbase.hstore.report.interval.edits", 2000);
while ((entry = reader.next()) != null) {
@@ -1945,12 +1985,13 @@ public class HRegion implements HeapSize
firstSeqIdInLog = key.getLogSeqNum();
}
// Now, figure if we should skip this edit.
- currentEditSeqId = Math.max(currentEditSeqId, key.getLogSeqNum());
- if (key.getLogSeqNum() <= minSeqId) {
+ if (key.getLogSeqNum() <= currentEditSeqId) {
skippedEdits++;
continue;
}
- for (KeyValue kv : val.getKeyValues()) {
+ currentEditSeqId = key.getLogSeqNum();
+ boolean flush = false;
+ for (KeyValue kv: val.getKeyValues()) {
// Check this edit is for me. Also, guard against writing the special
// METACOLUMN info such as HBASE::CACHEFLUSH entries
if (kv.matchingFamily(HLog.METAFAMILY) ||
@@ -1969,16 +2010,13 @@ public class HRegion implements HeapSize
skippedEdits++;
continue;
}
- // The edits' id has to be in excess of the original max seqid of the
- // targeted store.
- long storeMaxSeqId = familyToOriginalMaxSeqId.get(store.getFamily().getName());
- if (currentEditSeqId < storeMaxSeqId) {
- skippedEdits++;
- continue;
- }
- restoreEdit(kv);
+ // Once we are over the limit, restoreEdit will keep returning true to
+ // flush -- but don't flush until we've played all the kvs that make up
+ // the WALEdit.
+ flush = restoreEdit(store, kv);
editsCount++;
}
+ if (flush) internalFlushcache(null, currentEditSeqId);
// Every 'interval' edits, tell the reporter we're making progress.
// Have seen 60k edits taking 3minutes to complete.
@@ -1994,34 +2032,14 @@ public class HRegion implements HeapSize
return currentEditSeqId;
}
- /*
- * @param stores
- * @return Map of family name to maximum sequenceid.
- */
- private Map<byte [], Long> familyToMaxSeqId(final Map<byte [], Store> stores) {
- Map<byte [], Long> map = new TreeMap<byte [], Long>(Bytes.BYTES_COMPARATOR);
- for (Map.Entry<byte [], Store> e: stores.entrySet()) {
- map.put(e.getKey(), e.getValue().getMaxSequenceId());
- }
- return map;
- }
-
- /*
- * @param kv Apply this value to this region.
- * @throws IOException
+ /**
+ * Used by tests
+ * @param s Store to add edit too.
+ * @param kv KeyValue to add.
+ * @return True if we should flush.
*/
- // This method is protected so can be called from tests.
- protected void restoreEdit(final KeyValue kv) throws IOException {
- // This is really expensive to do per edit. Loads of object creation.
- // TODO: Optimization. Apply edits batched by family.
- Map<byte [], List<KeyValue>> map =
- new TreeMap<byte [], List<KeyValue>>(Bytes.BYTES_COMPARATOR);
- map.put(kv.getFamily(), Collections.singletonList(kv));
- if (kv.isDelete()) {
- delete(map, true);
- } else {
- put(map, true);
- }
+ protected boolean restoreEdit(final Store s, final KeyValue kv) {
+ return isFlushSize(this.memstoreSize.addAndGet(s.add(kv)));
}
/*
@@ -3007,14 +3025,11 @@ public class HRegion implements HeapSize
try {
Store store = stores.get(family);
- // TODO call the proper GET API
// Get the old value:
Get get = new Get(row);
get.addColumn(family, qualifier);
- List<KeyValue> results = new ArrayList<KeyValue>();
- NavigableSet<byte[]> qualifiers = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- qualifiers.add(qualifier);
- store.get(get, qualifiers, results);
+
+ List<KeyValue> results = get(get);
if (!results.isEmpty()) {
KeyValue kv = results.get(0);
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Tue Oct 11 01:59:50 2011
@@ -50,6 +50,10 @@ import java.util.concurrent.atomic.Atomi
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -2433,7 +2437,7 @@ public class HRegionServer implements HR
if (message != null) {
System.err.println(message);
}
- System.err.println("Usage: java org.apache.hbase.HRegionServer start|stop");
+ System.err.println("Usage: java org.apache.hbase.HRegionServer start|stop [-D <conf.param=value>]");
System.exit(0);
}
@@ -2467,15 +2471,26 @@ public class HRegionServer implements HR
*/
protected static void doMain(final String [] args,
final Class<? extends HRegionServer> regionServerClass) {
- if (args.length < 1) {
- printUsageAndExit();
- }
Configuration conf = HBaseConfiguration.create();
- // Process command-line args. TODO: Better cmd-line processing
- // (but hopefully something not as painful as cli options).
- for (String cmd: args) {
- if (cmd.equals("start")) {
+ Options opt = new Options();
+ opt.addOption("D", true, "Override HBase Configuration Settings");
+ try {
+ CommandLine cmd = new GnuParser().parse(opt, args);
+
+ if (cmd.hasOption("D")) {
+ for (String confOpt : cmd.getOptionValues("D")) {
+ String[] kv = confOpt.split("=", 2);
+ if (kv.length == 2) {
+ conf.set(kv[0], kv[1]);
+ LOG.debug("-D configuration override: " + kv[0] + "=" + kv[1]);
+ } else {
+ throw new ParseException("-D option format invalid: " + confOpt);
+ }
+ }
+ }
+
+ if (cmd.getArgList().contains("start")) {
try {
// If 'local', don't start a region server here. Defer to
// LocalHBaseCluster. It manages 'local' clusters.
@@ -2493,17 +2508,18 @@ public class HRegionServer implements HR
} catch (Throwable t) {
LOG.error( "Can not start region server because "+
StringUtils.stringifyException(t) );
+ System.exit(-1);
}
- break;
- }
-
- if (cmd.equals("stop")) {
- printUsageAndExit("To shutdown the regionserver run " +
- "bin/hbase-daemon.sh stop regionserver or send a kill signal to" +
- "the regionserver pid");
+ } else if (cmd.getArgList().contains("stop")) {
+ throw new ParseException("To shutdown the regionserver run " +
+ "bin/hbase-daemon.sh stop regionserver or send a kill signal to" +
+ "the regionserver pid");
+ } else {
+ throw new ParseException("Unknown argument(s): " +
+ org.apache.commons.lang.StringUtils.join(cmd.getArgs(), " "));
}
-
- // Print out usage if we get to here.
+ } catch (ParseException e) {
+ LOG.error("Could not parse", e);
printUsageAndExit();
}
}
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java Tue Oct 11 01:59:50 2011
@@ -111,9 +111,16 @@ public class KeyValueHeap implements Key
return false;
}
InternalScanner currentAsInternal = (InternalScanner)this.current;
- currentAsInternal.next(result, limit);
+ boolean mayContainsMoreRows = currentAsInternal.next(result, limit);
KeyValue pee = this.current.peek();
- if (pee == null) {
+ /*
+ * By definition, any InternalScanner must return false only when it has no
+ * further rows to be fetched. So, we can close a scanner if it returns
+ * false. All existing implementations seem to be fine with this. It is much
+ * more efficient to close scanners which are not needed than keep them in
+ * the heap. This is also required for certain optimizations.
+ */
+ if (pee == null || !mayContainsMoreRows) {
this.current.close();
} else {
this.heap.add(this.current);
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java Tue Oct 11 01:59:50 2011
@@ -24,14 +24,11 @@ import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.lang.management.RuntimeMXBean;
import java.rmi.UnexpectedException;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.NavigableSet;
-import java.util.Set;
import java.util.SortedSet;
-import java.util.concurrent.CopyOnWriteArraySet;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -41,7 +38,6 @@ import org.apache.hadoop.hbase.HConstant
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.HeapSize;
-import org.apache.hadoop.hbase.regionserver.DeleteCompare.DeleteCode;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
@@ -351,6 +347,74 @@ public class MemStore implements HeapSiz
}
}
+ /**
+ * Given the specs of a column, update it, first by inserting a new record,
+ * then removing the old one. Since there is only 1 KeyValue involved, the memstoreTS
+ * will be set to 0, thus ensuring that they instantly appear to anyone. The underlying
+ * store will ensure that the insert/delete each are atomic. A scanner/reader will either
+ * get the new value, or the old value and all readers will eventually only see the new
+ * value after the old was removed.
+ *
+ * @param row
+ * @param family
+ * @param qualifier
+ * @param newValue
+ * @param now
+ * @return
+ */
+ public long updateColumnValue(byte[] row,
+ byte[] family,
+ byte[] qualifier,
+ long newValue,
+ long now) {
+ this.lock.readLock().lock();
+ try {
+ // create a new KeyValue with 'now' and a 0 memstoreTS == immediately visible
+ KeyValue newKv = new KeyValue(row, family, qualifier,
+ now,
+ Bytes.toBytes(newValue));
+
+ long addedSize = add(newKv);
+
+ // now find and RM the old one(s) to prevent version explosion:
+ SortedSet<KeyValue> ss = kvset.tailSet(newKv);
+ Iterator<KeyValue> it = ss.iterator();
+ while ( it.hasNext() ) {
+ KeyValue kv = it.next();
+
+ if (kv == newKv) {
+ // ignore the one i just put in (heh)
+ continue;
+ }
+ // if this isnt the row we are interested in, then bail:
+ if (0 != Bytes.compareTo(
+ newKv.getBuffer(), newKv.getRowOffset(), newKv.getRowLength(),
+ kv.getBuffer(), kv.getRowOffset(), kv.getRowLength())) {
+ break; // rows dont match, bail.
+ }
+
+ // if the qualifier matches and it's a put, just RM it out of the kvset.
+ if (0 == Bytes.compareTo(
+ newKv.getBuffer(), newKv.getQualifierOffset(), newKv.getQualifierLength(),
+ kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength())) {
+
+ // to be extra safe we only remove Puts that have a memstoreTS==0
+ if (kv.getType() == KeyValue.Type.Put.getCode() &&
+ kv.getMemstoreTS() == 0) {
+ // false means there was a change, so give us the size.
+ addedSize -= heapSizeChange(kv, false);
+
+ it.remove();
+ }
+ }
+ }
+
+ return addedSize;
+ } finally {
+ this.lock.readLock().unlock();
+ }
+ }
+
/*
* Immutable data structure to hold member found in set and the set it was
* found in. Include set because it is carrying context.
@@ -400,103 +464,6 @@ public class MemStore implements HeapSiz
}
}
- //
- // HBASE-880/1249/1304
- //
-
- /**
- * Perform a single-row Get on the and snapshot, placing results
- * into the specified KV list.
- * <p>
- * This will return true if it is determined that the query is complete
- * and it is not necessary to check any storefiles after this.
- * <p>
- * Otherwise, it will return false and you should continue on.
- * @param matcher Column matcher
- * @param result List to add results to
- * @return true if done with store (early-out), false if not
- */
- public boolean get(QueryMatcher matcher, List<KeyValue> result) {
- this.lock.readLock().lock();
- try {
- if(internalGet(this.kvset, matcher, result) || matcher.isDone()) {
- return true;
- }
- matcher.update();
- return internalGet(this.snapshot, matcher, result) || matcher.isDone();
- } finally {
- this.lock.readLock().unlock();
- }
- }
-
- /**
- * Gets from either the memstore or the snapshop, and returns a code
- * to let you know which is which.
- *
- * @param matcher query matcher
- * @param result puts results here
- * @return 1 == memstore, 2 == snapshot, 0 == none
- */
- int getWithCode(QueryMatcher matcher, List<KeyValue> result) {
- this.lock.readLock().lock();
- try {
- boolean fromMemstore = internalGet(this.kvset, matcher, result);
- if (fromMemstore || matcher.isDone())
- return 1;
-
- matcher.update();
- boolean fromSnapshot = internalGet(this.snapshot, matcher, result);
- if (fromSnapshot || matcher.isDone())
- return 2;
-
- return 0;
- } finally {
- this.lock.readLock().unlock();
- }
- }
-
- /**
- * Small utility functions for use by Store.incrementColumnValue
- * _only_ under the threat of pain and everlasting race conditions.
- */
- void readLockLock() {
- this.lock.readLock().lock();
- }
- void readLockUnlock() {
- this.lock.readLock().unlock();
- }
-
- /**
- *
- * @param set memstore or snapshot
- * @param matcher query matcher
- * @param result list to add results to
- * @return true if done with store (early-out), false if not
- */
- boolean internalGet(final NavigableSet<KeyValue> set,
- final QueryMatcher matcher, final List<KeyValue> result) {
- if(set.isEmpty()) return false;
- // Seek to startKey
- SortedSet<KeyValue> tail = set.tailSet(matcher.getStartKey());
- for (KeyValue kv : tail) {
- QueryMatcher.MatchCode res = matcher.match(kv);
- switch(res) {
- case INCLUDE:
- result.add(kv);
- break;
- case SKIP:
- break;
- case NEXT:
- return false;
- case DONE:
- return true;
- default:
- throw new RuntimeException("Unexpected " + res);
- }
- }
- return false;
- }
-
/**
* Check if this memstore may contain the required keys
* @param scan
@@ -653,7 +620,7 @@ public class MemStore implements HeapSiz
}
public final static long FIXED_OVERHEAD = ClassSize.align(
- ClassSize.OBJECT + (7 * ClassSize.REFERENCE));
+ ClassSize.OBJECT + (9 * ClassSize.REFERENCE));
public final static long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
ClassSize.REENTRANT_LOCK + ClassSize.ATOMIC_LONG +
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ReadWriteConsistencyControl.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ReadWriteConsistencyControl.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ReadWriteConsistencyControl.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ReadWriteConsistencyControl.java Tue Oct 11 01:59:50 2011
@@ -41,19 +41,40 @@ public class ReadWriteConsistencyControl
private static final ThreadLocal<Long> perThreadReadPoint =
new ThreadLocal<Long>();
+ /**
+ * Get this thread's read point. Used primarily by the memstore scanner to
+ * know which values to skip (ie: have not been completed/committed to
+ * memstore).
+ */
public static long getThreadReadPoint() {
return perThreadReadPoint.get();
}
+ /**
+ * Set the thread read point to the given value. The thread RWCC
+ * is used by the Memstore scanner so it knows which values to skip.
+ * Give it a value of 0 if you want everything.
+ */
public static void setThreadReadPoint(long readPoint) {
perThreadReadPoint.set(readPoint);
}
+ /**
+ * Set the thread RWCC read point to whatever the current read point is in
+ * this particular instance of RWCC. Returns the new thread read point value.
+ */
public static long resetThreadReadPoint(ReadWriteConsistencyControl rwcc) {
perThreadReadPoint.set(rwcc.memstoreReadPoint());
return getThreadReadPoint();
}
+ /**
+ * Set the thread RWCC read point to 0 (include everything).
+ */
+ public static void resetThreadReadPoint() {
+ perThreadReadPoint.set(0L);
+ }
+
public WriteEntry beginMemstoreInsert() {
synchronized (writeQueue) {
long nextWriteNumber = ++memstoreWrite;
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java Tue Oct 11 01:59:50 2011
@@ -20,9 +20,12 @@
package org.apache.hadoop.hbase.regionserver;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
+import org.apache.hadoop.hbase.io.TimeRange;
import org.apache.hadoop.hbase.util.Bytes;
import java.util.NavigableSet;
@@ -30,13 +33,36 @@ import java.util.NavigableSet;
/**
* A query matcher that is specifically designed for the scan case.
*/
-public class ScanQueryMatcher extends QueryMatcher {
+public class ScanQueryMatcher {
// Optimization so we can skip lots of compares when we decide to skip
// to the next row.
private boolean stickyNextRow;
+ private byte[] stopRow;
+
+ protected TimeRange tr;
+
+ protected Filter filter;
+
+ /** Keeps track of deletes */
+ protected DeleteTracker deletes;
+
+ /** Keeps track of columns and versions */
+ protected ColumnTracker columns;
+
+ /** Key to seek to in memstore and StoreFiles */
+ protected KeyValue startKey;
+
+ /** Oldest allowed version stamp for TTL enforcement */
+ protected long oldestStamp;
+
+ /** Row comparator for the region this query is for */
+ KeyValue.KeyComparator rowComparator;
+
+ /** Row the query is on */
+ protected byte [] row;
/**
- * Constructs a QueryMatcher for a Scan.
+ * Constructs a ScanQueryMatcher for a Scan.
* @param scan
* @param family
* @param columns
@@ -50,6 +76,7 @@ public class ScanQueryMatcher extends Qu
this.oldestStamp = System.currentTimeMillis() - ttl;
this.rowComparator = rowComparator;
this.deletes = new ScanDeleteTracker();
+ this.stopRow = scan.getStopRow();
this.startKey = KeyValue.createFirstOnRow(scan.getStartRow());
this.filter = scan.getFilter();
@@ -140,17 +167,37 @@ public class ScanQueryMatcher extends Qu
return MatchCode.SKIP;
}
- if (!tr.withinTimeRange(timestamp)) {
+ if (!this.deletes.isEmpty() &&
+ deletes.isDeleted(bytes, offset, qualLength, timestamp)) {
return MatchCode.SKIP;
}
- if (!this.deletes.isEmpty() &&
- deletes.isDeleted(bytes, offset, qualLength, timestamp)) {
+ int timestampComparison = tr.compare(timestamp);
+ if (timestampComparison >= 1) {
return MatchCode.SKIP;
+ } else if (timestampComparison <= -1) {
+ return getNextRowOrNextColumn(bytes, offset, qualLength);
}
- MatchCode colChecker = columns.checkColumn(bytes, offset, qualLength);
+ /**
+ * Filters should be checked before checking column trackers. If we do
+ * otherwise, as was previously being done, ColumnTracker may increment its
+ * counter for even that KV which may be discarded later on by Filter. This
+ * would lead to incorrect results in certain cases.
+ */
+ if (filter != null) {
+ ReturnCode filterResponse = filter.filterKeyValue(kv);
+ if (filterResponse == ReturnCode.SKIP) {
+ return MatchCode.SKIP;
+ } else if (filterResponse == ReturnCode.NEXT_COL) {
+ return getNextRowOrNextColumn(bytes, offset, qualLength);
+ } else if (filterResponse == ReturnCode.NEXT_ROW) {
+ stickyNextRow = true;
+ return MatchCode.SEEK_NEXT_ROW;
+ }
+ }
+ MatchCode colChecker = columns.checkColumn(bytes, offset, qualLength);
// if SKIP -> SEEK_NEXT_COL
// if (NEXT,DONE) -> SEEK_NEXT_ROW
// if (INCLUDE) -> INCLUDE
@@ -161,39 +208,114 @@ public class ScanQueryMatcher extends Qu
return MatchCode.SEEK_NEXT_ROW;
}
- // else INCLUDE
- // if (colChecker == MatchCode.INCLUDE)
- // give the filter a chance to run.
- if (filter == null)
- return MatchCode.INCLUDE;
-
- ReturnCode filterResponse = filter.filterKeyValue(kv);
- if (filterResponse == ReturnCode.INCLUDE)
- return MatchCode.INCLUDE;
+ return MatchCode.INCLUDE;
- if (filterResponse == ReturnCode.SKIP)
- return MatchCode.SKIP;
- else if (filterResponse == ReturnCode.NEXT_COL)
+ }
+
+ public MatchCode getNextRowOrNextColumn(byte[] bytes, int offset,
+ int qualLength) {
+ if (columns instanceof ExplicitColumnTracker) {
+ //We only come here when we know that columns is an instance of
+ //ExplicitColumnTracker so we should never have a cast exception
+ ((ExplicitColumnTracker)columns).doneWithColumn(bytes, offset,
+ qualLength);
+ if (columns.getColumnHint() == null) {
+ return MatchCode.SEEK_NEXT_ROW;
+ } else {
+ return MatchCode.SEEK_NEXT_COL;
+ }
+ } else {
return MatchCode.SEEK_NEXT_COL;
- // else if (filterResponse == ReturnCode.NEXT_ROW)
+ }
+ }
- stickyNextRow = true;
- return MatchCode.SEEK_NEXT_ROW;
+ public boolean moreRowsMayExistAfter(KeyValue kv) {
+ if (!Bytes.equals(stopRow , HConstants.EMPTY_END_ROW) &&
+ rowComparator.compareRows(kv.getBuffer(),kv.getRowOffset(),
+ kv.getRowLength(), stopRow, 0, stopRow.length) >= 0) {
+ return false;
+ } else {
+ return true;
+ }
}
/**
* Set current row
* @param row
*/
- @Override
public void setRow(byte [] row) {
this.row = row;
reset();
}
- @Override
public void reset() {
- super.reset();
+ this.deletes.reset();
+ this.columns.reset();
+
stickyNextRow = false;
}
+
+ // should be in KeyValue.
+ protected boolean isDelete(byte type) {
+ return (type != KeyValue.Type.Put.getCode());
+ }
+
+ protected boolean isExpired(long timestamp) {
+ return (timestamp < oldestStamp);
+ }
+
+ /**
+ *
+ * @return the start key
+ */
+ public KeyValue getStartKey() {
+ return this.startKey;
+ }
+
+ /**
+ * {@link #match} return codes. These instruct the scanner moving through
+ * memstores and StoreFiles what to do with the current KeyValue.
+ * <p>
+ * Additionally, this contains "early-out" language to tell the scanner to
+ * move on to the next File (memstore or Storefile), or to return immediately.
+ */
+ public static enum MatchCode {
+ /**
+ * Include KeyValue in the returned result
+ */
+ INCLUDE,
+
+ /**
+ * Do not include KeyValue in the returned result
+ */
+ SKIP,
+
+ /**
+ * Do not include, jump to next StoreFile or memstore (in time order)
+ */
+ NEXT,
+
+ /**
+ * Do not include, return current result
+ */
+ DONE,
+
+ /**
+ * These codes are used by the ScanQueryMatcher
+ */
+
+ /**
+ * Done with the row, seek there.
+ */
+ SEEK_NEXT_ROW,
+ /**
+ * Done with column, seek to next.
+ */
+ SEEK_NEXT_COL,
+
+ /**
+ * Done with scan, thanks to the row filter.
+ */
+ DONE_SCAN,
+ }
}
\ No newline at end of file
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java Tue Oct 11 01:59:50 2011
@@ -22,7 +22,7 @@ package org.apache.hadoop.hbase.regionse
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.regionserver.QueryMatcher.MatchCode;
+import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode;
import org.apache.hadoop.hbase.util.Bytes;
/**
@@ -65,15 +65,15 @@ public class ScanWildcardColumnTracker i
currentCount = 0;
if (++currentCount > maxVersions)
- return MatchCode.SKIP;
- return MatchCode.INCLUDE;
+ return ScanQueryMatcher.MatchCode.SKIP;
+ return ScanQueryMatcher.MatchCode.INCLUDE;
}
int cmp = Bytes.compareTo(bytes, offset, length,
columnBuffer, columnOffset, columnLength);
if (cmp == 0) {
if (++currentCount > maxVersions)
- return MatchCode.SKIP; // skip to next col
- return MatchCode.INCLUDE;
+ return ScanQueryMatcher.MatchCode.SKIP; // skip to next col
+ return ScanQueryMatcher.MatchCode.INCLUDE;
}
// new col > old col
@@ -84,8 +84,8 @@ public class ScanWildcardColumnTracker i
columnLength = length;
currentCount = 0;
if (++currentCount > maxVersions)
- return MatchCode.SKIP;
- return MatchCode.INCLUDE;
+ return ScanQueryMatcher.MatchCode.SKIP;
+ return ScanQueryMatcher.MatchCode.INCLUDE;
}
// new col < oldcol
@@ -102,8 +102,8 @@ public class ScanWildcardColumnTracker i
columnLength = length;
currentCount = 0;
if (++currentCount > maxVersions)
- return MatchCode.SKIP;
- return MatchCode.INCLUDE;
+ return ScanQueryMatcher.MatchCode.SKIP;
+ return ScanQueryMatcher.MatchCode.INCLUDE;
}
@Override
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Tue Oct 11 01:59:50 2011
@@ -190,7 +190,7 @@ public class Store implements HeapSize {
this.storefiles = ImmutableList.copyOf(loadStoreFiles());
}
- HColumnDescriptor getFamily() {
+ public HColumnDescriptor getFamily() {
return this.family;
}
@@ -958,13 +958,6 @@ public class Store implements HeapSize {
return wantedVersions > maxVersions ? maxVersions: wantedVersions;
}
- static void expiredOrDeleted(final Set<KeyValue> set, final KeyValue kv) {
- boolean b = set.remove(kv);
- if (LOG.isDebugEnabled()) {
- LOG.debug(kv.toString() + " expired: " + b);
- }
- }
-
static boolean isExpired(final KeyValue key, final long oldestTimestamp) {
return key.getTimestamp() < oldestTimestamp;
}
@@ -1204,7 +1197,7 @@ public class Store implements HeapSize {
* Return a scanner for both the memstore and the HStore files
* @throws IOException
*/
- protected KeyValueScanner getScanner(Scan scan,
+ public KeyValueScanner getScanner(Scan scan,
final NavigableSet<byte []> targetCols) throws IOException {
lock.readLock().lock();
try {
@@ -1289,85 +1282,6 @@ public class Store implements HeapSize {
}
/**
- * Convenience method that implements the old MapFile.getClosest on top of
- * HFile Scanners. getClosest used seek to the asked-for key or just after
- * (HFile seeks to the key or just before).
- * @param s Scanner to use
- * @param kv Key to find.
- * @return True if we were able to seek the scanner to <code>b</code> or to
- * the key just after.
- * @throws IOException
- */
- static boolean getClosest(final HFileScanner s, final KeyValue kv)
- throws IOException {
- // Pass offsets to key content of a KeyValue; thats whats in the hfile index.
- int result = s.seekTo(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength());
- if (result < 0) {
- // Not in file. Will the first key do?
- if (!s.seekTo()) {
- return false;
- }
- } else if (result > 0) {
- // Less than what was asked for but maybe < because we're asking for
- // r/c/HConstants.LATEST_TIMESTAMP -- what was returned was r/c-1/SOME_TS...
- // A next will get us a r/c/SOME_TS.
- if (!s.next()) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Retrieve results from this store given the specified Get parameters.
- * @param get Get operation
- * @param columns List of columns to match, can be empty (not null)
- * @param result List to add results to
- * @throws IOException
- */
- public void get(Get get, NavigableSet<byte[]> columns, List<KeyValue> result)
- throws IOException {
- KeyComparator keyComparator = this.comparator.getRawComparator();
-
- // Column matching and version enforcement
- QueryMatcher matcher = new QueryMatcher(get, this.family.getName(), columns,
- this.ttl, keyComparator, versionsToReturn(get.getMaxVersions()));
- this.lock.readLock().lock();
- try {
- // Read from memstore
- if(this.memstore.get(matcher, result)) {
- // Received early-out from memstore
- return;
- }
-
- // Check if we even have storefiles
- if (this.storefiles.isEmpty()) {
- return;
- }
-
- // Get storefiles for this store
- List<HFileScanner> storefileScanners = new ArrayList<HFileScanner>();
- for (StoreFile sf : Iterables.reverse(this.storefiles)) {
- StoreFile.Reader r = sf.getReader();
- if (r == null) {
- LOG.warn("StoreFile " + sf + " has a null Reader");
- continue;
- }
- // Get a scanner that caches the block and uses pread
- storefileScanners.add(r.getScanner(true, true));
- }
-
- // StoreFileGetScan will handle reading this store's storefiles
- StoreFileGetScan scanner = new StoreFileGetScan(storefileScanners, matcher);
-
- // Run a GET scan and put results into the specified list
- scanner.get(result);
- } finally {
- this.lock.readLock().unlock();
- }
- }
-
- /**
* Increments the value for the given row/family/qualifier.
*
* This function will always be seen as atomic by other readers
@@ -1384,49 +1298,18 @@ public class Store implements HeapSize {
public long updateColumnValue(byte [] row, byte [] f,
byte [] qualifier, long newValue)
throws IOException {
- List<KeyValue> result = new ArrayList<KeyValue>();
- KeyComparator keyComparator = this.comparator.getRawComparator();
-
- KeyValue kv = null;
- // Setting up the QueryMatcher
- Get get = new Get(row);
- NavigableSet<byte[]> qualifiers =
- new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- qualifiers.add(qualifier);
- QueryMatcher matcher = new QueryMatcher(get, f, qualifiers, this.ttl,
- keyComparator, 1);
- // lock memstore snapshot for this critical section:
this.lock.readLock().lock();
- memstore.readLockLock();
try {
- int memstoreCode = this.memstore.getWithCode(matcher, result);
+ long now = System.currentTimeMillis();
+
+ return this.memstore.updateColumnValue(row,
+ f,
+ qualifier,
+ newValue,
+ now);
- if (memstoreCode != 0) {
- // was in memstore (or snapshot)
- kv = result.get(0).clone();
- byte [] buffer = kv.getBuffer();
- int valueOffset = kv.getValueOffset();
- Bytes.putBytes(buffer, valueOffset, Bytes.toBytes(newValue), 0,
- Bytes.SIZEOF_LONG);
- if (memstoreCode == 2) {
- // from snapshot, assign new TS
- long currTs = System.currentTimeMillis();
- if (currTs == kv.getTimestamp()) {
- currTs++; // unlikely but catastrophic
- }
- Bytes.putBytes(buffer, kv.getTimestampOffset(),
- Bytes.toBytes(currTs), 0, Bytes.SIZEOF_LONG);
- }
- } else {
- kv = new KeyValue(row, f, qualifier,
- System.currentTimeMillis(),
- Bytes.toBytes(newValue));
- }
- return add(kv);
- // end lock
} finally {
- memstore.readLockUnlock();
this.lock.readLock().unlock();
}
}
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java Tue Oct 11 01:59:50 2011
@@ -231,7 +231,7 @@ class StoreScanner implements KeyValueSc
KeyValue kv;
List<KeyValue> results = new ArrayList<KeyValue>();
LOOP: while((kv = this.heap.peek()) != null) {
- QueryMatcher.MatchCode qcode = matcher.match(kv);
+ ScanQueryMatcher.MatchCode qcode = matcher.match(kv);
//DebugPrint.println("SS peek kv = " + kv + " with qcode = " + qcode);
switch(qcode) {
case INCLUDE:
@@ -256,6 +256,10 @@ class StoreScanner implements KeyValueSc
return false;
case SEEK_NEXT_ROW:
+ if (!matcher.moreRowsMayExistAfter(kv)) {
+ outResult.addAll(results);
+ return false;
+ }
heap.next();
break;
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java?rev=1181336&r1=1181335&r2=1181336&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java Tue Oct 11 01:59:50 2011
@@ -96,7 +96,7 @@ public class TimeRangeTracker implements
* @param timestamp the timestamp value to include
*/
private void includeTimestamp(final long timestamp) {
- if (maximumTimestamp==-1) {
+ if (maximumTimestamp == -1) {
minimumTimestamp = timestamp;
maximumTimestamp = timestamp;
}