You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2009/11/25 23:30:39 UTC
svn commit: r884310 [3/7] - in /hadoop/hbase/branches/0.20_on_hadoop-0.18.3:
./ bin/ conf/ lib/ src/contrib/ src/contrib/ec2/ src/contrib/ec2/bin/
src/contrib/ec2/bin/image/
src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/tableindexed...
Added: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/docs/src/documentation/content/xdocs/cygwin.xml
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/docs/src/documentation/content/xdocs/cygwin.xml?rev=884310&view=auto
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/docs/src/documentation/content/xdocs/cygwin.xml (added)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/docs/src/documentation/content/xdocs/cygwin.xml Wed Nov 25 22:30:29 2009
@@ -0,0 +1,254 @@
+<?xml version="1.0"?>
+<!--
+ Copyright 2002-2008 The Apache Software Foundation
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN"
+ "http://forrest.apache.org/dtd/document-v20.dtd">
+
+
+<document>
+ <header>
+ <title>Installing HBase on Windows using Cygwin</title>
+ </header>
+
+<body>
+<section>
+<title>Introduction</title>
+<p><a title="HBase project" href="http://hadoop.apache.org/hbase" target="_blank">HBase</a> is a distributed, column-oriented store, modeled after Google's <a title="Google's BigTable" href="http://labs.google.com/papers/bigtable.html" target="_blank">BigTable</a>. HBase is built on top of <a title="Hadoop project" href="http://hadoop.apache.org">Hadoop</a> for its <a title="Hadoop MapReduce project" href="http://hadoop.apache.org/mapreduce" target="_blank">MapReduce </a>and <a title="Hadoop DFS project" href="http://hadoop.apache.org/hdfs">distributed file system</a> implementation. All these projects are open-source and part of the <a title="The Apache Software Foundation" href="http://www.apache.org/" target="_blank">Apache Software Foundation</a>.</p>
+
+<p style="text-align: justify; ">As being distributed, large scale platforms, the Hadoop and HBase projects mainly focus on <em><strong>*nix</strong></em><strong> environments</strong> for production installations. However, being developed in <strong>Java</strong>, both projects are fully <strong>portable</strong> across platforms and, hence, also to the <strong>Windows operating system</strong>. For ease of development the projects rely on <a title="Cygwin site" href="http://www.cygwin.com/" target="_blank">Cygwin</a> to have a *nix-like environment on Windows to run the shell scripts.</p>
+</section>
+<section>
+<title>Purpose</title>
+<p style="text-align: justify; ">This document explains the <strong>intricacies of running HBase on Windows using Cygwin</strong> as an all-in-one single-node installation for testing and development. The HBase <a title="HBase Overview" href="http://hadoop.apache.org/hbase/docs/current/api/overview-summary.html#overview_description" target="_blank">Overview</a> and <a title="HBase QuickStart" href="http://hadoop.apache.org/common/docs/current/quickstart.html" target="_blank">QuickStart</a> guides on the other hand go a long way in explaning how to setup <a title="HBase project" href="http://hadoop.apache.org/hbase" target="_blank">HBase</a> in more complex deployment scenario's.</p>
+</section>
+
+<section>
+<title >Installation</title>
+<p style="text-align: justify; ">For running HBase on Windows, 3 technologies are required: <strong>Java, Cygwin and SSH</strong>. The following paragraphs detail the installation of each of the aforementioned technologies.</p>
+<section>
+<title>Java</title>
+<p style="text-align: justify; ">HBase depends on the <a title="Java Platform, Standard Edition, 6 Release" href="http://java.sun.com/javase/6/" target="_blank">Java Platform, Standard Edition, 6 Release</a>. So the target system has to be provided with at least the Java Runtime Environment (JRE); however if the system will also be used for development, the Jave Development Kit (JDK) is preferred. You can download the latest versions for both from <a title="Java SE Downloads" href="http://java.sun.com/javase/downloads/index.jsp" target="_blank">Sun's download page</a>. Installation is a simple GUI wizard that guides you through the process.</p>
+</section>
+<section>
+<title>Cygwin</title>
+<p style="text-align: justify; ">Cygwin is probably the oddest technology in this solution stack. It provides a dynamic link library that emulates most of a *nix environment on Windows. On top of that a whole bunch of the most common *nix tools are supplied. Combined, the DLL with the tools form a very *nix-alike environment on Windows.</p>
+
+<p style="text-align: justify; ">For installation, Cygwin provides the <a title="Cygwin Setup Utility" href="http://cygwin.com/setup.exe" target="_blank"><strong><code>setup.exe</code> utility</strong></a> that tracks the versions of all installed components on the target system and provides the mechanism for <strong>installing</strong> or <strong>updating </strong>everything from the mirror sites of Cygwin.</p>
+
+<p style="text-align: justify; ">To support installation, the <code>setup.exe</code> utility uses 2 directories on the target system. The <strong>Root</strong> directory for Cygwin (defaults to <code>C:\cygwin)</code> which will become <code>/</code> within the eventual Cygwin installation; and the <strong>Local Package </strong>directory (e.g. <code>C:\cygsetup</code> that is the cache where <code>setup.exe</code> stores the packages before they are installed. The cache must not be the same folder as the Cygwin root.</p>
+
+<p style="text-align: justify; ">Perform following steps to install Cygwin, which are elaboratly detailed in the <a title="Setting Up Cygwin" href="http://cygwin.com/cygwin-ug-net/setup-net.html" target="_self">2nd chapter</a> of the <a title="Cygwin User's Guide" href="http://cygwin.com/cygwin-ug-net/cygwin-ug-net.html" target="_blank">Cygwin User's Guide</a>:</p>
+
+<ol style="text-align: justify; ">
+ <li>Make sure you have <code>Administrator</code> privileges on the target system.</li>
+ <li>Choose and create you <strong>Root</strong> and <strong>Local Package</strong> directories. A good suggestion is to use <code>C:\cygwin\root</code> and <code>C:\cygwin\setup</code> folders.</li>
+ <li>Download the <code>setup.exe</code> utility and save it to the <strong>Local Package</strong> directory.</li>
+ <li>Run the <code>setup.exe</code> utility,
+<ol>
+ <li>Choose the <code>Install from Internet</code> option,</li>
+ <li>Choose your <strong>Root</strong> and <strong>Local Package</strong> folders</li>
+ <li>and select an appropriate mirror.</li>
+ <li>Don't select any additional packages yet, as we only want to install Cygwin for now.</li>
+ <li>Wait for download and install</li>
+ <li>Finish the installation</li>
+</ol>
+</li>
+ <li>Optionally, you can now also add a shortcut to your Start menu pointing to the <code>setup.exe</code> utility in the <strong>Local Package </strong>folder.</li>
+ <li>Add <code>CYGWIN_HOME</code> system-wide environment variable that points to your <strong>Root </strong>directory.</li>
+ <li>Add <code>%CYGWIN_HOME%\bin</code> to the end of your <code>PATH</code> environment variable.</li>
+ <li>Reboot the sytem after making changes to the environment variables otherwise the OS will not be able to find the Cygwin utilities.</li>
+ <li>Test your installation by running your freshly created shortcuts or the <code>Cygwin.bat</code> command in the <strong>Root</strong> folder. You should end up in a terminal window that is running a <a title="Bash Reference Manual" href="http://www.gnu.org/software/bash/manual/bashref.html" target="_blank">Bash shell</a>. Test the shell by issuing following commands:
+<ol>
+ <li><code>cd /</code> should take you to thr <strong>Root</strong> directory in Cygwin;</li>
+ <li>the <code>LS</code> commands that should list all files and folders in the current directory.</li>
+ <li>Use the <code>exit</code> command to end the terminal.</li>
+</ol>
+</li>
+ <li>When needed, to <strong>uninstall</strong> Cygwin you can simply delete the <strong>Root</strong> and <strong>Local Package</strong> directory, and the <strong>shortcuts</strong> that were created during installation.</li>
+</ol>
+</section>
+<section><title>SSH</title>
+<p style="text-align: justify; ">HBase (and Hadoop) rely on <a title="Secure Shell" href="http://nl.wikipedia.org/wiki/Secure_Shell" target="_blank"><strong>SSH</strong></a> for interprocess/-node <strong>communication</strong> and launching<strong> remote commands</strong>. SSH will be provisioned on the target system via Cygwin, which supports running Cygwin programs as <strong>Windows services</strong>!</p>
+
+<ol style="text-align: justify; ">
+ <li>Rerun the <code><strong>setup.exe</strong></code><strong> utility</strong>.</li>
+ <li>Leave all parameters as is, skipping through the wizard using the <code>Next</code> button until the <code>Select Packages</code> panel is shown.</li>
+ <li>Maximize the window and click the <code>View</code> button to toggle to the list view, which is ordered alfabetically on <code>Package</code>, making it easier to find the packages we'll need.</li>
+ <li>Select the following packages by clicking the status word (normally <code>Skip</code>) so it's marked for installation. Use the <code>Next </code>button to download and install the packages.
+<ol>
+ <li>OpenSSH</li>
+ <li>tcp_wrappers</li>
+ <li>diffutils</li>
+ <li>zlib</li>
+</ol>
+</li>
+ <li>Wait for the install to complete and finish the installation.</li>
+</ol>
+</section>
+<section>
+<title>HBase</title>
+<p style="text-align: justify; ">Download the <strong>latest release </strong>of HBase from the <a title="HBase Releases" href="http://hadoop.apache.org/hbase/releases.html" target="_blank">website</a>. As the HBase distributable is just a zipped archive, installation is as simple as unpacking the archive so it ends up in its final <strong>installation</strong> directory. Notice that HBase has to be installed in Cygwin and a good directory suggestion is to use <code>/usr/local/</code> (or [<code><strong>Root</strong> directory]\usr\local</code> in Windows slang). You should end up with a <code>/usr/local/hbase-<em><version></em></code> installation in Cygwin.</p>
+
+This finishes installation. We go on with the configuration.
+</section>
+</section>
+<section>
+<title>Configuration</title>
+<p style="text-align: justify; ">There are 3 parts left to configure: <strong>Java, SSH and HBase</strong> itself. Following paragraphs explain eacht topic in detail.</p>
+<section>
+<title>Java</title>
+<p style="text-align: justify; ">One important thing to remember in shell scripting in general (i.e. *nix and Windows) is that managing, manipulating and assembling path names that contains spaces can be very hard, due to the need to escape and quote those characters and strings. So we try to stay away from spaces in path names. *nix environments can help us out here very easily by using <strong>symbolic links</strong>.</p>
+
+<ol style="text-align: justify; ">
+ <li style="text-align: justify; ">Create a link in <code>/usr/local</code> to the Java home directory by using the following command and substituting the name of your chosen Java environment:
+<pre>LN -s /cygdrive/c/Program\ Files/Java/<em><jre name> </em>/usr/local/<em><jre name></em></pre>
+</li>
+ <li>Test your java installation by changing directories to your Java folder <code>CD /usr/local/<em><jre name></em></code> and issueing the command <code>./bin/java -version</code>. This should output your version of the chosen JRE.</li>
+</ol>
+</section>
+<section>
+<title>SSH</title>
+<p style="text-align: justify; ">Configuring <strong>SSH </strong>is quite elaborate, but primarily a question of launching it by default as a<strong> Windows service</strong>.</p>
+
+<ol style="text-align: justify; ">
+ <li style="text-align: justify; ">On Windows Vista and above make sure you run the Cygwin shell with <strong>elevated privileges</strong>, by right-clicking on the shortcut an using <code>Run as Administrator</code>.</li>
+ <li style="text-align: justify; ">First of all, we have to make sure the <strong>rights on some crucial files</strong> are correct. Use the commands underneath. You can verify all rights by using the <code>LS -L</code> command on the different files. Also, notice the auto-completion feature in the shell using <code><TAB></code> is extremely handy in these situations.
+<ol>
+ <li><code>chmod +r /etc/passwd</code> to make the passwords file readable for all</li>
+ <li><code>chmod u+w /etc/passwd</code> to make the passwords file writable for the owner</li>
+ <li><code>chmod +r /etc/group</code> to make the groups file readable for all</li>
+</ol>
+<ol>
+ <li><code>chmod u+w /etc/group</code> to make the groups file writable for the owner</li>
+</ol>
+<ol>
+ <li><code>chmod 755 /var</code> to make the var folder writable to owner and readable and executable to all</li>
+</ol>
+</li>
+ <li>Edit the <strong>/etc/hosts.allow</strong> file using your favorite editor (why not VI in the shell!) and make sure the following two lines are in there before the <code>PARANOID</code> line:
+<ol>
+ <li><code>ALL : localhost 127.0.0.1/32 : allow</code></li>
+ <li><code>ALL : [::1]/128 : allow</code></li>
+</ol>
+</li>
+ <li>Next we have to <strong>configure SSH</strong> by using the script <code>ssh-host-config</code>
+<ol>
+ <li>If this script asks to overwrite an existing <code>/etc/ssh_config</code>, answer <code>yes</code>.</li>
+ <li>If this script asks to overwrite an existing <code>/etc/sshd_config</code>, answer <code>yes</code>.</li>
+ <li>If this script asks to use privilege separation, answer <code>yes</code>.</li>
+ <li>If this script asks to install <code>sshd</code> as a service, answer <code>yes</code>. Make sure you started your shell as Adminstrator!</li>
+ <li>If this script asks for the CYGWIN value, just <code><enter></code> as the default is <code>ntsec</code>.</li>
+ <li>If this script asks to create the <code>sshd</code> account, answer <code>yes</code>.</li>
+ <li>If this script asks to use a different user name as service account, answer <code>no</code> as the default will suffice.</li>
+ <li>If this script asks to create the <code>cyg_server</code> account, answer <code>yes</code>. Enter a password for the account.</li>
+</ol>
+</li>
+ <li><strong>Start the SSH service</strong> using <code>net start sshd</code> or <code>cygrunsrv --start sshd</code>. Notice that <code>cygrunsrv</code> is the utility that make the process run as a Windows service. Confirm that you see a message stating that <code>the CYGWIN sshd service was started succesfully.</code></li>
+ <li>Harmonize Windows and Cygwin<strong> user account</strong> by using the commands:
+<ol>
+ <li><code>mkpasswd -cl > /etc/passwd</code></li>
+ <li><code>mkgroup --local > /etc/group</code></li>
+</ol>
+</li>
+ <li><strong>Test </strong>the installation of SSH:
+<ol>
+ <li>Open a new Cygwin terminal</li>
+ <li>Use the command <code>whoami</code> to verify your userID</li>
+ <li>Issue an <code>ssh localhost</code> to connect to the system itself
+<ol>
+ <li>Answer <code>yes</code> when presented with the server's fingerprint</li>
+ <li>Issue your password when prompted</li>
+ <li>test a few commands in the remote session</li>
+ <li>The <code>exit</code> command should take you back to your first shell in Cygwin</li>
+</ol>
+</li>
+ <li><code>Exit</code> should terminate the Cygwin shell.</li>
+</ol>
+</li>
+</ol>
+</section>
+<section>
+<title>HBase</title>
+If all previous configurations are working properly, we just need some tinkering at the <strong>HBase config</strong> files to properly resolve on Windows/Cygwin. All files and paths referenced here start from the HBase <code>[<strong>installation</strong> directory]</code> as working directory.
+<ol>
+ <li>HBase uses the <code>./conf/<strong>hbase-env.sh</strong></code> to configure its dependencies on the runtime environment. Copy and uncomment following lines just underneath their original, change them to fit your environemnt. They should read something like:
+<ol>
+ <li><code>export JAVA_HOME=/usr/local/<em><jre name></em></code></li>
+ <li><code>export HBASE_IDENT_STRING=$HOSTNAME</code> as this most likely does not inlcude spaces.</li>
+</ol>
+</li>
+ <li>HBase uses the ./conf/<code><strong>hbase-default.xml</strong></code> file for configuration. Some properties do not resolve to existing directories because the JVM runs on Windows. This is the major issue to keep in mind when working with Cygwin: within the shell all paths are *nix-alike, hence relative to the root <code>/</code>. However, every parameter that is to be consumed within the windows processes themself, need to be Windows settings, hence <code>C:\</code>-alike. Change following propeties in the configuration file, adjusting paths where necessary to conform with your own installation:
+<ol>
+ <li><code>hbase.rootdir</code> must read e.g. <code>file:///C:/cygwin/root/tmp/hbase/data</code></li>
+ <li><code>hbase.tmp.dir</code> must read <code>C:/cygwin/root/tmp/hbase/tmp</code></li>
+ <li><code>hbase.zookeeper.quorum</code> must read <code>127.0.0.1</code> because for some reason <code>localhost</code> doesn't seem to resolve properly on Cygwin.</li>
+</ol>
+</li>
+ <li>Make sure the configured <code>hbase.rootdir</code> and <code>hbase.tmp.dir</code> <strong>directories exist</strong> and have the proper<strong> rights</strong> set up e.g. by issuing a <code>chmod 777</code> on them.</li>
+</ol>
+</section>
+</section>
+<section>
+<title>Testing</title>
+<p>
+This should conclude the installation and configuration of HBase on Windows using Cygwin. So it's time <strong>to test it</strong>.
+<ol>
+ <li>Start a Cygwin<strong> terminal</strong>, if you haven't already.</li>
+ <li>Change directory to HBase <strong>installation</strong> using <code>CD /usr/local/hbase-<em><version></em></code>, preferably using auto-completion.</li>
+ <li><strong>Start HBase</strong> using the command <code>./bin/start-hbase.sh</code>
+<ol>
+ <li>When prompted to accept the SSH fingerprint, answer <code>yes</code>.</li>
+ <li>When prompted, provide your password. Maybe multiple times.</li>
+ <li>When the command completes, the HBase server should have started.</li>
+ <li>However, to be absolutely certain, check the logs in the <code>./logs</code> directory for any exceptions.</li>
+</ol>
+</li>
+ <li>Next we <strong>start the HBase shell</strong> using the command <code>./bin/hbase shell</code></li>
+ <li>We run some simple <strong>test commands</strong>
+<ol>
+ <li>Create a simple table using command <code>create 'test', 'data'</code></li>
+ <li>Verify the table exists using the command <code>list</code></li>
+ <li>Insert data into the table using e.g.
+<pre>put 'test', 'row1', 'data:1', 'value1'
+put 'test', 'row2', 'data:2', 'value2'
+put 'test', 'row3', 'data:3', 'value3'</pre>
+</li>
+ <li>List all rows in the table using the command <code>scan 'test'</code> that should list all the rows previously inserted. Notice how 3 new columns where added without changing the schema!</li>
+ <li>Finally we get rid of the table by issuing <code>disable 'test'</code> followed by <code>drop 'test'</code> and verified by <code>list</code> which should give an empty listing.</li>
+</ol>
+</li>
+ <li><strong>Leave the shell</strong> by <code>exit</code></li>
+ <li>To <strong>stop the HBase server</strong> issue the <code>./bin/stop-hbase.sh</code> command. And wait for it to complete!!! Killing the process might corrupt your data on disk.</li>
+ <li>In case of <strong>problems</strong>,
+<ol>
+ <li>verify the HBase logs in the <code>./logs</code> directory.</li>
+ <li>Try to fix the problem</li>
+ <li>Get help on the forums or IRC (<code>#hbase@freenode.net</code>). People are very active and keen to help out!</li>
+ <li>Stopr, restart and retest the server.</li>
+</ol>
+</li>
+</ol>
+</p>
+</section>
+
+<section>
+<title>Conclusion</title>
+<p>
+Now your <strong>HBase </strong>server is running, <strong>start coding</strong> and build that next killer app on this particular, but scalable datastore!
+</p>
+</section>
+</body>
+</document>
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/docs/src/documentation/content/xdocs/metrics.xml
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/docs/src/documentation/content/xdocs/metrics.xml?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/docs/src/documentation/content/xdocs/metrics.xml (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/docs/src/documentation/content/xdocs/metrics.xml Wed Nov 25 22:30:29 2009
@@ -63,5 +63,118 @@
in ganglia, the stats are aggregated rather than reported per instance.
</p>
</section>
+
+ <section>
+ <title> Using with JMX </title>
+ <p>
+ In addition to the standard output contexts supported by the Hadoop
+ metrics package, you can also export HBase metrics via Java Management
+ Extensions (JMX). This will allow viewing HBase stats in JConsole or
+ any other JMX client.
+ </p>
+ <section>
+ <title>Enable HBase stats collection</title>
+ <p>
+ To enable JMX support in HBase, first edit
+ <code>$HBASE_HOME/conf/hadoop-metrics.properties</code> to support
+ metrics refreshing. (If you've already configured
+ <code>hadoop-metrics.properties</code> for another output context,
+ you can skip this step).
+ </p>
+ <source>
+# Configuration of the "hbase" context for null
+hbase.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+hbase.period=60
+
+# Configuration of the "jvm" context for null
+jvm.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+jvm.period=60
+
+# Configuration of the "rpc" context for null
+rpc.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+rpc.period=60
+ </source>
+ </section>
+ <section>
+ <title>Setup JMX remote access</title>
+ <p>
+ For remote access, you will need to configure JMX remote passwords
+ and access profiles. Create the files:
+ </p>
+ <dl>
+ <dt><code>$HBASE_HOME/conf/jmxremote.passwd</code> (set permissions
+ to 600)</dt>
+ <dd>
+ <source>
+monitorRole monitorpass
+controlRole controlpass
+ </source>
+ </dd>
+
+ <dt><code>$HBASE_HOME/conf/jmxremote.access</code></dt>
+ <dd>
+ <source>
+monitorRole readonly
+controlRole readwrite
+ </source>
+ </dd>
+ </dl>
+ </section>
+ <section>
+ <title>Configure JMX in HBase startup</title>
+ <p>
+ Finally, edit the <code>$HBASE_HOME/conf/hbase-env.sh</code> and
+ <code>$HBASE_HOME/bin/hbase</code> scripts for JMX support:
+ </p>
+ <dl>
+ <dt><code>$HBASE_HOME/conf/hbase-env.sh</code></dt>
+ <dd>
+ <p>Add the lines:</p>
+ <source>
+JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false"
+JMX_OPTS="$JMX_OPTS -Dcom.sun.management.jmxremote.password.file=$HBASE_HOME/conf/jmxremote.passwd"
+JMX_OPTS="$JMX_OPTS -Dcom.sun.management.jmxremote.access.file=$HBASE_HOME/conf/jmxremote.access"
+
+export HBASE_MASTER_OPTS="$JMX_OPTS -Dcom.sun.management.jmxremote.port=10101"
+export HBASE_REGIONSERVER_OPTS="$JMX_OPTS -Dcom.sun.management.jmxremote.port=10102"
+ </source>
+ </dd>
+ <dt><code>$HBASE_HOME/bin/hbase</code></dt>
+ <dd>
+ <p>Towards the end of the script, replace the lines:</p>
+ <source>
+ # figure out which class to run
+if [ "$COMMAND" = "shell" ] ; then
+ CLASS="org.jruby.Main ${HBASE_HOME}/bin/hirb.rb"
+elif [ "$COMMAND" = "master" ] ; then
+ CLASS='org.apache.hadoop.hbase.master.HMaster'
+elif [ "$COMMAND" = "regionserver" ] ; then
+ CLASS='org.apache.hadoop.hbase.regionserver.HRegionServer'
+ </source>
+ <p>
+ with the lines: (adding the "HBASE_OPTS=..." lines for "master" and
+ "regionserver" commands)
+ </p>
+ <source>
+ # figure out which class to run
+if [ "$COMMAND" = "shell" ] ; then
+ CLASS="org.jruby.Main ${HBASE_HOME}/bin/hirb.rb"
+elif [ "$COMMAND" = "master" ] ; then
+ CLASS='org.apache.hadoop.hbase.master.HMaster'
+ HBASE_OPTS="$HBASE_OPTS $HBASE_MASTER_OPTS"
+elif [ "$COMMAND" = "regionserver" ] ; then
+ CLASS='org.apache.hadoop.hbase.regionserver.HRegionServer'
+ HBASE_OPTS="$HBASE_OPTS $HBASE_REGIONSERVER_OPTS"
+ </source>
+ </dd>
+ </dl>
+ <p>
+ After restarting the processes you want to monitor, you should now be
+ able to run JConsole (included with the JDK since JDK 5.0) to view
+ the statistics via JMX. HBase MBeans are exported under the
+ <strong><code>hadoop</code></strong> domain in JMX.
+ </p>
+ </section>
+ </section>
</body>
</document>
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/docs/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/docs/src/documentation/content/xdocs/site.xml?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/docs/src/documentation/content/xdocs/site.xml (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/docs/src/documentation/content/xdocs/site.xml Wed Nov 25 22:30:29 2009
@@ -36,6 +36,7 @@
<started label="Getting Started" href="ext:api/started" />
<api label="API Docs" href="ext:api/index" />
<api label="HBase Metrics" href="metrics.html" />
+ <api label="HBase on Windows" href="cygwin.html" />
<wiki label="Wiki" href="ext:wiki" />
<faq label="FAQ" href="ext:faq" />
<lists label="Mailing Lists" href="ext:lists" />
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/docs/src/documentation/content/xdocs/tabs.xml
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/docs/src/documentation/content/xdocs/tabs.xml?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/docs/src/documentation/content/xdocs/tabs.xml (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/docs/src/documentation/content/xdocs/tabs.xml Wed Nov 25 22:30:29 2009
@@ -32,5 +32,4 @@
<tab label="Project" href="http://hadoop.apache.org/hbase/" />
<tab label="Wiki" href="http://wiki.apache.org/hadoop/Hbase" />
<tab label="HBase Documentation" dir="" />
-
</tabs>
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HConstants.java?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HConstants.java Wed Nov 25 22:30:29 2009
@@ -223,7 +223,7 @@
/**
* Max length a row can have because of the limitation in TFile.
*/
- static final int MAX_ROW_LENGTH = 1024*64;
+ static final int MAX_ROW_LENGTH = Short.MAX_VALUE;
/** When we encode strings, we always specify UTF8 encoding */
static final String UTF8_ENCODING = "UTF-8";
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerInfo.java?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerInfo.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerInfo.java Wed Nov 25 22:30:29 2009
@@ -22,6 +22,8 @@
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.Map;
+import java.util.HashMap;
import org.apache.hadoop.io.WritableComparable;
@@ -40,6 +42,7 @@
private int infoPort;
private String serverName = null;
private String name;
+ private static Map<String,String> dnsCache = new HashMap<String,String>();
/** default constructor - used by Writable */
public HServerInfo() {
@@ -73,7 +76,7 @@
this.infoPort = other.getInfoPort();
this.name = other.getName();
}
-
+
/**
* @return the load
*/
@@ -228,8 +231,19 @@
public static String getServerName(String serverAddress, long startCode) {
String name = null;
if (serverAddress != null) {
- HServerAddress address = new HServerAddress(serverAddress);
- name = getServerName(address.getHostname(), address.getPort(), startCode);
+ int colonIndex = serverAddress.lastIndexOf(':');
+ if(colonIndex < 0) {
+ throw new IllegalArgumentException("Not a host:port pair: " + serverAddress);
+ }
+ String host = serverAddress.substring(0, colonIndex);
+ int port =
+ Integer.valueOf(serverAddress.substring(colonIndex + 1)).intValue();
+ if(!dnsCache.containsKey(host)) {
+ HServerAddress address = new HServerAddress(serverAddress);
+ dnsCache.put(host, address.getHostname());
+ }
+ host = dnsCache.get(host);
+ name = getServerName(host, port, startCode);
}
return name;
}
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/KeyValue.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/KeyValue.java?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/KeyValue.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/KeyValue.java Wed Nov 25 22:30:29 2009
@@ -1044,6 +1044,9 @@
* @return True if matching families.
*/
public boolean matchingFamily(final byte [] family) {
+ if (this.length == 0 || this.bytes.length == 0) {
+ return false;
+ }
int o = getFamilyOffset();
int l = getFamilyLength(o);
return Bytes.compareTo(family, 0, family.length, this.bytes, o, l) == 0;
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Delete.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Delete.java?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Delete.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Delete.java Wed Nov 25 22:30:29 2009
@@ -51,16 +51,21 @@
* {@link #deleteColumn(byte [],byte [],long) deleteColumn}
* for each column version to delete.
* <p>
- * Specifying timestamps calling constructor, deleteFamily, and deleteColumns
- * will delete all versions with a timestamp less than or equal to that
- * specified. Specifying a timestamp to deleteColumn will delete versions
- * only with a timestamp equal to that specified.
- * <p>The timestamp passed to the constructor is only used ONLY for delete of
+ * Specifying timestamps, deleteFamily and deleteColumns will delete all
+ * versions with a timestamp less than or equal to that passed. If no
+ * timestamp is specified, an entry is added with a timestamp of 'now'
+ * where 'now' is the servers's System.currentTimeMillis().
+ * Specifying a timestamp to the deleteColumn method will
+ * delete versions only with a timestamp equal to that specified.
+ * If no timestamp is passed to deleteColumn, internally, it figures the
+ * most recent cell's timestamp and adds a delete at that timestamp; i.e.
+ * it deletes the most recently added cell.
+ * <p>The timestamp passed to the constructor is used ONLY for delete of
* rows. For anything less -- a deleteColumn, deleteColumns or
* deleteFamily -- then you need to use the method overrides that take a
* timestamp. The constructor timestamp is not referenced.
*/
-public class Delete implements Writable {
+public class Delete implements Writable, Row, Comparable<Row> {
private byte [] row = null;
// This ts is only used when doing a deleteRow. Anything less,
private long ts;
@@ -117,6 +122,10 @@
this.familyMap.putAll(d.getFamilyMap());
}
+ public int compareTo(final Row d) {
+ return Bytes.compareTo(this.getRow(), d.getRow());
+ }
+
/**
* Method to check if the familyMap is empty
* @return true if empty, false otherwise
@@ -356,4 +365,4 @@
}
}
}
-}
\ No newline at end of file
+}
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Get.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Get.java?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Get.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Get.java Wed Nov 25 22:30:29 2009
@@ -28,12 +28,14 @@
import java.util.TreeMap;
import java.util.TreeSet;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.io.HbaseObjectWritable;
import org.apache.hadoop.hbase.io.TimeRange;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableFactories;
/**
* Used to perform Get operations on a single row.
@@ -355,7 +357,8 @@
this.maxVersions = in.readInt();
boolean hasFilter = in.readBoolean();
if (hasFilter) {
- this.filter = (Filter)HbaseObjectWritable.readObject(in, null);
+ this.filter = (Filter)createForName(Bytes.toString(Bytes.readByteArray(in)));
+ this.filter.readFields(in);
}
this.tr = new TimeRange();
tr.readFields(in);
@@ -387,7 +390,8 @@
out.writeBoolean(false);
} else {
out.writeBoolean(true);
- HbaseObjectWritable.writeObject(out, this.filter, Filter.class, null);
+ Bytes.writeByteArray(out, Bytes.toBytes(filter.getClass().getName()));
+ filter.write(out);
}
tr.write(out);
out.writeInt(familyMap.size());
@@ -406,4 +410,15 @@
}
}
}
+
+ @SuppressWarnings("unchecked")
+ private Writable createForName(String className) {
+ try {
+ Class<? extends Writable> clazz =
+ (Class<? extends Writable>) Class.forName(className);
+ return WritableFactories.newInstance(clazz, new Configuration());
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException("Can't find class " + className);
+ }
+ }
}
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java Wed Nov 25 22:30:29 2009
@@ -238,7 +238,7 @@
} catch (RemoteException e) {
throw RemoteExceptionHandler.decodeRemoteException(e);
}
-
+ final int batchCount = this.conf.getInt("hbase.admin.scanner.caching", 10);
// Wait until first region is deleted
HRegionInterface server =
connection.getHRegionConnection(firstMetaServer.getServerAddress());
@@ -247,31 +247,34 @@
long scannerId = -1L;
try {
Scan scan = new Scan().addColumn(HConstants.CATALOG_FAMILY,
- HConstants.REGIONINFO_QUALIFIER);
-
+ HConstants.REGIONINFO_QUALIFIER);
scannerId = server.openScanner(
- firstMetaServer.getRegionInfo().getRegionName(),
- scan);
- Result values = server.next(scannerId);
- if (values == null || values.size() == 0) {
+ firstMetaServer.getRegionInfo().getRegionName(), scan);
+ // Get a batch at a time.
+ Result [] values = server.next(scannerId, batchCount);
+ if (values == null || values.length == 0) {
break;
}
boolean found = false;
- NavigableMap<byte[], byte[]> infoValues = values.getFamilyMap(HConstants.CATALOG_FAMILY);
- for (Map.Entry<byte [], byte []> e: infoValues.entrySet()) {
- if (Bytes.equals(e.getKey(), HConstants.REGIONINFO_QUALIFIER)) {
- info = (HRegionInfo) Writables.getWritable(
- e.getValue(), info);
-
- if (Bytes.equals(info.getTableDesc().getName(), tableName)) {
- found = true;
+ for (int i = 0; i < values.length; i++) {
+ Result r = values[i];
+ NavigableMap<byte[], byte[]> infoValues =
+ r.getFamilyMap(HConstants.CATALOG_FAMILY);
+ for (Map.Entry<byte[], byte[]> e : infoValues.entrySet()) {
+ if (Bytes.equals(e.getKey(), HConstants.REGIONINFO_QUALIFIER)) {
+ info = (HRegionInfo) Writables.getWritable(e.getValue(), info);
+ if (Bytes.equals(info.getTableDesc().getName(), tableName)) {
+ found = true;
+ } else {
+ found = false;
+ break;
+ }
}
}
}
if (!found) {
break;
}
-
} catch (IOException ex) {
if(tries == numRetries - 1) { // no more tries left
if (ex instanceof RemoteException) {
@@ -279,7 +282,6 @@
}
throw ex;
}
-
} finally {
if (scannerId != -1L) {
try {
@@ -289,7 +291,6 @@
}
}
}
-
try {
Thread.sleep(getPauseTime(tries));
} catch (InterruptedException e) {
@@ -301,6 +302,8 @@
LOG.info("Deleted " + Bytes.toString(tableName));
}
+
+
/**
* Brings a table on-line (enables it).
* Synchronous operation.
@@ -330,16 +333,17 @@
}
// Wait until all regions are enabled
-
- for (int tries = 0;
- (tries < numRetries) && (!isTableEnabled(tableName));
- tries++) {
+ boolean enabled = false;
+ for (int tries = 0; tries < this.numRetries; tries++) {
+ enabled = isTableEnabled(tableName);
+ if (enabled) break;
+ long sleep = getPauseTime(tries);
if (LOG.isDebugEnabled()) {
- LOG.debug("Sleep. Waiting for all regions to be enabled from " +
- Bytes.toString(tableName));
+ LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " +
+ "enabled in " + Bytes.toString(tableName));
}
try {
- Thread.sleep(getPauseTime(tries));
+ Thread.sleep(sleep);
} catch (InterruptedException e) {
// continue
}
@@ -348,8 +352,8 @@
Bytes.toString(tableName));
}
}
- if (!isTableEnabled(tableName))
- throw new IOException("unable to enable table " +
+ if (!enabled)
+ throw new IOException("Unable to enable table " +
Bytes.toString(tableName));
LOG.info("Enabled table " + Bytes.toString(tableName));
}
@@ -385,9 +389,10 @@
}
// Wait until all regions are disabled
- for (int tries = 0;
- (tries < numRetries) && (!isTableDisabled(tableName));
- tries++) {
+ boolean disabled = false;
+ for (int tries = 0; tries < this.numRetries; tries++) {
+ disabled = isTableDisabled(tableName);
+ if (disabled) break;
if (LOG.isDebugEnabled()) {
LOG.debug("Sleep. Waiting for all regions to be disabled from " +
Bytes.toString(tableName));
@@ -402,7 +407,7 @@
Bytes.toString(tableName));
}
}
- if (!isTableDisabled(tableName)) {
+ if (!disabled) {
throw new RegionException("Retries exhausted, it took too long to wait"+
" for the table " + Bytes.toString(tableName) + " to be disabled.");
}
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnection.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnection.java?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnection.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnection.java Wed Nov 25 22:30:29 2009
@@ -183,12 +183,22 @@
/**
- * Process a batch of rows. Currently it only works for updates until
- * HBASE-880 is available. Does the retries.
- * @param list A batch of rows to process
+ * Process a batch of Puts. Does the retries.
+ * @param list A batch of Puts to process.
* @param tableName The name of the table
+ * @return Count of committed Puts. On fault, < list.size().
* @throws IOException
*/
public int processBatchOfRows(ArrayList<Put> list, byte[] tableName)
- throws IOException;
+ throws IOException;
+
+ /**
+ * Process a batch of Deletes. Does the retries.
+ * @param list A batch of Deletes to process.
+ * @return Count of committed Deletes. On fault, < list.size().
+ * @param tableName The name of the table
+ * @throws IOException
+ */
+ public int processBatchOfDeletes(ArrayList<Delete> list, byte[] tableName)
+ throws IOException;
}
\ No newline at end of file
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java Wed Nov 25 22:30:29 2009
@@ -65,6 +65,9 @@
* Used by {@link HTable} and {@link HBaseAdmin}
*/
public class HConnectionManager implements HConstants {
+ private static final Delete [] DELETE_ARRAY_TYPE = new Delete[0];
+ private static final Put [] PUT_ARRAY_TYPE = new Put[0];
+
/*
* Not instantiable.
*/
@@ -194,7 +197,7 @@
* Get this watcher's ZKW, instanciate it if necessary.
* @return ZKW
*/
- public ZooKeeperWrapper getZooKeeperWrapper() throws IOException {
+ public synchronized ZooKeeperWrapper getZooKeeperWrapper() throws IOException {
if(zooKeeperWrapper == null) {
zooKeeperWrapper = new ZooKeeperWrapper(conf, this);
}
@@ -320,12 +323,15 @@
if (tryMaster.isMasterRunning()) {
this.master = tryMaster;
+ this.masterLock.notifyAll();
break;
}
} catch (IOException e) {
if (tries == numRetries - 1) {
// This was our last chance - don't bother sleeping
+ LOG.info("getMaster attempt " + tries + " of " + this.numRetries +
+ " failed; no more retrying.", e);
break;
}
LOG.info("getMaster attempt " + tries + " of " + this.numRetries +
@@ -335,7 +341,7 @@
// Cannot connect to master or it is not running. Sleep & retry
try {
- Thread.sleep(getPauseTime(tries));
+ this.masterLock.wait(getPauseTime(tries));
} catch (InterruptedException e) {
// continue
}
@@ -445,8 +451,8 @@
* Returns true if all regions are offline
* Returns false in any other case
*/
- private boolean testTableOnlineState(byte[] tableName,
- boolean online) throws IOException {
+ private boolean testTableOnlineState(byte[] tableName, boolean online)
+ throws IOException {
if (!tableExists(tableName)) {
throw new TableNotFoundException(Bytes.toString(tableName));
}
@@ -454,7 +460,6 @@
// The root region is always enabled
return true;
}
-
int rowsScanned = 0;
int rowsOffline = 0;
byte[] startKey =
@@ -463,6 +468,8 @@
HRegionInfo currentRegion = null;
Scan scan = new Scan(startKey);
scan.addColumn(CATALOG_FAMILY, REGIONINFO_QUALIFIER);
+ int rows = this.conf.getInt("hbase.meta.scanner.caching", 100);
+ scan.setCaching(rows);
ScannerCallable s = new ScannerCallable(this,
(Bytes.equals(tableName, HConstants.META_TABLE_NAME) ?
HConstants.ROOT_TABLE_NAME : HConstants.META_TABLE_NAME), scan);
@@ -477,10 +484,10 @@
currentRegion = s.getHRegionInfo();
Result r = null;
Result [] rrs = null;
- while ((rrs = getRegionServerWithRetries(s)) != null) {
+ while ((rrs = getRegionServerWithRetries(s)) != null && rrs.length > 0) {
r = rrs[0];
- byte [] value = r.getValue(HConstants.CATALOG_FAMILY,
- HConstants.REGIONINFO_QUALIFIER);
+ byte [] value = r.getValue(HConstants.CATALOG_FAMILY,
+ HConstants.REGIONINFO_QUALIFIER);
if (value != null) {
HRegionInfo info = Writables.getHRegionInfoOrNull(value);
if (info != null) {
@@ -492,18 +499,19 @@
}
}
endKey = currentRegion.getEndKey();
- } while (!(endKey == null ||
+ } while (!(endKey == null ||
Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY)));
- }
- finally {
+ } finally {
s.setClose();
+ // Doing below will call 'next' again and this will close the scanner
+ // Without it we leave scanners open.
+ getRegionServerWithRetries(s);
}
- boolean onlineOffline =
- online ? rowsOffline == 0 : rowsOffline == rowsScanned;
- return rowsScanned > 0 && onlineOffline;
-
+ LOG.debug("Rowscanned=" + rowsScanned + ", rowsOffline=" + rowsOffline);
+ boolean onOffLine = online? rowsOffline == 0: rowsOffline == rowsScanned;
+ return rowsScanned > 0 && onOffLine;
}
-
+
private static class HTableDescriptorFinder
implements MetaScanner.MetaScannerVisitor {
byte[] tableName;
@@ -845,7 +853,7 @@
public HRegionInterface getHRegionConnection(
HServerAddress regionServer, boolean getMaster)
throws IOException {
- if(getMaster) {
+ if (getMaster) {
getMaster();
}
HRegionInterface server;
@@ -923,9 +931,9 @@
"Timed out trying to locate root region");
}
- // get a connection to the region server
- HRegionInterface server = getHRegionConnection(rootRegionAddress);
try {
+ // Get a connection to the region server
+ HRegionInterface server = getHRegionConnection(rootRegionAddress);
// if this works, then we're good, and we have an acceptable address,
// so we can stop doing retries and return the result.
server.getRegionInfo(HRegionInfo.ROOT_REGIONINFO.getRegionName());
@@ -1055,85 +1063,167 @@
return location;
}
- public int processBatchOfRows(ArrayList<Put> list, byte[] tableName)
- throws IOException {
- if (list.isEmpty()) {
- return 0;
+ /*
+ * Helper class for batch updates.
+ * Holds code shared doing batch puts and batch deletes.
+ */
+ private abstract class Batch {
+ final HConnection c;
+
+ private Batch(final HConnection c) {
+ this.c = c;
+ }
+
+ /**
+ * This is the method subclasses must implement.
+ * @param currentList
+ * @param tableName
+ * @param row
+ * @return Count of items processed or -1 if all.
+ * @throws IOException
+ * @throws RuntimeException
+ */
+ abstract int doCall(final List<Row> currentList,
+ final byte [] row, final byte [] tableName)
+ throws IOException, RuntimeException;
+
+ /**
+ * Process the passed <code>list</code>.
+ * @param list
+ * @param tableName
+ * @return Count of how many added or -1 if all added.
+ * @throws IOException
+ */
+ int process(final ArrayList<? extends Row> list, final byte[] tableName)
+ throws IOException {
+ byte [] region = getRegionName(tableName, list.get(0).getRow(), false);
+ byte [] currentRegion = region;
+ boolean isLastRow = false;
+ boolean retryOnlyOne = false;
+ List<Row> currentList = new ArrayList<Row>();
+ int i, tries;
+ for (i = 0, tries = 0; i < list.size() && tries < numRetries; i++) {
+ Row row = list.get(i);
+ currentList.add(row);
+ // If the next record goes to a new region, then we are to clear
+ // currentList now during this cycle.
+ isLastRow = (i + 1) == list.size();
+ if (!isLastRow) {
+ region = getRegionName(tableName, list.get(i + 1).getRow(), false);
+ }
+ if (!Bytes.equals(currentRegion, region) || isLastRow || retryOnlyOne) {
+ int index = doCall(currentList, row.getRow(), tableName);
+ // index is == -1 if all processed successfully, else its index
+ // of last record successfully processed.
+ if (index != -1) {
+ if (tries == numRetries - 1) {
+ throw new RetriesExhaustedException("Some server, retryOnlyOne=" +
+ retryOnlyOne + ", index=" + index + ", islastrow=" + isLastRow +
+ ", tries=" + tries + ", numtries=" + numRetries + ", i=" + i +
+ ", listsize=" + list.size() + ", region=" +
+ Bytes.toStringBinary(region), currentRegion, row.getRow(),
+ tries, new ArrayList<Throwable>());
+ }
+ tries = doBatchPause(currentRegion, tries);
+ i = i - currentList.size() + index;
+ retryOnlyOne = true;
+ // Reload location.
+ region = getRegionName(tableName, list.get(i + 1).getRow(), true);
+ } else {
+ // Reset these flags/counters on successful batch Put
+ retryOnlyOne = false;
+ tries = 0;
+ }
+ currentRegion = region;
+ currentList.clear();
+ }
+ }
+ return i;
+ }
+
+ /*
+ * @param t
+ * @param r
+ * @param re
+ * @return Region name that holds passed row <code>r</code>
+ * @throws IOException
+ */
+ private byte [] getRegionName(final byte [] t, final byte [] r,
+ final boolean re)
+ throws IOException {
+ HRegionLocation location = getRegionLocationForRowWithRetries(t, r, re);
+ return location.getRegionInfo().getRegionName();
+ }
+
+ /*
+ * Do pause processing before retrying...
+ * @param currentRegion
+ * @param tries
+ * @return New value for tries.
+ */
+ private int doBatchPause(final byte [] currentRegion, final int tries) {
+ int localTries = tries;
+ long sleepTime = getPauseTime(tries);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Reloading region " + Bytes.toStringBinary(currentRegion) +
+ " location because regionserver didn't accept updates; tries=" +
+ tries + " of max=" + numRetries + ", waiting=" + sleepTime + "ms");
+ }
+ try {
+ Thread.sleep(sleepTime);
+ localTries++;
+ } catch (InterruptedException e) {
+ // continue
+ }
+ return localTries;
}
- boolean retryOnlyOne = false;
- if (list.size() > 1) {
- Collections.sort(list);
- }
- List<Put> currentPuts = new ArrayList<Put>();
- HRegionLocation location =
- getRegionLocationForRowWithRetries(tableName, list.get(0).getRow(),
- false);
- byte [] currentRegion = location.getRegionInfo().getRegionName();
- byte [] region = currentRegion;
- boolean isLastRow = false;
- Put [] putarray = new Put[0];
- int i, tries;
- for (i = 0, tries = 0; i < list.size() && tries < this.numRetries; i++) {
- Put put = list.get(i);
- currentPuts.add(put);
- // If the next Put goes to a new region, then we are to clear
- // currentPuts now during this cycle.
- isLastRow = (i + 1) == list.size();
- if (!isLastRow) {
- location = getRegionLocationForRowWithRetries(tableName,
- list.get(i + 1).getRow(), false);
- region = location.getRegionInfo().getRegionName();
- }
- if (!Bytes.equals(currentRegion, region) || isLastRow || retryOnlyOne) {
- final Put [] puts = currentPuts.toArray(putarray);
- int index = getRegionServerWithRetries(new ServerCallable<Integer>(
- this, tableName, put.getRow()) {
+ }
+
+ public int processBatchOfRows(final ArrayList<Put> list,
+ final byte[] tableName)
+ throws IOException {
+ if (list.isEmpty()) return 0;
+ if (list.size() > 1) Collections.sort(list);
+ Batch b = new Batch(this) {
+ @Override
+ int doCall(final List<Row> currentList, final byte [] row,
+ final byte [] tableName)
+ throws IOException, RuntimeException {
+ final Put [] puts = currentList.toArray(PUT_ARRAY_TYPE);
+ return getRegionServerWithRetries(new ServerCallable<Integer>(this.c,
+ tableName, row) {
public Integer call() throws IOException {
return server.put(location.getRegionInfo().getRegionName(), puts);
}
});
- // index is == -1 if all puts processed successfully, else its index
- // of last Put successfully processed.
- if (index != -1) {
- if (tries == numRetries - 1) {
- throw new RetriesExhaustedException("Some server, retryOnlyOne=" +
- retryOnlyOne + ", index=" + index + ", islastrow=" + isLastRow +
- ", tries=" + tries + ", numtries=" + numRetries + ", i=" + i +
- ", listsize=" + list.size() + ", location=" + location +
- ", region=" + Bytes.toStringBinary(region),
- currentRegion, put.getRow(), tries, new ArrayList<Throwable>());
- }
- long sleepTime = getPauseTime(tries);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Reloading region " + Bytes.toStringBinary(currentRegion) +
- " location because regionserver didn't accept updates; " +
- "tries=" + tries + " of max=" + this.numRetries +
- ", waiting=" + sleepTime + "ms");
- }
- try {
- Thread.sleep(sleepTime);
- tries++;
- } catch (InterruptedException e) {
- // continue
- }
- i = i - puts.length + index;
- retryOnlyOne = true;
- // Reload location.
- location = getRegionLocationForRowWithRetries(tableName,
- list.get(i + 1).getRow(), true);
- region = location.getRegionInfo().getRegionName();
- } else {
- // Reset these flags/counters on successful batch Put
- retryOnlyOne = false;
- tries = 0;
- }
- currentRegion = region;
- currentPuts.clear();
}
- }
- return i;
+ };
+ return b.process(list, tableName);
}
+ public int processBatchOfDeletes(final ArrayList<Delete> list,
+ final byte[] tableName)
+ throws IOException {
+ if (list.isEmpty()) return 0;
+ if (list.size() > 1) Collections.sort(list);
+ Batch b = new Batch(this) {
+ @Override
+ int doCall(final List<Row> currentList, final byte [] row,
+ final byte [] tableName)
+ throws IOException, RuntimeException {
+ final Delete [] deletes = currentList.toArray(DELETE_ARRAY_TYPE);
+ return getRegionServerWithRetries(new ServerCallable<Integer>(this.c,
+ tableName, row) {
+ public Integer call() throws IOException {
+ return server.delete(location.getRegionInfo().getRegionName(),
+ deletes);
+ }
+ });
+ }
+ };
+ return b.process(list, tableName);
+ }
+
void close(boolean stopProxy) {
if (master != null) {
if (stopProxy) {
@@ -1151,4 +1241,4 @@
}
}
}
-}
+}
\ No newline at end of file
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HTable.java?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HTable.java Wed Nov 25 22:30:29 2009
@@ -438,6 +438,23 @@
}
/**
+ * Bulk commit a List of Deletes to the table.
+ * @param deletes List of deletes. List is modified by this method. On
+ * exception holds deletes that were NOT applied.
+ * @throws IOException
+ * @since 0.20.1
+ */
+ public synchronized void delete(final ArrayList<Delete> deletes)
+ throws IOException {
+ int last = 0;
+ try {
+ last = connection.processBatchOfDeletes(deletes, this.tableName);
+ } finally {
+ deletes.subList(0, last).clear();
+ }
+ }
+
+ /**
* Commit a Put to the table.
* <p>
* If autoFlush is false, the update is buffered.
@@ -463,12 +480,12 @@
* @since 0.20.0
*/
public synchronized void put(final List<Put> puts) throws IOException {
- for(Put put : puts) {
+ for (Put put : puts) {
validatePut(put);
writeBuffer.add(put);
currentWriteBufferSize += put.heapSize();
}
- if(autoFlush || currentWriteBufferSize > writeBufferSize) {
+ if (autoFlush || currentWriteBufferSize > writeBufferSize) {
flushCommits();
}
}
@@ -692,6 +709,14 @@
}
}
+ /**
+ * Get the write buffer
+ * @return the current write buffer
+ */
+ public ArrayList<Put> getWriteBuffer() {
+ return writeBuffer;
+ }
+
// Old API. Pre-hbase-880, hbase-1304.
/**
@@ -1823,7 +1848,7 @@
}
public void initialize() throws IOException {
- nextScanner(this.caching);
+ nextScanner(this.caching, false);
}
protected Scan getScan() {
@@ -1834,13 +1859,36 @@
return lastNext;
}
+ /**
+ * @param endKey
+ * @return Returns true if the passed region endkey.
+ */
+ private boolean checkScanStopRow(final byte [] endKey) {
+ if (this.scan.getStopRow().length > 0) {
+ // there is a stop row, check to see if we are past it.
+ byte [] stopRow = scan.getStopRow();
+ int cmp = Bytes.compareTo(stopRow, 0, stopRow.length,
+ endKey, 0, endKey.length);
+ if (cmp <= 0) {
+ // stopRow <= endKey (endKey is equals to or larger than stopRow)
+ // This is a stop.
+ return true;
+ }
+ }
+ return false; //unlikely.
+ }
+
/*
* Gets a scanner for the next region. If this.currentRegion != null, then
* we will move to the endrow of this.currentRegion. Else we will get
- * scanner at the scan.getStartRow().
+ * scanner at the scan.getStartRow(). We will go no further, just tidy
+ * up outstanding scanners, if <code>currentRegion != null</code> and
+ * <code>done</code> is true.
* @param nbRows
+ * @param done Server-side says we're done scanning.
*/
- private boolean nextScanner(int nbRows) throws IOException {
+ private boolean nextScanner(int nbRows, final boolean done)
+ throws IOException {
// Close the previous scanner if it's open
if (this.callable != null) {
this.callable.setClose();
@@ -1850,21 +1898,24 @@
// Where to start the next scanner
byte [] localStartKey = null;
-
- // if we're at the end of the table, then close and return false
- // to stop iterating
+
+ // if we're at end of table, close and return false to stop iterating
if (this.currentRegion != null) {
- if (CLIENT_LOG.isDebugEnabled()) {
- CLIENT_LOG.debug("Finished with region " + this.currentRegion);
- }
byte [] endKey = this.currentRegion.getEndKey();
if (endKey == null ||
Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY) ||
- filterSaysStop(endKey)) {
+ checkScanStopRow(endKey) ||
+ done) {
close();
+ if (CLIENT_LOG.isDebugEnabled()) {
+ CLIENT_LOG.debug("Finished with scanning at " + this.currentRegion);
+ }
return false;
}
localStartKey = endKey;
+ if (CLIENT_LOG.isDebugEnabled()) {
+ CLIENT_LOG.debug("Finished with region " + this.currentRegion);
+ }
} else {
localStartKey = this.scan.getStartRow();
}
@@ -1895,40 +1946,6 @@
return s;
}
- /**
- * @param endKey
- * @return Returns true if the passed region endkey is judged beyond
- * filter.
- */
- private boolean filterSaysStop(final byte [] endKey) {
- if (scan.getStopRow().length > 0) {
- // there is a stop row, check to see if we are past it.
- byte [] stopRow = scan.getStopRow();
- int cmp = Bytes.compareTo(stopRow, 0, stopRow.length,
- endKey, 0, endKey.length);
- if (cmp <= 0) {
- // stopRow <= endKey (endKey is equals to or larger than stopRow)
- // This is a stop.
- return true;
- }
- }
-
- if(!scan.hasFilter()) {
- return false;
- }
-
- if (scan.getFilter() != null) {
- // Let the filter see current row.
- scan.getFilter().filterRowKey(endKey, 0, endKey.length);
- return scan.getFilter().filterAllRemaining();
- }
- if (scan.getOldFilter() != null) {
- scan.getOldFilter().filterRowKey(endKey, 0, endKey.length);
- return scan.getOldFilter().filterAllRemaining();
- }
- return false; //unlikely.
- }
-
public Result next() throws IOException {
// If the scanner is closed but there is some rows left in the cache,
// it will first empty it before returning null
@@ -1946,6 +1963,9 @@
boolean skipFirst = false;
do {
try {
+ // Server returns a null values if scanning is to stop. Else,
+ // returns an empty array if scanning is to go on and we've just
+ // exhausted current region.
values = getConnection().getRegionServerWithRetries(callable);
if (skipFirst) {
skipFirst = false;
@@ -1959,12 +1979,14 @@
}
// Else, its signal from depths of ScannerCallable that we got an
// NSRE on a next and that we need to reset the scanner.
- this.scan.setStartRow(this.lastResult.getRow());
- // Clear region as flag to nextScanner to use this.scan.startRow.
+ if (this.lastResult != null) {
+ this.scan.setStartRow(this.lastResult.getRow());
+ // Skip first row returned. We already let it out on previous
+ // invocation.
+ skipFirst = true;
+ }
+ // Clear region
this.currentRegion = null;
- // Skip first row returned. We already let it out on previous
- // invocation.
- skipFirst = true;
continue;
} catch (IOException e) {
if (e instanceof UnknownScannerException &&
@@ -1983,7 +2005,8 @@
this.lastResult = rs;
}
}
- } while (countdown > 0 && nextScanner(countdown));
+ // Values == null means server-side filter has determined we must STOP
+ } while (countdown > 0 && nextScanner(countdown, values == null));
}
if (cache.size() > 0) {
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HTablePool.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HTablePool.java?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HTablePool.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HTablePool.java Wed Nov 25 22:30:29 2009
@@ -114,7 +114,12 @@
}
}
- private HTable newHTable(String tableName) {
+ /**
+ * @param tableName
+ * @return HTable instance.
+ * @deprecated Use createHTable
+ */
+ protected HTable newHTable(String tableName) {
try {
return new HTable(config, Bytes.toBytes(tableName));
} catch(IOException ioe) {
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/MetaScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/MetaScanner.java?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/MetaScanner.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/MetaScanner.java Wed Nov 25 22:30:29 2009
@@ -47,20 +47,26 @@
// Scan over each meta region
ScannerCallable callable = null;
+ int rows = configuration.getInt("hbase.meta.scanner.caching", 100);
do {
Scan scan = new Scan(startRow).addFamily(CATALOG_FAMILY);
callable = new ScannerCallable(connection, META_TABLE_NAME, scan);
// Open scanner
connection.getRegionServerWithRetries(callable);
try {
- Result r = null;
- do {
+ callable.setCaching(rows);
+ done: do {
+ //we have all the rows here
Result [] rrs = connection.getRegionServerWithRetries(callable);
if (rrs == null || rrs.length == 0 || rrs[0].size() == 0) {
- break;
+ break done; //exit completely
}
- r = rrs[0];
- } while(visitor.processRow(r));
+ for (int i = 0; i < rrs.length; i++) {
+ if (!visitor.processRow(rrs[i]))
+ break done; //exit completely
+ }
+ //here, we didn't break anywhere. Check if we have more rows
+ } while(true);
// Advance the startRow to the end key of the current region
startRow = callable.getHRegionInfo().getEndKey();
} finally {
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Put.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Put.java?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Put.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Put.java Wed Nov 25 22:30:29 2009
@@ -44,7 +44,7 @@
* for each column to be inserted, execute {@link #add(byte[], byte[], byte[]) add} or
* {@link #add(byte[], byte[], long, byte[]) add} if setting the timestamp.
*/
-public class Put implements HeapSize, Writable, Comparable<Put> {
+public class Put implements HeapSize, Writable, Row, Comparable<Row> {
private byte [] row = null;
private long timestamp = HConstants.LATEST_TIMESTAMP;
private long lockId = -1L;
@@ -90,6 +90,8 @@
*/
public Put(Put putToCopy) {
this(putToCopy.getRow(), putToCopy.getRowLock());
+ this.timestamp = putToCopy.timestamp;
+ this.writeToWAL = putToCopy.writeToWAL;
this.familyMap =
new TreeMap<byte [], List<KeyValue>>(Bytes.BYTES_COMPARATOR);
for(Map.Entry<byte [], List<KeyValue>> entry :
@@ -157,10 +159,10 @@
int res = Bytes.compareTo(this.row, 0, row.length,
kv.getBuffer(), kv.getRowOffset(), kv.getRowLength());
if(res != 0) {
- throw new IOException("The row in the recently added KeyValue " +
- Bytes.toStringBinary(kv.getBuffer(), kv.getRowOffset(),
- kv.getRowLength()) + " doesn't match the original one " +
- Bytes.toStringBinary(this.row));
+ throw new IOException("The row in the recently added KeyValue " +
+ Bytes.toStringBinary(kv.getBuffer(), kv.getRowOffset(),
+ kv.getRowLength()) + " doesn't match the original one " +
+ Bytes.toStringBinary(this.row));
}
list.add(kv);
familyMap.put(family, list);
@@ -217,6 +219,8 @@
/**
* Method for setting the timestamp
+ * NOTE - This does not affect the timestamp for values previously added to this Put.
+ * It only affects the timestamp for values added after this method is called.
* @param timestamp
*/
public Put setTimeStamp(long timestamp) {
@@ -292,7 +296,7 @@
return sb.toString();
}
- public int compareTo(Put p) {
+ public int compareTo(Row p) {
return Bytes.compareTo(this.getRow(), p.getRow());
}
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Result.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Result.java?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Result.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Result.java Wed Nov 25 22:30:29 2009
@@ -23,6 +23,7 @@
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
@@ -32,6 +33,7 @@
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.SplitKeyValue;
import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.hbase.util.Bytes;
@@ -69,6 +71,7 @@
// We're not using java serialization. Transient here is just a marker to say
// that this is where we cache row if we're ever asked for it.
private transient byte [] row = null;
+ private ImmutableBytesWritable bytes = null;
/**
* Constructor used for Writable.
@@ -92,6 +95,14 @@
public Result(List<KeyValue> kvs) {
this(kvs.toArray(new KeyValue[0]));
}
+
+ /**
+ * Instantiate a Result from the specified raw binary format.
+ * @param bytes raw binary format of Result
+ */
+ public Result(ImmutableBytesWritable bytes) {
+ this.bytes = bytes;
+ }
/**
* Method for retrieving the row that this result is for
@@ -99,8 +110,10 @@
*/
public synchronized byte [] getRow() {
if (this.row == null) {
- this.row =
- this.kvs == null || this.kvs.length == 0? null: this.kvs[0].getRow();
+ if(this.kvs == null) {
+ readFields();
+ }
+ this.row = this.kvs.length == 0? null: this.kvs[0].getRow();
}
return this.row;
}
@@ -110,6 +123,9 @@
* @return unsorted array of KeyValues
*/
public KeyValue[] raw() {
+ if(this.kvs == null) {
+ readFields();
+ }
return kvs;
}
@@ -119,7 +135,10 @@
* @return The sorted list of KeyValue's.
*/
public List<KeyValue> list() {
- return Arrays.asList(sorted());
+ if(this.kvs == null) {
+ readFields();
+ }
+ return isEmpty()? null: Arrays.asList(sorted());
}
/**
@@ -352,6 +371,9 @@
* @return a RowResult
*/
public RowResult getRowResult() {
+ if(this.kvs == null) {
+ readFields();
+ }
return RowResult.createRowResult(Arrays.asList(kvs));
}
@@ -367,10 +389,25 @@
}
/**
+ * Returns the raw binary encoding of this Result.<p>
+ *
+ * Please note, there may be an offset into the underlying byte array of the
+ * returned ImmutableBytesWritable. Be sure to use both
+ * {@link ImmutableBytesWritable#get()} and {@link ImmutableBytesWritable#getOffset()}
+ * @return pointer to raw binary of Result
+ */
+ public ImmutableBytesWritable getBytes() {
+ return this.bytes;
+ }
+
+ /**
* Check if the underlying KeyValue [] is empty or not
* @return true if empty
*/
public boolean isEmpty() {
+ if(this.kvs == null) {
+ readFields();
+ }
return this.kvs == null || this.kvs.length == 0;
}
@@ -378,6 +415,9 @@
* @return the size of the underlying KeyValue []
*/
public int size() {
+ if(this.kvs == null) {
+ readFields();
+ }
return this.kvs == null? 0: this.kvs.length;
}
@@ -411,20 +451,34 @@
throws IOException {
familyMap = null;
row = null;
- int numKeys = in.readInt();
- this.kvs = new KeyValue[numKeys];
- if(numKeys == 0) {
+ this.kvs = null;
+ int totalBuffer = in.readInt();
+ if(totalBuffer == 0) {
+ bytes = null;
return;
}
- int totalBuffer = in.readInt();
- byte [] buf = new byte[totalBuffer];
- int offset = 0;
- for(int i=0; i<numKeys; i++) {
- int keyLength = in.readInt();
- in.readFully(buf, offset, keyLength);
- kvs[i] = new KeyValue(buf, offset, keyLength);
+ byte [] raw = new byte[totalBuffer];
+ in.readFully(raw, 0, totalBuffer);
+ bytes = new ImmutableBytesWritable(raw, 0, totalBuffer);
+ }
+
+ //Create KeyValue[] when needed
+ private void readFields() {
+ if (bytes == null) {
+ this.kvs = new KeyValue[0];
+ return;
+ }
+ byte [] buf = bytes.get();
+ int offset = bytes.getOffset();
+ int finalOffset = bytes.getSize() + offset;
+ List<KeyValue> kvs = new ArrayList<KeyValue>();
+ while(offset < finalOffset) {
+ int keyLength = Bytes.toInt(buf, offset);
+ offset += Bytes.SIZEOF_INT;
+ kvs.add(new KeyValue(buf, offset, keyLength));
offset += keyLength;
}
+ this.kvs = kvs.toArray(new KeyValue[kvs.size()]);
}
public void write(final DataOutput out)
@@ -432,11 +486,9 @@
if(isEmpty()) {
out.writeInt(0);
} else {
- int len = this.kvs.length;
- out.writeInt(len);
int totalLen = 0;
for(KeyValue kv : kvs) {
- totalLen += kv.getLength();
+ totalLen += kv.getLength() + Bytes.SIZEOF_INT;
}
out.writeInt(totalLen);
for(KeyValue kv : kvs) {
@@ -455,11 +507,12 @@
out.writeInt(results.length);
int bufLen = 0;
for(Result result : results) {
+ bufLen += Bytes.SIZEOF_INT;
if(result == null || result.isEmpty()) {
continue;
}
for(KeyValue key : result.raw()) {
- bufLen += key.getLength();
+ bufLen += key.getLength() + Bytes.SIZEOF_INT;
}
}
out.writeInt(bufLen);
@@ -488,14 +541,22 @@
int offset = 0;
for(int i=0;i<numResults;i++) {
int numKeys = in.readInt();
- KeyValue [] keys = new KeyValue[numKeys];
+ offset += Bytes.SIZEOF_INT;
+ if(numKeys == 0) {
+ results[i] = new Result((ImmutableBytesWritable)null);
+ continue;
+ }
+ int initialOffset = offset;
for(int j=0;j<numKeys;j++) {
int keyLen = in.readInt();
+ Bytes.putInt(buf, offset, keyLen);
+ offset += Bytes.SIZEOF_INT;
in.readFully(buf, offset, keyLen);
- keys[j] = new KeyValue(buf, offset, keyLen);
offset += keyLen;
}
- results[i] = new Result(keys);
+ int totalLength = offset - initialOffset;
+ results[i] = new Result(new ImmutableBytesWritable(buf, initialOffset,
+ totalLength));
}
return results;
}
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java Wed Nov 25 22:30:29 2009
@@ -26,6 +26,11 @@
*/
public class RetriesExhaustedException extends IOException {
private static final long serialVersionUID = 1876775844L;
+
+ public RetriesExhaustedException(final String msg) {
+ super(msg);
+ }
+
/**
* Create a new RetriesExhaustedException from the list of prior failures.
* @param serverName name of HRegionServer
@@ -35,11 +40,9 @@
* @param exceptions List of exceptions that failed before giving up
*/
public RetriesExhaustedException(String serverName, final byte [] regionName,
- final byte [] row,
- int numTries, List<Throwable> exceptions) {
+ final byte [] row, int numTries, List<Throwable> exceptions) {
super(getMessage(serverName, regionName, row, numTries, exceptions));
}
-
private static String getMessage(String serverName, final byte [] regionName,
final byte [] row,
@@ -59,4 +62,4 @@
}
return buffer.toString();
}
-}
+}
\ No newline at end of file
Added: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Row.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Row.java?rev=884310&view=auto
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Row.java (added)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Row.java Wed Nov 25 22:30:29 2009
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+/**
+ * Has a row.
+ */
+interface Row {
+ /**
+ * @return The row.
+ */
+ public byte [] getRow();
+}
\ No newline at end of file
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Scan.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Scan.java?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Scan.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Scan.java Wed Nov 25 22:30:29 2009
@@ -69,12 +69,16 @@
* {@link #setMaxVersions(int) setMaxVersions}.
* <p>
* To add a filter, execute {@link #setFilter(org.apache.hadoop.hbase.filter.Filter) setFilter}.
+ * <p>
+ * Expert: To explicitly disable server-side block caching for this scan,
+ * execute {@link #setCacheBlocks(boolean)}.
*/
public class Scan implements Writable {
private byte [] startRow = HConstants.EMPTY_START_ROW;
private byte [] stopRow = HConstants.EMPTY_END_ROW;
private int maxVersions = 1;
private int caching = -1;
+ private boolean cacheBlocks = true;
private Filter filter = null;
private RowFilterInterface oldFilter = null;
private TimeRange tr = new TimeRange();
@@ -123,6 +127,7 @@
stopRow = scan.getStopRow();
maxVersions = scan.getMaxVersions();
caching = scan.getCaching();
+ cacheBlocks = scan.getCacheBlocks();
filter = scan.getFilter(); // clone?
oldFilter = scan.getOldFilter(); // clone?
TimeRange ctr = scan.getTimeRange();
@@ -230,27 +235,32 @@
* @return The columns in an old style string format.
*/
public String getInputColumns() {
- String cols = "";
+ StringBuilder cols = new StringBuilder();
for (Map.Entry<byte[], NavigableSet<byte[]>> e :
familyMap.entrySet()) {
byte[] fam = e.getKey();
- if (cols.length() > 0) cols += " ";
+ if (cols.length() > 0) {
+ cols.append(" ");
+ }
NavigableSet<byte[]> quals = e.getValue();
// check if this family has qualifiers
if (quals != null && quals.size() > 0) {
- String cs = "";
for (byte[] qual : quals) {
- if (cs.length() > 0) cs += " ";
+ if (cols.length() > 0) {
+ cols.append(" ");
+ }
// encode values to make parsing easier later
- cs += Bytes.toStringBinary(fam) + ":" + Bytes.toStringBinary(qual);
+ cols.append(Bytes.toStringBinary(fam));
+ cols.append(":");
+ cols.append(Bytes.toStringBinary(qual));
}
- cols += cs;
} else {
// only add the family but with old style delimiter
- cols += Bytes.toStringBinary(fam) + ":";
+ cols.append(Bytes.toStringBinary(fam));
+ cols.append(":");
}
}
- return cols;
+ return cols.toString();
}
/**
@@ -449,6 +459,29 @@
}
/**
+ * Set whether blocks should be cached for this Scan.
+ * <p>
+ * This is true by default. When true, default settings of the table and
+ * family are used (this will never override caching blocks if the block
+ * cache is disabled for that family or entirely).
+ *
+ * @param cacheBlocks if false, default settings are overridden and blocks
+ * will not be cached
+ */
+ public void setCacheBlocks(boolean cacheBlocks) {
+ this.cacheBlocks = cacheBlocks;
+ }
+
+ /**
+ * Get whether blocks should be cached for this Scan.
+ * @return true if default caching should be used, false if blocks should not
+ * be cached
+ */
+ public boolean getCacheBlocks() {
+ return cacheBlocks;
+ }
+
+ /**
* @return String
*/
@Override
@@ -462,6 +495,8 @@
sb.append("" + this.maxVersions);
sb.append(", caching=");
sb.append("" + this.caching);
+ sb.append(", cacheBlocks=");
+ sb.append("" + this.cacheBlocks);
sb.append(", timeRange=");
sb.append("[" + this.tr.getMin() + "," + this.tr.getMax() + ")");
sb.append(", families=");
@@ -518,6 +553,7 @@
this.stopRow = Bytes.readByteArray(in);
this.maxVersions = in.readInt();
this.caching = in.readInt();
+ this.cacheBlocks = in.readBoolean();
if(in.readBoolean()) {
this.filter = (Filter)createForName(Bytes.toString(Bytes.readByteArray(in)));
this.filter.readFields(in);
@@ -550,6 +586,7 @@
Bytes.writeByteArray(out, this.stopRow);
out.writeInt(this.maxVersions);
out.writeInt(this.caching);
+ out.writeBoolean(this.cacheBlocks);
if(this.filter == null) {
out.writeBoolean(false);
} else {
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ScannerCallable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ScannerCallable.java?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ScannerCallable.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ScannerCallable.java Wed Nov 25 22:30:29 2009
@@ -29,6 +29,7 @@
import org.apache.hadoop.hbase.RemoteExceptionHandler;
import org.apache.hadoop.ipc.RemoteException;
+
/**
* Retries scanner operations such as create, next, etc.
* Used by {@link ResultScanner}s made by {@link HTable}.
@@ -75,7 +76,7 @@
try {
rrs = server.next(scannerId, caching);
} catch (IOException e) {
- IOException ioe = null;
+ IOException ioe = null;
if (e instanceof RemoteException) {
ioe = RemoteExceptionHandler.decodeRemoteException((RemoteException)e);
}
@@ -86,26 +87,26 @@
throw new DoNotRetryIOException("Reset scanner", ioe);
}
}
- return rrs == null || rrs.length == 0? null: rrs;
+ return rrs;
}
return null;
}
private void close() {
- if (this.scannerId == -1L) {
- return;
- }
- try {
- this.server.close(this.scannerId);
- } catch (IOException e) {
- // ignore
- }
- this.scannerId = -1L;
+ if (this.scannerId == -1L) {
+ return;
+ }
+ try {
+ this.server.close(this.scannerId);
+ } catch (IOException e) {
+ // Ignore, probably already closed
+ }
+ this.scannerId = -1L;
}
protected long openScanner() throws IOException {
- return server.openScanner(
- this.location.getRegionInfo().getRegionName(), scan);
+ return this.server.openScanner(this.location.getRegionInfo().getRegionName(),
+ this.scan);
}
protected Scan getScan() {
@@ -116,7 +117,7 @@
* Call this when the next invocation of call should close the scanner
*/
public void setClose() {
- closed = true;
+ this.closed = true;
}
/**
@@ -144,4 +145,4 @@
public void setCaching(int caching) {
this.caching = caching;
}
-}
\ No newline at end of file
+}
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/BinaryComparator.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/BinaryComparator.java?rev=884310&r1=884309&r2=884310&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/BinaryComparator.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/BinaryComparator.java Wed Nov 25 22:30:29 2009
@@ -62,4 +62,4 @@
public int compareTo(byte [] value) {
return Bytes.compareTo(this.value, value);
}
-}
+}
\ No newline at end of file