You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2016/07/18 11:23:28 UTC
[39/52] [partial] hbase-site git commit: Published site at
9bc7ecfb9dec6bfe14a12b6d3bfd11392d7752b8.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/5f066b2e/apidocs/src-html/org/apache/hadoop/hbase/mapreduce/MultiHFileOutputFormat.html
----------------------------------------------------------------------
diff --git a/apidocs/src-html/org/apache/hadoop/hbase/mapreduce/MultiHFileOutputFormat.html b/apidocs/src-html/org/apache/hadoop/hbase/mapreduce/MultiHFileOutputFormat.html
new file mode 100644
index 0000000..39ebcf3
--- /dev/null
+++ b/apidocs/src-html/org/apache/hadoop/hbase/mapreduce/MultiHFileOutputFormat.html
@@ -0,0 +1,174 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html lang="en">
+<head>
+<title>Source code</title>
+<link rel="stylesheet" type="text/css" href="../../../../../../stylesheet.css" title="Style">
+</head>
+<body>
+<div class="sourceContainer">
+<pre><span class="sourceLineNo">001</span>/**<a name="line.1"></a>
+<span class="sourceLineNo">002</span> * Licensed to the Apache Software Foundation (ASF) under one or more contributor license<a name="line.2"></a>
+<span class="sourceLineNo">003</span> * agreements. See the NOTICE file distributed with this work for additional information regarding<a name="line.3"></a>
+<span class="sourceLineNo">004</span> * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the<a name="line.4"></a>
+<span class="sourceLineNo">005</span> * "License"); you may not use this file except in compliance with the License. You may obtain a<a name="line.5"></a>
+<span class="sourceLineNo">006</span> * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable<a name="line.6"></a>
+<span class="sourceLineNo">007</span> * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"<a name="line.7"></a>
+<span class="sourceLineNo">008</span> * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License<a name="line.8"></a>
+<span class="sourceLineNo">009</span> * for the specific language governing permissions and limitations under the License.<a name="line.9"></a>
+<span class="sourceLineNo">010</span> */<a name="line.10"></a>
+<span class="sourceLineNo">011</span>package org.apache.hadoop.hbase.mapreduce;<a name="line.11"></a>
+<span class="sourceLineNo">012</span><a name="line.12"></a>
+<span class="sourceLineNo">013</span>import java.io.IOException;<a name="line.13"></a>
+<span class="sourceLineNo">014</span>import java.util.HashMap;<a name="line.14"></a>
+<span class="sourceLineNo">015</span>import java.util.Map;<a name="line.15"></a>
+<span class="sourceLineNo">016</span>import org.apache.commons.logging.Log;<a name="line.16"></a>
+<span class="sourceLineNo">017</span>import org.apache.commons.logging.LogFactory;<a name="line.17"></a>
+<span class="sourceLineNo">018</span>import org.apache.hadoop.hbase.classification.InterfaceAudience;<a name="line.18"></a>
+<span class="sourceLineNo">019</span>import org.apache.hadoop.hbase.classification.InterfaceStability;<a name="line.19"></a>
+<span class="sourceLineNo">020</span>import org.apache.hadoop.conf.Configuration;<a name="line.20"></a>
+<span class="sourceLineNo">021</span>import org.apache.hadoop.fs.FileSystem;<a name="line.21"></a>
+<span class="sourceLineNo">022</span>import org.apache.hadoop.fs.Path;<a name="line.22"></a>
+<span class="sourceLineNo">023</span>import org.apache.hadoop.hbase.Cell;<a name="line.23"></a>
+<span class="sourceLineNo">024</span>import org.apache.hadoop.hbase.io.ImmutableBytesWritable;<a name="line.24"></a>
+<span class="sourceLineNo">025</span>import org.apache.hadoop.hbase.util.Bytes;<a name="line.25"></a>
+<span class="sourceLineNo">026</span>import org.apache.hadoop.mapreduce.RecordWriter;<a name="line.26"></a>
+<span class="sourceLineNo">027</span>import org.apache.hadoop.mapreduce.TaskAttemptContext;<a name="line.27"></a>
+<span class="sourceLineNo">028</span>import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;<a name="line.28"></a>
+<span class="sourceLineNo">029</span>import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;<a name="line.29"></a>
+<span class="sourceLineNo">030</span>import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2;<a name="line.30"></a>
+<span class="sourceLineNo">031</span><a name="line.31"></a>
+<span class="sourceLineNo">032</span>import com.google.common.annotations.VisibleForTesting;<a name="line.32"></a>
+<span class="sourceLineNo">033</span>/**<a name="line.33"></a>
+<span class="sourceLineNo">034</span> * Create 3 level tree directory, first level is using table name as parent directory and then use<a name="line.34"></a>
+<span class="sourceLineNo">035</span> * family name as child directory, and all related HFiles for one family are under child directory<a name="line.35"></a>
+<span class="sourceLineNo">036</span> * -tableName1<a name="line.36"></a>
+<span class="sourceLineNo">037</span> * -columnFamilyName1<a name="line.37"></a>
+<span class="sourceLineNo">038</span> * -columnFamilyName2<a name="line.38"></a>
+<span class="sourceLineNo">039</span> * -HFiles<a name="line.39"></a>
+<span class="sourceLineNo">040</span> * -tableName2<a name="line.40"></a>
+<span class="sourceLineNo">041</span> * -columnFamilyName1<a name="line.41"></a>
+<span class="sourceLineNo">042</span> * -HFiles<a name="line.42"></a>
+<span class="sourceLineNo">043</span> * -columnFamilyName2<a name="line.43"></a>
+<span class="sourceLineNo">044</span> * <p><a name="line.44"></a>
+<span class="sourceLineNo">045</span> */<a name="line.45"></a>
+<span class="sourceLineNo">046</span>@InterfaceAudience.Public<a name="line.46"></a>
+<span class="sourceLineNo">047</span>@InterfaceStability.Evolving<a name="line.47"></a>
+<span class="sourceLineNo">048</span>@VisibleForTesting<a name="line.48"></a>
+<span class="sourceLineNo">049</span>public class MultiHFileOutputFormat extends FileOutputFormat<ImmutableBytesWritable, Cell> {<a name="line.49"></a>
+<span class="sourceLineNo">050</span> private static final Log LOG = LogFactory.getLog(MultiHFileOutputFormat.class);<a name="line.50"></a>
+<span class="sourceLineNo">051</span><a name="line.51"></a>
+<span class="sourceLineNo">052</span> @Override<a name="line.52"></a>
+<span class="sourceLineNo">053</span> public RecordWriter<ImmutableBytesWritable, Cell><a name="line.53"></a>
+<span class="sourceLineNo">054</span> getRecordWriter(final TaskAttemptContext context) throws IOException, InterruptedException {<a name="line.54"></a>
+<span class="sourceLineNo">055</span> return createMultiHFileRecordWriter(context);<a name="line.55"></a>
+<span class="sourceLineNo">056</span> }<a name="line.56"></a>
+<span class="sourceLineNo">057</span><a name="line.57"></a>
+<span class="sourceLineNo">058</span> static <V extends Cell> RecordWriter<ImmutableBytesWritable, V><a name="line.58"></a>
+<span class="sourceLineNo">059</span> createMultiHFileRecordWriter(final TaskAttemptContext context) throws IOException {<a name="line.59"></a>
+<span class="sourceLineNo">060</span><a name="line.60"></a>
+<span class="sourceLineNo">061</span> // Get the path of the output directory<a name="line.61"></a>
+<span class="sourceLineNo">062</span> final Path outputPath = FileOutputFormat.getOutputPath(context);<a name="line.62"></a>
+<span class="sourceLineNo">063</span> final Path outputDir = new FileOutputCommitter(outputPath, context).getWorkPath();<a name="line.63"></a>
+<span class="sourceLineNo">064</span> final Configuration conf = context.getConfiguration();<a name="line.64"></a>
+<span class="sourceLineNo">065</span> final FileSystem fs = outputDir.getFileSystem(conf);<a name="line.65"></a>
+<span class="sourceLineNo">066</span><a name="line.66"></a>
+<span class="sourceLineNo">067</span> // Map of tables to writers<a name="line.67"></a>
+<span class="sourceLineNo">068</span> final Map<ImmutableBytesWritable, RecordWriter<ImmutableBytesWritable, V>> tableWriters =<a name="line.68"></a>
+<span class="sourceLineNo">069</span> new HashMap<ImmutableBytesWritable, RecordWriter<ImmutableBytesWritable, V>>();<a name="line.69"></a>
+<span class="sourceLineNo">070</span><a name="line.70"></a>
+<span class="sourceLineNo">071</span> return new RecordWriter<ImmutableBytesWritable, V>() {<a name="line.71"></a>
+<span class="sourceLineNo">072</span> @Override<a name="line.72"></a>
+<span class="sourceLineNo">073</span> public void write(ImmutableBytesWritable tableName, V cell)<a name="line.73"></a>
+<span class="sourceLineNo">074</span> throws IOException, InterruptedException {<a name="line.74"></a>
+<span class="sourceLineNo">075</span> RecordWriter<ImmutableBytesWritable, V> tableWriter = tableWriters.get(tableName);<a name="line.75"></a>
+<span class="sourceLineNo">076</span> // if there is new table, verify that table directory exists<a name="line.76"></a>
+<span class="sourceLineNo">077</span> if (tableWriter == null) {<a name="line.77"></a>
+<span class="sourceLineNo">078</span> // using table name as directory name<a name="line.78"></a>
+<span class="sourceLineNo">079</span> final Path tableOutputDir = new Path(outputDir, Bytes.toString(tableName.copyBytes()));<a name="line.79"></a>
+<span class="sourceLineNo">080</span> fs.mkdirs(tableOutputDir);<a name="line.80"></a>
+<span class="sourceLineNo">081</span> LOG.info("Writing Table '" + tableName.toString() + "' data into following directory"<a name="line.81"></a>
+<span class="sourceLineNo">082</span> + tableOutputDir.toString());<a name="line.82"></a>
+<span class="sourceLineNo">083</span><a name="line.83"></a>
+<span class="sourceLineNo">084</span> // Create writer for one specific table<a name="line.84"></a>
+<span class="sourceLineNo">085</span> tableWriter = new HFileOutputFormat2.HFileRecordWriter<V>(context, tableOutputDir);<a name="line.85"></a>
+<span class="sourceLineNo">086</span> // Put table into map<a name="line.86"></a>
+<span class="sourceLineNo">087</span> tableWriters.put(tableName, tableWriter);<a name="line.87"></a>
+<span class="sourceLineNo">088</span> }<a name="line.88"></a>
+<span class="sourceLineNo">089</span> // Write <Row, Cell> into tableWriter<a name="line.89"></a>
+<span class="sourceLineNo">090</span> // in the original code, it does not use Row<a name="line.90"></a>
+<span class="sourceLineNo">091</span> tableWriter.write(null, cell);<a name="line.91"></a>
+<span class="sourceLineNo">092</span> }<a name="line.92"></a>
+<span class="sourceLineNo">093</span><a name="line.93"></a>
+<span class="sourceLineNo">094</span> @Override<a name="line.94"></a>
+<span class="sourceLineNo">095</span> public void close(TaskAttemptContext c) throws IOException, InterruptedException {<a name="line.95"></a>
+<span class="sourceLineNo">096</span> for (RecordWriter<ImmutableBytesWritable, V> writer : tableWriters.values()) {<a name="line.96"></a>
+<span class="sourceLineNo">097</span> writer.close(c);<a name="line.97"></a>
+<span class="sourceLineNo">098</span> }<a name="line.98"></a>
+<span class="sourceLineNo">099</span> }<a name="line.99"></a>
+<span class="sourceLineNo">100</span> };<a name="line.100"></a>
+<span class="sourceLineNo">101</span> }<a name="line.101"></a>
+<span class="sourceLineNo">102</span>}<a name="line.102"></a>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+</pre>
+</div>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/5f066b2e/book.html
----------------------------------------------------------------------
diff --git a/book.html b/book.html
index 2dee923..9b4d569 100644
--- a/book.html
+++ b/book.html
@@ -5292,15 +5292,6 @@ See <a href="https://issues.apache.org/jira/browse/HBASE-6389">HBASE-6389 Modify
starting region assignments</a> for more detail.</p>
</div>
</div>
-<div class="sect3">
-<h4 id="backup.master.fail.fast"><a class="anchor" href="#backup.master.fail.fast"></a>9.1.2. If a backup Master exists, make the primary Master fail fast</h4>
-<div class="paragraph">
-<p>If the primary Master loses its connection with ZooKeeper, it will fall into a loop where it keeps trying to reconnect.
-Disable this functionality if you are running more than one Master: i.e. a backup Master.
-Failing to do so, the dying Master may continue to receive RPCs though another Master has assumed the role of primary.
-See the configuration <a href="#fail.fast.expired.active.master">fail.fast.expired.active.master</a>.</p>
-</div>
-</div>
</div>
<div class="sect2">
<h3 id="recommended_configurations"><a class="anchor" href="#recommended_configurations"></a>9.2. Recommended Configurations</h3>
@@ -13986,9 +13977,9 @@ For background, see <a href="https://issues.apache.org/jira/browse/HBASE-2643">H
<div class="sect4">
<h5 id="_performance_improvements_during_log_splitting"><a class="anchor" href="#_performance_improvements_during_log_splitting"></a>Performance Improvements during Log Splitting</h5>
<div class="paragraph">
-<p>WAL log splitting and recovery can be resource intensive and take a long time, depending on the number of RegionServers involved in the crash and the size of the regions. <a href="#distributed.log.splitting">[distributed.log.splitting]</a> was developed to improve performance during log splitting.</p>
+<p>WAL log splitting and recovery can be resource intensive and take a long time, depending on the number of RegionServers involved in the crash and the size of the regions. <a href="#distributed.log.splitting">Enabling or Disabling Distributed Log Splitting</a> was developed to improve performance during log splitting.</p>
</div>
-<div class="paragraph">
+<div id="distributed.log.splitting" class="paragraph">
<div class="title">Enabling or Disabling Distributed Log Splitting</div>
<p>Distributed log processing is enabled by default since HBase 0.92.
The setting is controlled by the <code>hbase.master.distributed.log.splitting</code> property, which can be set to <code>true</code> or <code>false</code>, but defaults to <code>true</code>.</p>
@@ -14213,7 +14204,7 @@ A split log worker directly replays edits from the WAL of the failed RegionServe
When a region is in "recovering" state, it can accept writes but no reads (including Append and Increment), region splits or merges.</p>
</div>
<div class="paragraph">
-<p>Distributed Log Replay extends the <a href="#distributed.log.splitting">[distributed.log.splitting]</a> framework.
+<p>Distributed Log Replay extends the <a href="#distributed.log.splitting">Enabling or Disabling Distributed Log Splitting</a> framework.
It works by directly replaying WAL edits to another RegionServer instead of creating <em>recovered.edits</em> files.
It provides the following advantages over distributed log splitting alone:</p>
</div>
@@ -19862,9 +19853,10 @@ an easier way to load a coprocessor dynamically.</p>
<div class="listingblock">
<div class="content">
<pre class="CodeRay highlight"><code data-lang="java">TableName tableName = TableName.valueOf(<span class="string"><span class="delimiter">"</span><span class="content">users</span><span class="delimiter">"</span></span>);
-<span class="predefined-type">String</span> path = <span class="string"><span class="delimiter">"</span><span class="content">hdfs://<namenode>:<port>/user/<hadoop-user>/coprocessor.jar</span><span class="delimiter">"</span></span>;
+Path path = <span class="keyword">new</span> Path(<span class="string"><span class="delimiter">"</span><span class="content">hdfs://<namenode>:<port>/user/<hadoop-user>/coprocessor.jar</span><span class="delimiter">"</span></span>);
<span class="predefined-type">Configuration</span> conf = HBaseConfiguration.create();
-HBaseAdmin admin = <span class="keyword">new</span> HBaseAdmin(conf);
+<span class="predefined-type">Connection</span> connection = ConnectionFactory.createConnection(conf);
+Admin admin = connection.getAdmin();
admin.disableTable(tableName);
HTableDescriptor hTableDescriptor = <span class="keyword">new</span> HTableDescriptor(tableName);
HColumnDescriptor columnFamily1 = <span class="keyword">new</span> HColumnDescriptor(<span class="string"><span class="delimiter">"</span><span class="content">personalDet</span><span class="delimiter">"</span></span>);
@@ -20081,23 +20073,15 @@ empty result. Otherwise, process the request as normal.</p>
<span class="directive">private</span> <span class="directive">static</span> <span class="directive">final</span> <span class="type">byte</span><span class="type">[]</span> VALUE = Bytes.toBytes(<span class="string"><span class="delimiter">"</span><span class="content">You can't see Admin details</span><span class="delimiter">"</span></span>);
<span class="annotation">@Override</span>
- <span class="directive">public</span> <span class="type">void</span> preGetOp(<span class="directive">final</span> ObserverContext e, <span class="directive">final</span> Get get, <span class="directive">final</span> <span class="predefined-type">List</span> results)
+ <span class="directive">public</span> <span class="type">void</span> preGetOp(<span class="directive">final</span> ObserverContext<RegionCoprocessorEnvironment> e, <span class="directive">final</span> Get get, <span class="directive">final</span> <span class="predefined-type">List</span><Cell> results)
<span class="directive">throws</span> <span class="exception">IOException</span> {
<span class="keyword">if</span> (Bytes.equals(get.getRow(),ADMIN)) {
- Cell c = CellUtil.createCell(get.getRow(),COLUMN _FAMILY, COLUMN,
+ Cell c = CellUtil.createCell(get.getRow(),COLUMN_FAMILY, COLUMN,
<span class="predefined-type">System</span>.currentTimeMillis(), (<span class="type">byte</span>)<span class="integer">4</span>, VALUE);
results.add(c);
e.bypass();
}
-
- <span class="predefined-type">List</span> kvs = <span class="keyword">new</span> <span class="predefined-type">ArrayList</span>(results.size());
- <span class="keyword">for</span> (Cell c : results) {
- kvs.add(KeyValueUtil.ensureKeyValue(c));
- }
- preGet(e, get, kvs);
- results.clear();
- results.addAll(kvs);
}
}</code></pre>
</div>
@@ -20109,7 +20093,7 @@ the <code>preScannerOpen()</code> method to filter the <code>admin</code> row fr
<div class="listingblock">
<div class="content">
<pre class="CodeRay highlight"><code data-lang="java"><span class="annotation">@Override</span>
-<span class="directive">public</span> RegionScanner preScannerOpen(<span class="directive">final</span> ObserverContext e, <span class="directive">final</span> Scan scan,
+<span class="directive">public</span> RegionScanner preScannerOpen(<span class="directive">final</span> ObserverContext<RegionCoprocessorEnvironment> e, <span class="directive">final</span> Scan scan,
<span class="directive">final</span> RegionScanner s) <span class="directive">throws</span> <span class="exception">IOException</span> {
<span class="predefined-type">Filter</span> filter = <span class="keyword">new</span> RowFilter(CompareOp.NOT_EQUAL, <span class="keyword">new</span> BinaryComparator(ADMIN));
@@ -20126,10 +20110,10 @@ remove any <code>admin</code> results from the scan:</p>
<div class="listingblock">
<div class="content">
<pre class="CodeRay highlight"><code data-lang="java"><span class="annotation">@Override</span>
-<span class="directive">public</span> <span class="type">boolean</span> postScannerNext(<span class="directive">final</span> ObserverContext e, <span class="directive">final</span> InternalScanner s,
-<span class="directive">final</span> <span class="predefined-type">List</span> results, <span class="directive">final</span> <span class="type">int</span> limit, <span class="directive">final</span> <span class="type">boolean</span> hasMore) <span class="directive">throws</span> <span class="exception">IOException</span> {
+<span class="directive">public</span> <span class="type">boolean</span> postScannerNext(<span class="directive">final</span> ObserverContext<RegionCoprocessorEnvironment> e, <span class="directive">final</span> InternalScanner s,
+<span class="directive">final</span> <span class="predefined-type">List</span><<span class="predefined-type">Result</span>> results, <span class="directive">final</span> <span class="type">int</span> limit, <span class="directive">final</span> <span class="type">boolean</span> hasMore) <span class="directive">throws</span> <span class="exception">IOException</span> {
<span class="predefined-type">Result</span> result = <span class="predefined-constant">null</span>;
- <span class="predefined-type">Iterator</span> iterator = results.iterator();
+ <span class="predefined-type">Iterator</span><<span class="predefined-type">Result</span>> iterator = results.iterator();
<span class="keyword">while</span> (iterator.hasNext()) {
result = iterator.next();
<span class="keyword">if</span> (Bytes.equals(result.getRow(), ROWKEY)) {
@@ -34075,7 +34059,7 @@ The server will return cellblocks compressed using this same compressor as long
<div id="footer">
<div id="footer-text">
Version 2.0.0-SNAPSHOT<br>
-Last updated 2016-06-15 16:11:32 UTC
+Last updated 2016-04-10 14:30:44 UTC
</div>
</div>
</body>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/5f066b2e/bulk-loads.html
----------------------------------------------------------------------
diff --git a/bulk-loads.html b/bulk-loads.html
index b4c1ea7..bfe3494 100644
--- a/bulk-loads.html
+++ b/bulk-loads.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20160714" />
+ <meta name="Date-Revision-yyyymmdd" content="20160717" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase –
Bulk Loads in Apache HBase (TM)
@@ -305,7 +305,7 @@ under the License. -->
<a href="http://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2016-07-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2016-07-17</li>
</p>
</div>